Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Added functions to control the cache eviction #31

Open
wants to merge 4 commits into
base: tuner
Choose a base branch
from
Open

Added functions to control the cache eviction #31

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9426503
Updated the comment on logging
gobinathsr Nov 6, 2020
b4cd3dd
Revert "Updated comment to specify logging details"
gobinathsr Nov 6, 2020
e366246
Updating the comment to track logging info
gobinathsr Nov 6, 2020
5efbe8e
Root Cause:
gobinathsr Nov 6, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions include/spdk/nvmf.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,8 @@ struct spdk_nvmf_io_pacer_stat {
uint64_t calls;
uint64_t no_ios;
uint64_t period_ticks;
uint64_t nos_credit_unavailable;
uint64_t nos_bytes_above_allowed;
};

struct spdk_nvmf_transport_poll_group_stat {
Expand Down
117 changes: 94 additions & 23 deletions lib/nvmf/io_pacer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ struct io_pacer_queue {
STAILQ_HEAD(, io_pacer_queue_entry) queue;
};

struct spdk_io_pacer_tuner;
struct spdk_io_pacer_tuner2;
struct spdk_io_pacer {
uint64_t period_ticks;
int64_t credit;
Expand All @@ -72,6 +74,12 @@ struct spdk_io_pacer {
struct io_pacer_queue *queues;
struct spdk_poller *poller;
uint32_t disk_credit;
uint32_t pacer_tuner_type;
uint32_t pacer_tuner_type_proposal;
union {
struct spdk_io_pacer_tuner *pacer_tuner;
struct spdk_io_pacer_tuner2 *pacer_tuner2;
};
};

struct spdk_io_pacer_tuner {
Expand Down Expand Up @@ -127,22 +135,19 @@ io_pacer_poll(void *arg)
#endif /* SPDK_CONFIG_VTUNE */

pacer->stat.calls++;
if (ticks_diff < pacer->period_ticks) {
return 0;
}
pacer->stat.total_ticks = cur_tick - pacer->first_tick;
pacer->last_tick = cur_tick - ticks_diff % pacer->period_ticks;
pacer->stat.polls++;

pacer->remaining_credit = spdk_min(pacer->remaining_credit + pacer->credit,
pacer->credit);

if (pacer->num_ios == 0) {
pacer->stat.no_ios++;
}
if (ticks_diff >= pacer->period_ticks){
pacer->stat.total_ticks = cur_tick - pacer->first_tick;
pacer->last_tick = cur_tick - ticks_diff % pacer->period_ticks;
pacer->stat.polls++;
pacer->remaining_credit = spdk_min(pacer->remaining_credit + pacer->credit,
pacer->credit);
if (pacer->num_ios == 0) {
pacer->stat.no_ios++;
}
}

while ((pacer->num_ios > 0) &&
(pacer->remaining_credit > 0) &&
(is_credit_available(pacer) == true) &&
(attempts_cnt < pacer->num_queues)) {
next_queue %= pacer->num_queues;
attempts_cnt++;
Expand All @@ -157,6 +162,12 @@ io_pacer_poll(void *arg)
entry = STAILQ_FIRST(&pacer->queues[next_queue].queue);
next_queue++;
if (entry != NULL) {
#if SPDK_NVMF_RDMA_IO_PACER_ALLOW_IO_WITHIN_AVAIL
if (!is_entry_size_within_limit(pacer, entry))
{
break;
}
#endif
STAILQ_REMOVE_HEAD(&pacer->queues[next_queue - 1].queue, link);
pacer->num_ios--;
pacer->next_queue = next_queue;
Expand All @@ -182,7 +193,8 @@ struct spdk_io_pacer *
spdk_io_pacer_create(uint32_t period_ns,
uint32_t credit,
uint32_t disk_credit,
spdk_io_pacer_pop_cb pop_cb)
spdk_io_pacer_pop_cb pop_cb,
uint32_t io_pacer_tuner_type)
{
struct spdk_io_pacer *pacer;

Expand All @@ -201,21 +213,24 @@ spdk_io_pacer_create(uint32_t period_ns,
pacer->pop_cb = pop_cb;
pacer->first_tick = spdk_get_ticks();
pacer->last_tick = spdk_get_ticks();
pacer->stat.nos_credit_unavailable = 0;
pacer->stat.nos_bytes_above_allowed = 0;
pacer->poller = SPDK_POLLER_REGISTER(io_pacer_poll, (void *)pacer, 0);
if (!pacer->poller) {
SPDK_ERRLOG("Failed to create poller for IO pacer\n");
spdk_io_pacer_destroy(pacer);
return NULL;
}

SPDK_NOTICELOG("Created IO pacer %p: period_ns %u, period_ticks %lu, max_queues %u, credit %ld, disk_credit %u, core %u\n",
SPDK_NOTICELOG("Created IO pacer %p: period_ns %u, period_ticks %lu, max_queues %u, credit %ld, disk_credit %u, core %u, Tuner type: %u\n",
pacer,
period_ns,
pacer->period_ticks,
pacer->max_queues,
pacer->credit,
pacer->disk_credit,
spdk_env_get_current_core());
spdk_env_get_current_core(),
io_pacer_tuner_type);

return pacer;
}
Expand Down Expand Up @@ -395,7 +410,7 @@ io_pacer_tune(void *arg)
tuner->min_pacer_period_ticks);

static __thread uint32_t log_counter = 0;
/* Try to log once per second */
/* Try to log once per second so as to reduce num of traces */
if (log_counter % (SPDK_SEC_TO_NSEC / tuner->period_ns) == 0) {
SPDK_NOTICELOG("IO pacer tuner %p: pacer %p, bytes %lu, io period %lu ns, new period %lu ns, new period %lu ticks, min %lu, max %lu\n",
tuner,
Expand Down Expand Up @@ -435,6 +450,9 @@ spdk_io_pacer_tuner_create(struct spdk_io_pacer *pacer,
tuner->step_ns = step_ns;
tuner->min_pacer_period_ticks = pacer->period_ticks;
tuner->max_pacer_period_ticks = 2 * tuner->min_pacer_period_ticks;
pacer->pacer_tuner_type = SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_01;
pacer->pacer_tuner_type_proposal = 0;
pacer->pacer_tuner = tuner;

if (0 != period_us) {
tuner->poller = SPDK_POLLER_REGISTER(io_pacer_tune, (void *)tuner, period_us);
Expand Down Expand Up @@ -475,6 +493,7 @@ struct spdk_io_pacer_tuner2 {
uint64_t min_pacer_period_ticks;
uint64_t max_pacer_period_ticks;
struct spdk_poller *poller;
uint64_t min_threshold_offset;
};

static int
Expand All @@ -490,16 +509,18 @@ io_pacer_tune2(void *arg)
new_period_ticks = spdk_min(new_period_ticks, tuner->max_pacer_period_ticks);

static __thread uint32_t log_counter = 0;
/* Try to log once per second, else it would be too much log */
/* Try to log once per second, to limit the log */
if (log_counter % (SPDK_SEC_TO_NSEC / tuner->period_ns) == 0) {
SPDK_NOTICELOG("IO pacer tuner %p: pacer %p, value %u, new period %lu ticks, min %lu, polls %u. ios %u\n",
SPDK_NOTICELOG("IO pacer tuner %p: pacer %p, value %u, new period %lu ticks, min %lu, polls %lu. ios %lu, no credit %lu, abv allow %lu\n",
tuner,
pacer,
v,
new_period_ticks,
tuner->min_pacer_period_ticks,
pacer->stat.polls,
pacer->stat.ios);
pacer->stat.ios,
pacer->stat.nos_credit_unavailable,
pacer->stat.nos_bytes_above_allowed);
}
log_counter++;

Expand All @@ -515,6 +536,7 @@ spdk_io_pacer_tuner2_create(struct spdk_io_pacer *pacer,
uint64_t factor)
{
struct spdk_io_pacer_tuner2 *tuner;
uint32_t min_threshold_margin;

assert(pacer != NULL);

Expand All @@ -531,6 +553,12 @@ spdk_io_pacer_tuner2_create(struct spdk_io_pacer *pacer,
tuner->factor = factor;
tuner->min_pacer_period_ticks = pacer->period_ticks;
tuner->max_pacer_period_ticks = 4 * tuner->min_pacer_period_ticks;
pacer->pacer_tuner_type = SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_02;
pacer->pacer_tuner_type_proposal = 0;
pacer->pacer_tuner2 = tuner;

min_threshold_margin = 100 / SPDK_NVMF_RDMA_IO_PACER_DEFAULT_MARGIN_ABOVE_CREDIT; /* variable parameter */
tuner->min_threshold_offset = (tuner->min_threshold / min_threshold_margin);

if (0 != period_us) {
tuner->poller = SPDK_POLLER_REGISTER(io_pacer_tune2, (void *)tuner, period_us);
Expand All @@ -541,12 +569,13 @@ spdk_io_pacer_tuner2_create(struct spdk_io_pacer *pacer,
}
}

SPDK_NOTICELOG("Created IO pacer tuner %p: pacer %p, period_ns %lu, threshold %u, factor %lu\n",
SPDK_NOTICELOG("Created IO pacer tuner %p: pacer %p, period_ns %lu, threshold %u, factor %lu, allowed offset %lu\n",
tuner,
pacer,
tuner->period_ns,
tuner->min_threshold,
tuner->factor);
tuner->factor,
tuner->min_threshold_offset);

return tuner;
}
Expand Down Expand Up @@ -574,3 +603,45 @@ spdk_io_pacer_tuner2_sub(struct spdk_io_pacer_tuner2 *tuner, uint32_t value)
assert(tuner != NULL);
tuner->value -= value;
}

bool
is_credit_available(struct spdk_io_pacer *pacer){
#if SPDK_NVMF_RDMA_IO_PACER_ALLOW_WITHIN_CREDIT_ONLY
/*
* Check the bytes in fight within allowed offset above the minimum allowed
* threshold value
*/
if (pacer->pacer_tuner_type == SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_02){
struct spdk_io_pacer_tuner2 *pacer_tuner = pacer->pacer_tuner2;
if ((pacer->remaining_credit > 0)&&
(pacer_tuner->value <= pacer_tuner->min_threshold + pacer_tuner->min_threshold_offset)){
return true;
}
} else if (pacer->remaining_credit > 0){
return true;
}
#else
else if (pacer->remaining_credit > 0){
return true;
}
#endif
pacer->stat.nos_credit_unavailable++;
return false;
}

#if SPDK_NVMF_RDMA_IO_PACER_ALLOW_IO_WITHIN_AVAIL
bool
is_entry_size_within_limit(struct spdk_io_pacer *pacer, struct io_pacer_queue_entry *entry)
{
if (pacer->pacer_tuner_type == SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_02){
if ((pacer_tuner->value + entry->size) <= (pacer_tuner->min_threshold + pacer_tuner->min_threshold_offset)){
return true;
} else {
pacer->stat.no_bytes_above_allowed++;
return false;
}
} else {
return true;
}
}
#endif
31 changes: 30 additions & 1 deletion lib/nvmf/io_pacer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,32 @@
#include "spdk_internal/log.h"
#include "spdk/nvmf.h"

/* Tuner types
*/
#define SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_01 01
#define SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_02 02
#define SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_03 03

#define SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_03_PROPOSAL_01 01 /* Bytes in flight */
#define SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_03_PROPOSAL_02 02 /* Avg Bytes in flght for N pacer periods */
#define SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_03_PROPOSAL_03 03 /* Read/Write Bytes in flight */
#define SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_03_PROPOSAL_04 04 /* Margin Bytes Range in flight */
#define SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_03_PROPOSAL_05 05 /* Margin proportionate Bytes Range in flight */
#define SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_03_PROPOSAL_06 06 /* Margin Inc/Dec w.r.t allowed bytes */
#define SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_03_PROPOSAL_07 07 /* Margin proportionate Inc/Dec w.r.t allowed bytes */

/*
* Configures credit availability to be calculated along with the
* bytes in flight within in the allowed margin offset
*/
#define SPDK_NVMF_RDMA_IO_PACER_ALLOW_WITHIN_CREDIT_ONLY 1
#define SPDK_NVMF_RDMA_IO_PACER_DEFAULT_MARGIN_ABOVE_CREDIT 1

/*
* Configure IO request to be processed only when within in the bytes flight
*/
#define SPDK_NVMF_RDMA_IO_PACER_ALLOW_IO_WITHIN_AVAIL 0

struct spdk_io_pacer;
struct spdk_io_pacer_tuner;
struct spdk_io_pacer_tuner2;
Expand All @@ -68,7 +94,8 @@ typedef void (*spdk_io_pacer_pop_cb)(void *io);
struct spdk_io_pacer *spdk_io_pacer_create(uint32_t period_ns,
uint32_t credit,
uint32_t disk_credit,
spdk_io_pacer_pop_cb pop_cb);
spdk_io_pacer_pop_cb pop_cb,
uint32_t io_pacer_tuner_type);
void spdk_io_pacer_destroy(struct spdk_io_pacer *pacer);
int spdk_io_pacer_create_queue(struct spdk_io_pacer *pacer, uint64_t key);
int spdk_io_pacer_destroy_queue(struct spdk_io_pacer *pacer, uint64_t key);
Expand All @@ -89,6 +116,8 @@ void spdk_io_pacer_tuner2_destroy(struct spdk_io_pacer_tuner2 *tuner);
void spdk_io_pacer_tuner2_add(struct spdk_io_pacer_tuner2 *tuner, uint32_t value);
void spdk_io_pacer_tuner2_sub(struct spdk_io_pacer_tuner2 *tuner, uint32_t value);

bool is_credit_available(struct spdk_io_pacer *pacer);

static inline void drive_stats_lock(struct spdk_io_pacer_drives_stats *stats) {
rte_spinlock_lock(&stats->lock);
}
Expand Down
15 changes: 9 additions & 6 deletions lib/nvmf/rdma.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2227,7 +2227,7 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport,
STAILQ_REMOVE_HEAD(&rgroup->group.pending_buf_queue, buf_link);


if (rgroup->pacer_tuner2 && (rtransport->transport.opts.io_pacer_tuner_type == 1)) {
if (rgroup->pacer_tuner2 && (rtransport->transport.opts.io_pacer_tuner_type == SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_02)) {
spdk_io_pacer_tuner2_add(rgroup->pacer_tuner2, rdma_req->req.length);
}

Expand Down Expand Up @@ -2406,7 +2406,7 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport,
}

if (rgroup->pacer_tuner2 &&
(rtransport->transport.opts.io_pacer_tuner_type == 1)) {
(rtransport->transport.opts.io_pacer_tuner_type == SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_02)) {
spdk_io_pacer_tuner2_sub(rgroup->pacer_tuner2, rdma_req->req.length);
}

Expand Down Expand Up @@ -2446,7 +2446,7 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport,
#define SPDK_NVMF_RDMA_DIF_INSERT_OR_STRIP false
#define SPDK_NVMF_RDMA_DEFAULT_IO_PACER_PERIOD 0
#define SPDK_NVMF_RDMA_DEFAULT_IO_PACER_THRESHOLD 0
#define SPDK_NVMF_RDMA_DEFAULT_IO_PACER_TUNER_TYPE 1 /* Buffers based tuner */
#define SPDK_NVMF_RDMA_DEFAULT_IO_PACER_TUNER_TYPE SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_02 /* Buffers based tuner */
#define SPDK_NVMF_RDMA_DEFAULT_IO_PACER_TUNER_PERIOD 10000 /* us */
#define SPDK_NVMF_RDMA_DEFAULT_IO_PACER_TUNER_STEP 1000 /* ns */
#define SPDK_NVMF_RDMA_DEFAULT_IO_PACER_TUNER_THRESHOLD 12*1024*1024
Expand Down Expand Up @@ -3561,15 +3561,17 @@ spdk_nvmf_rdma_poll_group_create(struct spdk_nvmf_transport *transport)
rgroup->pacer = spdk_io_pacer_create(transport->opts.io_pacer_period,
transport->opts.io_pacer_credit,
transport->opts.io_pacer_disk_credit,
nvmf_rdma_io_pacer_pop_cb);
nvmf_rdma_io_pacer_pop_cb,
transport->opts.io_pacer_tuner_type
);
if (!rgroup->pacer) {
SPDK_ERRLOG("Failed to create IO pacer\n");
spdk_nvmf_rdma_poll_group_destroy(&rgroup->group);
pthread_mutex_unlock(&rtransport->lock);
return NULL;
}

if (transport->opts.io_pacer_tuner_type == 0) {
if (transport->opts.io_pacer_tuner_type == SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_01) {
rgroup->pacer_tuner = spdk_io_pacer_tuner_create(rgroup->pacer,
transport->opts.io_pacer_tuner_period,
transport->opts.io_pacer_tuner_step);
Expand All @@ -3592,6 +3594,7 @@ spdk_nvmf_rdma_poll_group_create(struct spdk_nvmf_transport *transport)
return NULL;
}
}
SPDK_NOTICELOG("Created Tuner Type %u\n", transport->opts.io_pacer_tuner_type);
}

/* @todo: there is no good place to create queues. */
Expand Down Expand Up @@ -3718,7 +3721,7 @@ spdk_nvmf_rdma_poll_group_destroy(struct spdk_nvmf_transport_poll_group *group)
rtransport = SPDK_CONTAINEROF(rgroup->group.transport, struct spdk_nvmf_rdma_transport, transport);

if (rgroup->pacer) {
if (rtransport->transport.opts.io_pacer_tuner_type == 0) {
if (rtransport->transport.opts.io_pacer_tuner_type == SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_01) {
spdk_io_pacer_tuner_destroy(rgroup->pacer_tuner);
} else {
spdk_io_pacer_tuner2_destroy(rgroup->pacer_tuner2);
Expand Down