Skip to content

Commit

Permalink
blk-iocost: combine local_stat and desc_stat to stat
Browse files Browse the repository at this point in the history 
When we flush usage, wait, indebt stat in iocg_flush_stat(), we use
local_stat and desc_stat, which has no point since the leaf iocg
only has local_stat and the inner iocg only has desc_stat. Also
we don't need to flush percpu abs_vusage for these inner iocgs.

This patch combine local_stat and desc_stat to stat, only flush
percpu abs_vusage for active leaf iocgs, then build inner walk
list to propagate.

Signed-off-by: Chengming Zhou <[email protected]>
Acked-by: Tejun Heo <[email protected]>
Link: https://lore.kernel.org/r/[email protected]
Signed-off-by: Jens Axboe <[email protected]>
  • Loading branch information
@axboe
Chengming Zhou authored and axboe committed May 11, 2022
1 parent 069adba commit 2a371f7
Showing 1 changed file with 32 additions and 39 deletions.
71 changes: 32 additions & 39 deletions block/blk-iocost.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -533,8 +533,7 @@ struct ioc_gq {

/* statistics */
struct iocg_pcpu_stat __percpu *pcpu_stat;
struct iocg_stat local_stat;
struct iocg_stat desc_stat;
struct iocg_stat stat;
struct iocg_stat last_stat;
u64 last_stat_abs_vusage;
u64 usage_delta_us;
Expand Down Expand Up @@ -1371,7 +1370,7 @@ static bool iocg_kick_delay(struct ioc_gq *iocg, struct ioc_now *now)
return true;
} else {
if (iocg->indelay_since) {
iocg->local_stat.indelay_us += now->now - iocg->indelay_since;
iocg->stat.indelay_us += now->now - iocg->indelay_since;
iocg->indelay_since = 0;
}
iocg->delay = 0;
Expand Down Expand Up @@ -1419,7 +1418,7 @@ static void iocg_pay_debt(struct ioc_gq *iocg, u64 abs_vpay,

/* if debt is paid in full, restore inuse */
if (!iocg->abs_vdebt) {
iocg->local_stat.indebt_us += now->now - iocg->indebt_since;
iocg->stat.indebt_us += now->now - iocg->indebt_since;
iocg->indebt_since = 0;

propagate_weights(iocg, iocg->active, iocg->last_inuse,
Expand Down Expand Up @@ -1513,7 +1512,7 @@ static void iocg_kick_waitq(struct ioc_gq *iocg, bool pay_debt,

if (!waitqueue_active(&iocg->waitq)) {
if (iocg->wait_since) {
iocg->local_stat.wait_us += now->now - iocg->wait_since;
iocg->stat.wait_us += now->now - iocg->wait_since;
iocg->wait_since = 0;
}
return;
Expand Down Expand Up @@ -1641,11 +1640,30 @@ static void iocg_build_inner_walk(struct ioc_gq *iocg,
}
}

/* propagate the deltas to the parent */
static void iocg_flush_stat_upward(struct ioc_gq *iocg)
{
if (iocg->level > 0) {
struct iocg_stat *parent_stat =
&iocg->ancestors[iocg->level - 1]->stat;

parent_stat->usage_us +=
iocg->stat.usage_us - iocg->last_stat.usage_us;
parent_stat->wait_us +=
iocg->stat.wait_us - iocg->last_stat.wait_us;
parent_stat->indebt_us +=
iocg->stat.indebt_us - iocg->last_stat.indebt_us;
parent_stat->indelay_us +=
iocg->stat.indelay_us - iocg->last_stat.indelay_us;
}

iocg->last_stat = iocg->stat;
}

/* collect per-cpu counters and propagate the deltas to the parent */
static void iocg_flush_stat_one(struct ioc_gq *iocg, struct ioc_now *now)
static void iocg_flush_stat_leaf(struct ioc_gq *iocg, struct ioc_now *now)
{
struct ioc *ioc = iocg->ioc;
struct iocg_stat new_stat;
u64 abs_vusage = 0;
u64 vusage_delta;
int cpu;
Expand All @@ -1661,34 +1679,9 @@ static void iocg_flush_stat_one(struct ioc_gq *iocg, struct ioc_now *now)
iocg->last_stat_abs_vusage = abs_vusage;

iocg->usage_delta_us = div64_u64(vusage_delta, ioc->vtime_base_rate);
iocg->local_stat.usage_us += iocg->usage_delta_us;

/* propagate upwards */
new_stat.usage_us =
iocg->local_stat.usage_us + iocg->desc_stat.usage_us;
new_stat.wait_us =
iocg->local_stat.wait_us + iocg->desc_stat.wait_us;
new_stat.indebt_us =
iocg->local_stat.indebt_us + iocg->desc_stat.indebt_us;
new_stat.indelay_us =
iocg->local_stat.indelay_us + iocg->desc_stat.indelay_us;

/* propagate the deltas to the parent */
if (iocg->level > 0) {
struct iocg_stat *parent_stat =
&iocg->ancestors[iocg->level - 1]->desc_stat;

parent_stat->usage_us +=
new_stat.usage_us - iocg->last_stat.usage_us;
parent_stat->wait_us +=
new_stat.wait_us - iocg->last_stat.wait_us;
parent_stat->indebt_us +=
new_stat.indebt_us - iocg->last_stat.indebt_us;
parent_stat->indelay_us +=
new_stat.indelay_us - iocg->last_stat.indelay_us;
}
iocg->stat.usage_us += iocg->usage_delta_us;

iocg->last_stat = new_stat;
iocg_flush_stat_upward(iocg);
}

/* get stat counters ready for reading on all active iocgs */
Expand All @@ -1699,13 +1692,13 @@ static void iocg_flush_stat(struct list_head *target_iocgs, struct ioc_now *now)

/* flush leaves and build inner node walk list */
list_for_each_entry(iocg, target_iocgs, active_list) {
iocg_flush_stat_one(iocg, now);
iocg_flush_stat_leaf(iocg, now);
iocg_build_inner_walk(iocg, &inner_walk);
}

/* keep flushing upwards by walking the inner list backwards */
list_for_each_entry_safe_reverse(iocg, tiocg, &inner_walk, walk_list) {
iocg_flush_stat_one(iocg, now);
iocg_flush_stat_upward(iocg);
list_del_init(&iocg->walk_list);
}
}
Expand Down Expand Up @@ -2152,16 +2145,16 @@ static int ioc_check_iocgs(struct ioc *ioc, struct ioc_now *now)

/* flush wait and indebt stat deltas */
if (iocg->wait_since) {
iocg->local_stat.wait_us += now->now - iocg->wait_since;
iocg->stat.wait_us += now->now - iocg->wait_since;
iocg->wait_since = now->now;
}
if (iocg->indebt_since) {
iocg->local_stat.indebt_us +=
iocg->stat.indebt_us +=
now->now - iocg->indebt_since;
iocg->indebt_since = now->now;
}
if (iocg->indelay_since) {
iocg->local_stat.indelay_us +=
iocg->stat.indelay_us +=
now->now - iocg->indelay_since;
iocg->indelay_since = now->now;
}
Expand Down

0 comments on commit 2a371f7

Please sign in to comment.