Skip to content

Commit

Permalink
blkcg: move statistics update code to policies
Browse files Browse the repository at this point in the history 
As with conf/stats file handling code, there's no reason for stat
update code to live in blkcg core with policies calling into update
them.  The current organization is both inflexible and complex.

This patch moves stat update code to specific policies.  All
blkiocg_update_*_stats() functions which deal with BLKIO_POLICY_PROP
stats are collapsed into their cfq_blkiocg_update_*_stats()
counterparts.  blkiocg_update_dispatch_stats() is used by both
policies and duplicated as throtl_update_dispatch_stats() and
cfq_blkiocg_update_dispatch_stats().  This will be cleaned up later.

Signed-off-by: Tejun Heo <[email protected]>
  • Loading branch information
@htejun
htejun committed Apr 1, 2012
1 parent 2ce4d50 commit 629ed0b
Show file tree
Hide file tree
Showing 4 changed files with 259 additions and 397 deletions.
245 changes: 0 additions & 245 deletions block/blk-cgroup.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,251 +63,6 @@ struct blkio_cgroup *bio_blkio_cgroup(struct bio *bio)
}
EXPORT_SYMBOL_GPL(bio_blkio_cgroup);

#ifdef CONFIG_DEBUG_BLK_CGROUP
/* This should be called with the queue_lock held. */
static void blkio_set_start_group_wait_time(struct blkio_group *blkg,
struct blkio_policy_type *pol,
struct blkio_group *curr_blkg)
{
struct blkg_policy_data *pd = blkg->pd[pol->plid];

if (blkio_blkg_waiting(&pd->stats))
return;
if (blkg == curr_blkg)
return;
pd->stats.start_group_wait_time = sched_clock();
blkio_mark_blkg_waiting(&pd->stats);
}

/* This should be called with the queue_lock held. */
static void blkio_update_group_wait_time(struct blkio_group_stats *stats)
{
unsigned long long now;

if (!blkio_blkg_waiting(stats))
return;

now = sched_clock();
if (time_after64(now, stats->start_group_wait_time))
blkg_stat_add(&stats->group_wait_time,
now - stats->start_group_wait_time);
blkio_clear_blkg_waiting(stats);
}

/* This should be called with the queue_lock held. */
static void blkio_end_empty_time(struct blkio_group_stats *stats)
{
unsigned long long now;

if (!blkio_blkg_empty(stats))
return;

now = sched_clock();
if (time_after64(now, stats->start_empty_time))
blkg_stat_add(&stats->empty_time,
now - stats->start_empty_time);
blkio_clear_blkg_empty(stats);
}

void blkiocg_update_set_idle_time_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol)
{
struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;

lockdep_assert_held(blkg->q->queue_lock);
BUG_ON(blkio_blkg_idling(stats));

stats->start_idle_time = sched_clock();
blkio_mark_blkg_idling(stats);
}
EXPORT_SYMBOL_GPL(blkiocg_update_set_idle_time_stats);

void blkiocg_update_idle_time_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol)
{
struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;

lockdep_assert_held(blkg->q->queue_lock);

if (blkio_blkg_idling(stats)) {
unsigned long long now = sched_clock();

if (time_after64(now, stats->start_idle_time))
blkg_stat_add(&stats->idle_time,
now - stats->start_idle_time);
blkio_clear_blkg_idling(stats);
}
}
EXPORT_SYMBOL_GPL(blkiocg_update_idle_time_stats);

void blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol)
{
struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;

lockdep_assert_held(blkg->q->queue_lock);

blkg_stat_add(&stats->avg_queue_size_sum,
blkg_rwstat_sum(&stats->queued));
blkg_stat_add(&stats->avg_queue_size_samples, 1);
blkio_update_group_wait_time(stats);
}
EXPORT_SYMBOL_GPL(blkiocg_update_avg_queue_size_stats);

void blkiocg_set_start_empty_time(struct blkio_group *blkg,
struct blkio_policy_type *pol)
{
struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;

lockdep_assert_held(blkg->q->queue_lock);

if (blkg_rwstat_sum(&stats->queued))
return;

/*
* group is already marked empty. This can happen if cfqq got new
* request in parent group and moved to this group while being added
* to service tree. Just ignore the event and move on.
*/
if (blkio_blkg_empty(stats))
return;

stats->start_empty_time = sched_clock();
blkio_mark_blkg_empty(stats);
}
EXPORT_SYMBOL_GPL(blkiocg_set_start_empty_time);

void blkiocg_update_dequeue_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol,
unsigned long dequeue)
{
struct blkg_policy_data *pd = blkg->pd[pol->plid];

lockdep_assert_held(blkg->q->queue_lock);

blkg_stat_add(&pd->stats.dequeue, dequeue);
}
EXPORT_SYMBOL_GPL(blkiocg_update_dequeue_stats);
#else
static inline void blkio_set_start_group_wait_time(struct blkio_group *blkg,
struct blkio_policy_type *pol,
struct blkio_group *curr_blkg) { }
static inline void blkio_end_empty_time(struct blkio_group_stats *stats) { }
#endif

void blkiocg_update_io_add_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol,
struct blkio_group *curr_blkg, bool direction,
bool sync)
{
struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);

lockdep_assert_held(blkg->q->queue_lock);

blkg_rwstat_add(&stats->queued, rw, 1);
blkio_end_empty_time(stats);
blkio_set_start_group_wait_time(blkg, pol, curr_blkg);
}
EXPORT_SYMBOL_GPL(blkiocg_update_io_add_stats);

void blkiocg_update_io_remove_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol,
bool direction, bool sync)
{
struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);

lockdep_assert_held(blkg->q->queue_lock);

blkg_rwstat_add(&stats->queued, rw, -1);
}
EXPORT_SYMBOL_GPL(blkiocg_update_io_remove_stats);

void blkiocg_update_timeslice_used(struct blkio_group *blkg,
struct blkio_policy_type *pol,
unsigned long time,
unsigned long unaccounted_time)
{
struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;

lockdep_assert_held(blkg->q->queue_lock);

blkg_stat_add(&stats->time, time);
#ifdef CONFIG_DEBUG_BLK_CGROUP
blkg_stat_add(&stats->unaccounted_time, unaccounted_time);
#endif
}
EXPORT_SYMBOL_GPL(blkiocg_update_timeslice_used);

/*
* should be called under rcu read lock or queue lock to make sure blkg pointer
* is valid.
*/
void blkiocg_update_dispatch_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol,
uint64_t bytes, bool direction, bool sync)
{
int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
struct blkg_policy_data *pd = blkg->pd[pol->plid];
struct blkio_group_stats_cpu *stats_cpu;
unsigned long flags;

/* If per cpu stats are not allocated yet, don't do any accounting. */
if (pd->stats_cpu == NULL)
return;

/*
* Disabling interrupts to provide mutual exclusion between two
* writes on same cpu. It probably is not needed for 64bit. Not
* optimizing that case yet.
*/
local_irq_save(flags);

stats_cpu = this_cpu_ptr(pd->stats_cpu);

blkg_stat_add(&stats_cpu->sectors, bytes >> 9);
blkg_rwstat_add(&stats_cpu->serviced, rw, 1);
blkg_rwstat_add(&stats_cpu->service_bytes, rw, bytes);

local_irq_restore(flags);
}
EXPORT_SYMBOL_GPL(blkiocg_update_dispatch_stats);

void blkiocg_update_completion_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol,
uint64_t start_time,
uint64_t io_start_time, bool direction,
bool sync)
{
struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
unsigned long long now = sched_clock();
int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);

lockdep_assert_held(blkg->q->queue_lock);

if (time_after64(now, io_start_time))
blkg_rwstat_add(&stats->service_time, rw, now - io_start_time);
if (time_after64(io_start_time, start_time))
blkg_rwstat_add(&stats->wait_time, rw,
io_start_time - start_time);
}
EXPORT_SYMBOL_GPL(blkiocg_update_completion_stats);

/* Merged stats are per cpu. */
void blkiocg_update_io_merged_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol,
bool direction, bool sync)
{
struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);

lockdep_assert_held(blkg->q->queue_lock);

blkg_rwstat_add(&stats->merged, rw, 1);
}
EXPORT_SYMBOL_GPL(blkiocg_update_io_merged_stats);

/*
* Worker for allocating per cpu stat for blk groups. This is scheduled on
* the system_nrt_wq once there are some groups on the alloc_list waiting
Expand Down
94 changes: 0 additions & 94 deletions block/blk-cgroup.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,6 @@ enum blkg_rwstat_type {
BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,
};

/* blkg state flags */
enum blkg_state_flags {
BLKG_waiting = 0,
BLKG_idling,
BLKG_empty,
};

struct blkio_cgroup {
struct cgroup_subsys_state css;
unsigned int weight;
Expand Down Expand Up @@ -416,52 +409,6 @@ static inline void blkg_put(struct blkio_group *blkg) { }
#define BLKIO_WEIGHT_MAX 1000
#define BLKIO_WEIGHT_DEFAULT 500

#ifdef CONFIG_DEBUG_BLK_CGROUP
void blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol);
void blkiocg_update_dequeue_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol,
unsigned long dequeue);
void blkiocg_update_set_idle_time_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol);
void blkiocg_update_idle_time_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol);
void blkiocg_set_start_empty_time(struct blkio_group *blkg,
struct blkio_policy_type *pol);

#define BLKG_FLAG_FNS(name) \
static inline void blkio_mark_blkg_##name( \
struct blkio_group_stats *stats) \
{ \
stats->flags |= (1 << BLKG_##name); \
} \
static inline void blkio_clear_blkg_##name( \
struct blkio_group_stats *stats) \
{ \
stats->flags &= ~(1 << BLKG_##name); \
} \
static inline int blkio_blkg_##name(struct blkio_group_stats *stats) \
{ \
return (stats->flags & (1 << BLKG_##name)) != 0; \
} \

BLKG_FLAG_FNS(waiting)
BLKG_FLAG_FNS(idling)
BLKG_FLAG_FNS(empty)
#undef BLKG_FLAG_FNS
#else
static inline void blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol) { }
static inline void blkiocg_update_dequeue_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol, unsigned long dequeue) { }
static inline void blkiocg_update_set_idle_time_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol) { }
static inline void blkiocg_update_idle_time_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol) { }
static inline void blkiocg_set_start_empty_time(struct blkio_group *blkg,
struct blkio_policy_type *pol) { }
#endif

#ifdef CONFIG_BLK_CGROUP
extern struct blkio_cgroup blkio_root_cgroup;
extern struct blkio_cgroup *cgroup_to_blkio_cgroup(struct cgroup *cgroup);
Expand All @@ -471,28 +418,6 @@ extern struct blkio_group *blkg_lookup(struct blkio_cgroup *blkcg,
struct blkio_group *blkg_lookup_create(struct blkio_cgroup *blkcg,
struct request_queue *q,
bool for_root);
void blkiocg_update_timeslice_used(struct blkio_group *blkg,
struct blkio_policy_type *pol,
unsigned long time,
unsigned long unaccounted_time);
void blkiocg_update_dispatch_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol,
uint64_t bytes, bool direction, bool sync);
void blkiocg_update_completion_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol,
uint64_t start_time,
uint64_t io_start_time, bool direction,
bool sync);
void blkiocg_update_io_merged_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol,
bool direction, bool sync);
void blkiocg_update_io_add_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol,
struct blkio_group *curr_blkg, bool direction,
bool sync);
void blkiocg_update_io_remove_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol,
bool direction, bool sync);
#else
struct cgroup;
static inline struct blkio_cgroup *
Expand All @@ -502,24 +427,5 @@ bio_blkio_cgroup(struct bio *bio) { return NULL; }

static inline struct blkio_group *blkg_lookup(struct blkio_cgroup *blkcg,
void *key) { return NULL; }
static inline void blkiocg_update_timeslice_used(struct blkio_group *blkg,
struct blkio_policy_type *pol, unsigned long time,
unsigned long unaccounted_time) { }
static inline void blkiocg_update_dispatch_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol, uint64_t bytes,
bool direction, bool sync) { }
static inline void blkiocg_update_completion_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol, uint64_t start_time,
uint64_t io_start_time, bool direction, bool sync) { }
static inline void blkiocg_update_io_merged_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol, bool direction,
bool sync) { }
static inline void blkiocg_update_io_add_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol,
struct blkio_group *curr_blkg, bool direction,
bool sync) { }
static inline void blkiocg_update_io_remove_stats(struct blkio_group *blkg,
struct blkio_policy_type *pol, bool direction,
bool sync) { }
#endif
#endif /* _BLK_CGROUP_H */
Loading

0 comments on commit 629ed0b

Please sign in to comment.