Skip to content

Commit

Permalink
blk-mq: use percpu_ref for mq usage count
Browse files Browse the repository at this point in the history 
Currently, blk-mq uses a percpu_counter to keep track of how many
usages are in flight.  The percpu_counter is drained while freezing to
ensure that no usage is left in-flight after freezing is complete.
blk_mq_queue_enter/exit() and blk_mq_[un]freeze_queue() implement this
per-cpu gating mechanism.

This type of code has relatively high chance of subtle bugs which are
extremely difficult to trigger and it's way too hairy to be open coded
in blk-mq.  percpu_ref can serve the same purpose after the recent
changes.  This patch replaces the open-coded per-cpu usage counting
and draining mechanism with percpu_ref.

blk_mq_queue_enter() performs tryget_live on the ref and exit()
performs put.  blk_mq_freeze_queue() kills the ref and waits until the
reference count reaches zero.  blk_mq_unfreeze_queue() revives the ref
and wakes up the waiters.

Signed-off-by: Tejun Heo <[email protected]>
Cc: Jens Axboe <[email protected]>
Cc: Nicholas A. Bellinger <[email protected]>
Cc: Kent Overstreet <[email protected]>
Signed-off-by: Jens Axboe <[email protected]>
  • Loading branch information
@htejun @axboe
htejun authored and axboe committed Jul 1, 2014
1 parent 72d6f02 commit add703f
Show file tree
Hide file tree
Showing 2 changed files with 31 additions and 40 deletions.
68 changes: 29 additions & 39 deletions block/blk-mq.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,34 +78,32 @@ static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,

static int blk_mq_queue_enter(struct request_queue *q)
{
int ret;

__percpu_counter_add(&q->mq_usage_counter, 1, 1000000);
smp_mb();

/* we have problems freezing the queue if it's initializing */
if (!q->mq_freeze_depth)
return 0;

__percpu_counter_add(&q->mq_usage_counter, -1, 1000000);
while (true) {
int ret;

spin_lock_irq(q->queue_lock);
ret = wait_event_interruptible_lock_irq(q->mq_freeze_wq,
!q->mq_freeze_depth || blk_queue_dying(q),
*q->queue_lock);
/* inc usage with lock hold to avoid freeze_queue runs here */
if (!ret && !blk_queue_dying(q))
__percpu_counter_add(&q->mq_usage_counter, 1, 1000000);
else if (blk_queue_dying(q))
ret = -ENODEV;
spin_unlock_irq(q->queue_lock);
if (percpu_ref_tryget_live(&q->mq_usage_counter))
return 0;

return ret;
ret = wait_event_interruptible(q->mq_freeze_wq,
!q->mq_freeze_depth || blk_queue_dying(q));
if (blk_queue_dying(q))
return -ENODEV;
if (ret)
return ret;
}
}

static void blk_mq_queue_exit(struct request_queue *q)
{
__percpu_counter_add(&q->mq_usage_counter, -1, 1000000);
percpu_ref_put(&q->mq_usage_counter);
}

static void blk_mq_usage_counter_release(struct percpu_ref *ref)
{
struct request_queue *q =
container_of(ref, struct request_queue, mq_usage_counter);

wake_up_all(&q->mq_freeze_wq);
}

/*
Expand All @@ -118,18 +116,9 @@ void blk_mq_freeze_queue(struct request_queue *q)
q->mq_freeze_depth++;
spin_unlock_irq(q->queue_lock);

while (true) {
s64 count;

spin_lock_irq(q->queue_lock);
count = percpu_counter_sum(&q->mq_usage_counter);
spin_unlock_irq(q->queue_lock);

if (count == 0)
break;
blk_mq_start_hw_queues(q);
msleep(10);
}
percpu_ref_kill(&q->mq_usage_counter);
blk_mq_run_queues(q, false);
wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->mq_usage_counter));
}

static void blk_mq_unfreeze_queue(struct request_queue *q)
Expand All @@ -140,8 +129,10 @@ static void blk_mq_unfreeze_queue(struct request_queue *q)
wake = !--q->mq_freeze_depth;
WARN_ON_ONCE(q->mq_freeze_depth < 0);
spin_unlock_irq(q->queue_lock);
if (wake)
if (wake) {
percpu_ref_reinit(&q->mq_usage_counter);
wake_up_all(&q->mq_freeze_wq);
}
}

bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
Expand Down Expand Up @@ -1785,7 +1776,7 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
if (!q)
goto err_hctxs;

if (percpu_counter_init(&q->mq_usage_counter, 0))
if (percpu_ref_init(&q->mq_usage_counter, blk_mq_usage_counter_release))
goto err_map;

setup_timer(&q->timeout, blk_mq_rq_timer, (unsigned long) q);
Expand Down Expand Up @@ -1878,7 +1869,7 @@ void blk_mq_free_queue(struct request_queue *q)
blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
blk_mq_free_hw_queues(q, set);

percpu_counter_destroy(&q->mq_usage_counter);
percpu_ref_exit(&q->mq_usage_counter);

free_percpu(q->queue_ctx);
kfree(q->queue_hw_ctx);
Expand Down Expand Up @@ -2037,8 +2028,7 @@ static int __init blk_mq_init(void)
{
blk_mq_cpu_init();

/* Must be called after percpu_counter_hotcpu_callback() */
hotcpu_notifier(blk_mq_queue_reinit_notify, -10);
hotcpu_notifier(blk_mq_queue_reinit_notify, 0);

return 0;
}
Expand Down
3 changes: 2 additions & 1 deletion include/linux/blkdev.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#include <linux/bsg.h>
#include <linux/smp.h>
#include <linux/rcupdate.h>
#include <linux/percpu-refcount.h>

#include <asm/scatterlist.h>

Expand Down Expand Up @@ -484,7 +485,7 @@ struct request_queue {
#endif
struct rcu_head rcu_head;
wait_queue_head_t mq_freeze_wq;
struct percpu_counter mq_usage_counter;
struct percpu_ref mq_usage_counter;
struct list_head all_q_node;

struct blk_mq_tag_set *tag_set;
Expand Down

0 comments on commit add703f

Please sign in to comment.