Skip to content

Commit

Permalink
dm-verity: Convert from tasklet to BH workqueue
Browse files Browse the repository at this point in the history 
The only generic interface to execute asynchronously in the BH context is
tasklet; however, it's marked deprecated and has some design flaws. To
replace tasklets, BH workqueue support was recently added. A BH workqueue
behaves similarly to regular workqueues except that the queued work items
are executed in the BH context.

This commit converts dm-verity from tasklet to BH workqueue. It
backfills tasklet code that was removed with commit 0a9bab3
("dm-crypt, dm-verity: disable tasklets") and tweaks to use BH
workqueue (and does some renaming).

This is a minimal conversion which doesn't rename the related names
including the "try_verify_in_tasklet" option. If this patch is applied, a
follow-up patch would be necessary. I couldn't decide whether the option
name would need to be updated too.

Signed-off-by: Tejun Heo <[email protected]>
[snitzer: rename 'use_tasklet' to 'use_bh_wq' and 'in_tasklet' to 'in_bh']
Signed-off-by: Mike Snitzer <[email protected]>
  • Loading branch information
@htejun
htejun authored and Mike Snitzer committed Mar 2, 2024
1 parent fb6ad4a commit c375b22
Show file tree
Hide file tree
Showing 2 changed files with 45 additions and 24 deletions.
64 changes: 42 additions & 22 deletions drivers/md/dm-verity-target.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ static unsigned int dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE

module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, 0644);

static DEFINE_STATIC_KEY_FALSE(use_tasklet_enabled);
static DEFINE_STATIC_KEY_FALSE(use_bh_wq_enabled);

struct dm_verity_prefetch_work {
struct work_struct work;
Expand Down Expand Up @@ -299,7 +299,7 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,

verity_hash_at_level(v, block, level, &hash_block, &offset);

if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) {
if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
data = dm_bufio_get(v->bufio, hash_block, &buf);
if (data == NULL) {
/*
Expand Down Expand Up @@ -327,15 +327,14 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,

r = verity_hash(v, verity_io_hash_req(v, io),
data, 1 << v->hash_dev_block_bits,
verity_io_real_digest(v, io), !io->in_tasklet);
verity_io_real_digest(v, io), !io->in_bh);
if (unlikely(r < 0))
goto release_ret_r;

if (likely(memcmp(verity_io_real_digest(v, io), want_digest,
v->digest_size) == 0))
aux->hash_verified = 1;
else if (static_branch_unlikely(&use_tasklet_enabled) &&
io->in_tasklet) {
else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
/*
* Error handling code (FEC included) cannot be run in a
* tasklet since it may sleep, so fallback to work-queue.
Expand Down Expand Up @@ -576,7 +575,7 @@ static int verity_verify_io(struct dm_verity_io *io)
struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);
unsigned int b;

if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) {
if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
/*
* Copy the iterator in case we need to restart
* verification in a work-queue.
Expand Down Expand Up @@ -616,7 +615,7 @@ static int verity_verify_io(struct dm_verity_io *io)
continue;
}

r = verity_hash_init(v, req, &wait, !io->in_tasklet);
r = verity_hash_init(v, req, &wait, !io->in_bh);
if (unlikely(r < 0))
return r;

Expand All @@ -635,8 +634,7 @@ static int verity_verify_io(struct dm_verity_io *io)
if (v->validated_blocks)
set_bit(cur_block, v->validated_blocks);
continue;
} else if (static_branch_unlikely(&use_tasklet_enabled) &&
io->in_tasklet) {
} else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
/*
* Error handling code (FEC included) cannot be run in a
* tasklet since it may sleep, so fallback to work-queue.
Expand Down Expand Up @@ -690,7 +688,7 @@ static void verity_finish_io(struct dm_verity_io *io, blk_status_t status)
bio->bi_end_io = io->orig_bi_end_io;
bio->bi_status = status;

if (!static_branch_unlikely(&use_tasklet_enabled) || !io->in_tasklet)
if (!static_branch_unlikely(&use_bh_wq_enabled) || !io->in_bh)
verity_fec_finish_io(io);

bio_endio(bio);
Expand All @@ -700,11 +698,28 @@ static void verity_work(struct work_struct *w)
{
struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);

io->in_tasklet = false;
io->in_bh = false;

verity_finish_io(io, errno_to_blk_status(verity_verify_io(io)));
}

static void verity_bh_work(struct work_struct *w)
{
struct dm_verity_io *io = container_of(w, struct dm_verity_io, bh_work);
int err;

io->in_bh = true;
err = verity_verify_io(io);
if (err == -EAGAIN || err == -ENOMEM) {
/* fallback to retrying with work-queue */
INIT_WORK(&io->work, verity_work);
queue_work(io->v->verify_wq, &io->work);
return;
}

verity_finish_io(io, errno_to_blk_status(err));
}

static void verity_end_io(struct bio *bio)
{
struct dm_verity_io *io = bio->bi_private;
Expand All @@ -717,8 +732,13 @@ static void verity_end_io(struct bio *bio)
return;
}

INIT_WORK(&io->work, verity_work);
queue_work(io->v->verify_wq, &io->work);
if (static_branch_unlikely(&use_bh_wq_enabled) && io->v->use_bh_wq) {
INIT_WORK(&io->bh_work, verity_bh_work);
queue_work(system_bh_wq, &io->bh_work);
} else {
INIT_WORK(&io->work, verity_work);
queue_work(io->v->verify_wq, &io->work);
}
}

/*
Expand Down Expand Up @@ -885,7 +905,7 @@ static void verity_status(struct dm_target *ti, status_type_t type,
args++;
if (v->validated_blocks)
args++;
if (v->use_tasklet)
if (v->use_bh_wq)
args++;
if (v->signature_key_desc)
args += DM_VERITY_ROOT_HASH_VERIFICATION_OPTS;
Expand All @@ -912,7 +932,7 @@ static void verity_status(struct dm_target *ti, status_type_t type,
DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES);
if (v->validated_blocks)
DMEMIT(" " DM_VERITY_OPT_AT_MOST_ONCE);
if (v->use_tasklet)
if (v->use_bh_wq)
DMEMIT(" " DM_VERITY_OPT_TASKLET_VERIFY);
sz = verity_fec_status_table(v, sz, result, maxlen);
if (v->signature_key_desc)
Expand Down Expand Up @@ -1031,8 +1051,8 @@ static void verity_dtr(struct dm_target *ti)

kfree(v->signature_key_desc);

if (v->use_tasklet)
static_branch_dec(&use_tasklet_enabled);
if (v->use_bh_wq)
static_branch_dec(&use_bh_wq_enabled);

kfree(v);

Expand Down Expand Up @@ -1166,8 +1186,8 @@ static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v,
continue;

} else if (!strcasecmp(arg_name, DM_VERITY_OPT_TASKLET_VERIFY)) {
v->use_tasklet = true;
static_branch_inc(&use_tasklet_enabled);
v->use_bh_wq = true;
static_branch_inc(&use_bh_wq_enabled);
continue;

} else if (verity_is_fec_opt_arg(arg_name)) {
Expand Down Expand Up @@ -1338,7 +1358,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv)
}

v->tfm = crypto_alloc_ahash(v->alg_name, 0,
v->use_tasklet ? CRYPTO_ALG_ASYNC : 0);
v->use_bh_wq ? CRYPTO_ALG_ASYNC : 0);
if (IS_ERR(v->tfm)) {
ti->error = "Cannot initialize hash function";
r = PTR_ERR(v->tfm);
Expand Down Expand Up @@ -1463,7 +1483,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv)
v->bufio = dm_bufio_client_create(v->hash_dev->bdev,
1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux),
dm_bufio_alloc_callback, NULL,
v->use_tasklet ? DM_BUFIO_CLIENT_NO_SLEEP : 0);
v->use_bh_wq ? DM_BUFIO_CLIENT_NO_SLEEP : 0);
if (IS_ERR(v->bufio)) {
ti->error = "Cannot initialize dm-bufio";
r = PTR_ERR(v->bufio);
Expand All @@ -1482,7 +1502,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv)
* reducing wait times when reading from a dm-verity device.
*
* Also as required for the "try_verify_in_tasklet" feature: WQ_HIGHPRI
* allows verify_wq to preempt softirq since verification in tasklet
* allows verify_wq to preempt softirq since verification in BH workqueue
* will fall-back to using it for error handling (or if the bufio cache
* doesn't have required hashes).
*/
Expand Down
5 changes: 3 additions & 2 deletions drivers/md/dm-verity.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ struct dm_verity {
unsigned char levels; /* the number of tree levels */
unsigned char version;
bool hash_failed:1; /* set if hash of any block failed */
bool use_tasklet:1; /* try to verify in tasklet before work-queue */
bool use_bh_wq:1; /* try to verify in BH wq before normal work-queue */
unsigned int digest_size; /* digest size for the current hash algorithm */
unsigned int ahash_reqsize;/* the size of temporary space for crypto */
enum verity_mode mode; /* mode for handling verification errors */
Expand Down Expand Up @@ -84,9 +84,10 @@ struct dm_verity_io {

sector_t block;
unsigned int n_blocks;
bool in_tasklet;
bool in_bh;

struct work_struct work;
struct work_struct bh_work;

char *recheck_buffer;

Expand Down

0 comments on commit c375b22

Please sign in to comment.