dm-zone.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
 */

#include <linux/blkdev.h>
#include <linux/mm.h>
#include <linux/sched/mm.h>
#include <linux/slab.h>

#include "dm-core.h"

#define DM_MSG_PREFIX "zone"

#define DM_ZONE_INVALID_WP_OFST		UINT_MAX

/*
 * For internal zone reports bypassing the top BIO submission path.
 */
static int dm_blk_do_report_zones(struct mapped_device *md, struct dm_table *t,
				  sector_t sector, unsigned int nr_zones,
				  report_zones_cb cb, void *data)
{
	struct gendisk *disk = md->disk;
	int ret;
	struct dm_report_zones_args args = {
		.next_sector = sector,
		.orig_data = data,
		.orig_cb = cb,
	};

	do {
		struct dm_target *tgt;

		tgt = dm_table_find_target(t, args.next_sector);
		if (WARN_ON_ONCE(!tgt->type->report_zones))
			return -EIO;

		args.tgt = tgt;
		ret = tgt->type->report_zones(tgt, &args,
					      nr_zones - args.zone_idx);
		if (ret < 0)
			return ret;
	} while (args.zone_idx < nr_zones &&
		 args.next_sector < get_capacity(disk));

	return args.zone_idx;
}

/*
 * User facing dm device block device report zone operation. This calls the
 * report_zones operation for each target of a device table. This operation is
 * generally implemented by targets using dm_report_zones().
 */
int dm_blk_report_zones(struct gendisk *disk, sector_t sector,
			unsigned int nr_zones, report_zones_cb cb, void *data)
{
	struct mapped_device *md = disk->private_data;
	struct dm_table *map;
	int srcu_idx, ret;

	if (dm_suspended_md(md))
		return -EAGAIN;

	map = dm_get_live_table(md, &srcu_idx);
	if (!map)
		return -EIO;

	ret = dm_blk_do_report_zones(md, map, sector, nr_zones, cb, data);

	dm_put_live_table(md, srcu_idx);

	return ret;
}

static int dm_report_zones_cb(struct blk_zone *zone, unsigned int idx,
			      void *data)
{
	struct dm_report_zones_args *args = data;
	sector_t sector_diff = args->tgt->begin - args->start;

	/*
	 * Ignore zones beyond the target range.
	 */
	if (zone->start >= args->start + args->tgt->len)
		return 0;

	/*
	 * Remap the start sector and write pointer position of the zone
	 * to match its position in the target range.
	 */
	zone->start += sector_diff;
	if (zone->type != BLK_ZONE_TYPE_CONVENTIONAL) {
		if (zone->cond == BLK_ZONE_COND_FULL)
			zone->wp = zone->start + zone->len;
		else if (zone->cond == BLK_ZONE_COND_EMPTY)
			zone->wp = zone->start;
		else
			zone->wp += sector_diff;
	}

	args->next_sector = zone->start + zone->len;
	return args->orig_cb(zone, args->zone_idx++, args->orig_data);
}

/*
 * Helper for drivers of zoned targets to implement struct target_type
 * report_zones operation.
 */
int dm_report_zones(struct block_device *bdev, sector_t start, sector_t sector,
		    struct dm_report_zones_args *args, unsigned int nr_zones)
{
	/*
	 * Set the target mapping start sector first so that
	 * dm_report_zones_cb() can correctly remap zone information.
	 */
	args->start = start;

	return blkdev_report_zones(bdev, sector, nr_zones,
				   dm_report_zones_cb, args);
}
EXPORT_SYMBOL_GPL(dm_report_zones);

bool dm_is_zone_write(struct mapped_device *md, struct bio *bio)
{
	struct request_queue *q = md->queue;

	if (!blk_queue_is_zoned(q))
		return false;

	switch (bio_op(bio)) {
	case REQ_OP_WRITE_ZEROES:
	case REQ_OP_WRITE:
		return !op_is_flush(bio->bi_opf) && bio_sectors(bio);
	default:
		return false;
	}
}

void dm_cleanup_zoned_dev(struct mapped_device *md)
{
	struct request_queue *q = md->queue;

	if (q) {
		kfree(q->conv_zones_bitmap);
		q->conv_zones_bitmap = NULL;
		kfree(q->seq_zones_wlock);
		q->seq_zones_wlock = NULL;
	}

	kvfree(md->zwp_offset);
	md->zwp_offset = NULL;
	md->nr_zones = 0;
}

static unsigned int dm_get_zone_wp_offset(struct blk_zone *zone)
{
	switch (zone->cond) {
	case BLK_ZONE_COND_IMP_OPEN:
	case BLK_ZONE_COND_EXP_OPEN:
	case BLK_ZONE_COND_CLOSED:
		return zone->wp - zone->start;
	case BLK_ZONE_COND_FULL:
		return zone->len;
	case BLK_ZONE_COND_EMPTY:
	case BLK_ZONE_COND_NOT_WP:
	case BLK_ZONE_COND_OFFLINE:
	case BLK_ZONE_COND_READONLY:
	default:
		/*
		 * Conventional, offline and read-only zones do not have a valid
		 * write pointer. Use 0 as for an empty zone.
		 */
		return 0;
	}
}

static int dm_zone_revalidate_cb(struct blk_zone *zone, unsigned int idx,
				 void *data)
{
	struct mapped_device *md = data;
	struct request_queue *q = md->queue;

	switch (zone->type) {
	case BLK_ZONE_TYPE_CONVENTIONAL:
		if (!q->conv_zones_bitmap) {
			q->conv_zones_bitmap =
				kcalloc(BITS_TO_LONGS(q->nr_zones),
					sizeof(unsigned long), GFP_NOIO);
			if (!q->conv_zones_bitmap)
				return -ENOMEM;
		}
		set_bit(idx, q->conv_zones_bitmap);
		break;
	case BLK_ZONE_TYPE_SEQWRITE_REQ:
	case BLK_ZONE_TYPE_SEQWRITE_PREF:
		if (!q->seq_zones_wlock) {
			q->seq_zones_wlock =
				kcalloc(BITS_TO_LONGS(q->nr_zones),
					sizeof(unsigned long), GFP_NOIO);
			if (!q->seq_zones_wlock)
				return -ENOMEM;
		}
		if (!md->zwp_offset) {
			md->zwp_offset =
				kvcalloc(q->nr_zones, sizeof(unsigned int),
					 GFP_KERNEL);
			if (!md->zwp_offset)
				return -ENOMEM;
		}
		md->zwp_offset[idx] = dm_get_zone_wp_offset(zone);

		break;
	default:
		DMERR("Invalid zone type 0x%x at sectors %llu",
		      (int)zone->type, zone->start);
		return -ENODEV;
	}

	return 0;
}

/*
 * Revalidate the zones of a mapped device to initialize resource necessary
 * for zone append emulation. Note that we cannot simply use the block layer
 * blk_revalidate_disk_zones() function here as the mapped device is suspended
 * (this is called from __bind() context).
 */
static int dm_revalidate_zones(struct mapped_device *md, struct dm_table *t)
{
	struct request_queue *q = md->queue;
	unsigned int noio_flag;
	int ret;

	/*
	 * Check if something changed. If yes, cleanup the current resources
	 * and reallocate everything.
	 */
	if (!q->nr_zones || q->nr_zones != md->nr_zones)
		dm_cleanup_zoned_dev(md);
	if (md->nr_zones)
		return 0;

	/*
	 * Scan all zones to initialize everything. Ensure that all vmalloc
	 * operations in this context are done as if GFP_NOIO was specified.
	 */
	noio_flag = memalloc_noio_save();
	ret = dm_blk_do_report_zones(md, t, 0, q->nr_zones,
				     dm_zone_revalidate_cb, md);
	memalloc_noio_restore(noio_flag);
	if (ret < 0)
		goto err;
	if (ret != q->nr_zones) {
		ret = -EIO;
		goto err;
	}

	md->nr_zones = q->nr_zones;

	return 0;

err:
	DMERR("Revalidate zones failed %d", ret);
	dm_cleanup_zoned_dev(md);
	return ret;
}

static int device_not_zone_append_capable(struct dm_target *ti,
					  struct dm_dev *dev, sector_t start,
					  sector_t len, void *data)
{
	return !blk_queue_is_zoned(bdev_get_queue(dev->bdev));
}

static bool dm_table_supports_zone_append(struct dm_table *t)
{
	struct dm_target *ti;
	unsigned int i;

	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		ti = dm_table_get_target(t, i);

		if (ti->emulate_zone_append)
			return false;

		if (!ti->type->iterate_devices ||
		    ti->type->iterate_devices(ti, device_not_zone_append_capable, NULL))
			return false;
	}

	return true;
}

int dm_set_zones_restrictions(struct dm_table *t, struct request_queue *q)
{
	struct mapped_device *md = t->md;

	/*
	 * For a zoned target, the number of zones should be updated for the
	 * correct value to be exposed in sysfs queue/nr_zones.
	 */
	WARN_ON_ONCE(queue_is_mq(q));
	q->nr_zones = blkdev_nr_zones(md->disk);

	/* Check if zone append is natively supported */
	if (dm_table_supports_zone_append(t)) {
		clear_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
		dm_cleanup_zoned_dev(md);
		return 0;
	}

	/*
	 * Mark the mapped device as needing zone append emulation and
	 * initialize the emulation resources once the capacity is set.
	 */
	set_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
	if (!get_capacity(md->disk))
		return 0;

	return dm_revalidate_zones(md, t);
}

static int dm_update_zone_wp_offset_cb(struct blk_zone *zone, unsigned int idx,
				       void *data)
{
	unsigned int *wp_offset = data;

	*wp_offset = dm_get_zone_wp_offset(zone);

	return 0;
}

static int dm_update_zone_wp_offset(struct mapped_device *md, unsigned int zno,
				    unsigned int *wp_ofst)
{
	sector_t sector = zno * blk_queue_zone_sectors(md->queue);
	unsigned int noio_flag;
	struct dm_table *t;
	int srcu_idx, ret;

	t = dm_get_live_table(md, &srcu_idx);
	if (!t)
		return -EIO;

	/*
	 * Ensure that all memory allocations in this context are done as if
	 * GFP_NOIO was specified.
	 */
	noio_flag = memalloc_noio_save();
	ret = dm_blk_do_report_zones(md, t, sector, 1,
				     dm_update_zone_wp_offset_cb, wp_ofst);
	memalloc_noio_restore(noio_flag);

	dm_put_live_table(md, srcu_idx);

	if (ret != 1)
		return -EIO;

	return 0;
}

struct orig_bio_details {
	unsigned int op;
	unsigned int nr_sectors;
};

/*
 * First phase of BIO mapping for targets with zone append emulation:
 * check all BIO that change a zone writer pointer and change zone
 * append operations into regular write operations.
 */
static bool dm_zone_map_bio_begin(struct mapped_device *md,
				  unsigned int zno, struct bio *clone)
{
	sector_t zsectors = blk_queue_zone_sectors(md->queue);
	unsigned int zwp_offset = READ_ONCE(md->zwp_offset[zno]);

	/*
	 * If the target zone is in an error state, recover by inspecting the
	 * zone to get its current write pointer position. Note that since the
	 * target zone is already locked, a BIO issuing context should never
	 * see the zone write in the DM_ZONE_UPDATING_WP_OFST state.
	 */
	if (zwp_offset == DM_ZONE_INVALID_WP_OFST) {
		if (dm_update_zone_wp_offset(md, zno, &zwp_offset))
			return false;
		WRITE_ONCE(md->zwp_offset[zno], zwp_offset);
	}

	switch (bio_op(clone)) {
	case REQ_OP_ZONE_RESET:
	case REQ_OP_ZONE_FINISH:
		return true;
	case REQ_OP_WRITE_ZEROES:
	case REQ_OP_WRITE:
		/* Writes must be aligned to the zone write pointer */
		if ((clone->bi_iter.bi_sector & (zsectors - 1)) != zwp_offset)
			return false;
		break;
	case REQ_OP_ZONE_APPEND:
		/*
		 * Change zone append operations into a non-mergeable regular
		 * writes directed at the current write pointer position of the
		 * target zone.
		 */
		clone->bi_opf = REQ_OP_WRITE | REQ_NOMERGE |
			(clone->bi_opf & (~REQ_OP_MASK));
		clone->bi_iter.bi_sector += zwp_offset;
		break;
	default:
		DMWARN_LIMIT("Invalid BIO operation");
		return false;
	}

	/* Cannot write to a full zone */
	if (zwp_offset >= zsectors)
		return false;

	return true;
}

/*
 * Second phase of BIO mapping for targets with zone append emulation:
 * update the zone write pointer offset array to account for the additional
 * data written to a zone. Note that at this point, the remapped clone BIO
 * may already have completed, so we do not touch it.
 */
static blk_status_t dm_zone_map_bio_end(struct mapped_device *md, unsigned int zno,
					struct orig_bio_details *orig_bio_details,
					unsigned int nr_sectors)
{
	unsigned int zwp_offset = READ_ONCE(md->zwp_offset[zno]);

	/* The clone BIO may already have been completed and failed */
	if (zwp_offset == DM_ZONE_INVALID_WP_OFST)
		return BLK_STS_IOERR;

	/* Update the zone wp offset */
	switch (orig_bio_details->op) {
	case REQ_OP_ZONE_RESET:
		WRITE_ONCE(md->zwp_offset[zno], 0);
		return BLK_STS_OK;
	case REQ_OP_ZONE_FINISH:
		WRITE_ONCE(md->zwp_offset[zno],
			   blk_queue_zone_sectors(md->queue));
		return BLK_STS_OK;
	case REQ_OP_WRITE_ZEROES:
	case REQ_OP_WRITE:
		WRITE_ONCE(md->zwp_offset[zno], zwp_offset + nr_sectors);
		return BLK_STS_OK;
	case REQ_OP_ZONE_APPEND:
		/*
		 * Check that the target did not truncate the write operation
		 * emulating a zone append.
		 */
		if (nr_sectors != orig_bio_details->nr_sectors) {
			DMWARN_LIMIT("Truncated write for zone append");
			return BLK_STS_IOERR;
		}
		WRITE_ONCE(md->zwp_offset[zno], zwp_offset + nr_sectors);
		return BLK_STS_OK;
	default:
		DMWARN_LIMIT("Invalid BIO operation");
		return BLK_STS_IOERR;
	}
}

static inline void dm_zone_lock(struct request_queue *q,
				unsigned int zno, struct bio *clone)
{
	if (WARN_ON_ONCE(bio_flagged(clone, BIO_ZONE_WRITE_LOCKED)))
		return;

	wait_on_bit_lock_io(q->seq_zones_wlock, zno, TASK_UNINTERRUPTIBLE);
	bio_set_flag(clone, BIO_ZONE_WRITE_LOCKED);
}

static inline void dm_zone_unlock(struct request_queue *q,
				  unsigned int zno, struct bio *clone)
{
	if (!bio_flagged(clone, BIO_ZONE_WRITE_LOCKED))
		return;

	WARN_ON_ONCE(!test_bit(zno, q->seq_zones_wlock));
	clear_bit_unlock(zno, q->seq_zones_wlock);
	smp_mb__after_atomic();
	wake_up_bit(q->seq_zones_wlock, zno);

	bio_clear_flag(clone, BIO_ZONE_WRITE_LOCKED);
}

static bool dm_need_zone_wp_tracking(struct bio *bio)
{
	/*
	 * Special processing is not needed for operations that do not need the
	 * zone write lock, that is, all operations that target conventional
	 * zones and all operations that do not modify directly a sequential
	 * zone write pointer.
	 */
	if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))
		return false;
	switch (bio_op(bio)) {
	case REQ_OP_WRITE_ZEROES:
	case REQ_OP_WRITE:
	case REQ_OP_ZONE_RESET:
	case REQ_OP_ZONE_FINISH:
	case REQ_OP_ZONE_APPEND:
		return bio_zone_is_seq(bio);
	default:
		return false;
	}
}

/*
 * Special IO mapping for targets needing zone append emulation.
 */
int dm_zone_map_bio(struct dm_target_io *tio)
{
	struct dm_io *io = tio->io;
	struct dm_target *ti = tio->ti;
	struct mapped_device *md = io->md;
	struct request_queue *q = md->queue;
	struct bio *clone = &tio->clone;
	struct orig_bio_details orig_bio_details;
	unsigned int zno;
	blk_status_t sts;
	int r;

	/*
	 * IOs that do not change a zone write pointer do not need
	 * any additional special processing.
	 */
	if (!dm_need_zone_wp_tracking(clone))
		return ti->type->map(ti, clone);

	/* Lock the target zone */
	zno = bio_zone_no(clone);
	dm_zone_lock(q, zno, clone);

	orig_bio_details.nr_sectors = bio_sectors(clone);
	orig_bio_details.op = bio_op(clone);

	/*
	 * Check that the bio and the target zone write pointer offset are
	 * both valid, and if the bio is a zone append, remap it to a write.
	 */
	if (!dm_zone_map_bio_begin(md, zno, clone)) {
		dm_zone_unlock(q, zno, clone);
		return DM_MAPIO_KILL;
	}

	/*
	 * The target map function may issue and complete the IO quickly.
	 * Take an extra reference on the IO to make sure it does disappear
	 * until we run dm_zone_map_bio_end().
	 */
	dm_io_inc_pending(io);

	/* Let the target do its work */
	r = ti->type->map(ti, clone);
	switch (r) {
	case DM_MAPIO_SUBMITTED:
		/*
		 * The target submitted the clone BIO. The target zone will
		 * be unlocked on completion of the clone.
		 */
		sts = dm_zone_map_bio_end(md, zno, &orig_bio_details,
					  *tio->len_ptr);
		break;
	case DM_MAPIO_REMAPPED:
		/*
		 * The target only remapped the clone BIO. In case of error,
		 * unlock the target zone here as the clone will not be
		 * submitted.
		 */
		sts = dm_zone_map_bio_end(md, zno, &orig_bio_details,
					  *tio->len_ptr);
		if (sts != BLK_STS_OK)
			dm_zone_unlock(q, zno, clone);
		break;
	case DM_MAPIO_REQUEUE:
	case DM_MAPIO_KILL:
	default:
		dm_zone_unlock(q, zno, clone);
		sts = BLK_STS_IOERR;
		break;
	}

	/* Drop the extra reference on the IO */
	dm_io_dec_pending(io, sts);

	if (sts != BLK_STS_OK)
		return DM_MAPIO_KILL;

	return r;
}

/*
 * IO completion callback called from clone_endio().
 */
void dm_zone_endio(struct dm_io *io, struct bio *clone)
{
	struct mapped_device *md = io->md;
	struct request_queue *q = md->queue;
	struct bio *orig_bio = io->orig_bio;
	unsigned int zwp_offset;
	unsigned int zno;

	/*
	 * For targets that do not emulate zone append, we only need to
	 * handle native zone-append bios.
	 */
	if (!dm_emulate_zone_append(md)) {
		/*
		 * Get the offset within the zone of the written sector
		 * and add that to the original bio sector position.
		 */
		if (clone->bi_status == BLK_STS_OK &&
		    bio_op(clone) == REQ_OP_ZONE_APPEND) {
			sector_t mask = (sector_t)blk_queue_zone_sectors(q) - 1;

			orig_bio->bi_iter.bi_sector +=
				clone->bi_iter.bi_sector & mask;
		}

		return;
	}

	/*
	 * For targets that do emulate zone append, if the clone BIO does not
	 * own the target zone write lock, we have nothing to do.
	 */
	if (!bio_flagged(clone, BIO_ZONE_WRITE_LOCKED))
		return;

	zno = bio_zone_no(orig_bio);

	if (clone->bi_status != BLK_STS_OK) {
		/*
		 * BIOs that modify a zone write pointer may leave the zone
		 * in an unknown state in case of failure (e.g. the write
		 * pointer was only partially advanced). In this case, set
		 * the target zone write pointer as invalid unless it is
		 * already being updated.
		 */
		WRITE_ONCE(md->zwp_offset[zno], DM_ZONE_INVALID_WP_OFST);
	} else if (bio_op(orig_bio) == REQ_OP_ZONE_APPEND) {
		/*
		 * Get the written sector for zone append operation that were
		 * emulated using regular write operations.
		 */
		zwp_offset = READ_ONCE(md->zwp_offset[zno]);
		if (WARN_ON_ONCE(zwp_offset < bio_sectors(orig_bio)))
			WRITE_ONCE(md->zwp_offset[zno],
				   DM_ZONE_INVALID_WP_OFST);
		else
			orig_bio->bi_iter.bi_sector +=
				zwp_offset - bio_sectors(orig_bio);
	}

	dm_zone_unlock(q, zno, clone);
}