Skip to content

Commit

Permalink
lightnvm: move metadata mapping to lower level driver
Browse files Browse the repository at this point in the history 
Now that blk_rq_map_kern can map both kmem and vmem, move internal
metadata mapping down to the lower level driver.

Reviewed-by: Javier González <[email protected]>
Reviewed-by: Christoph Hellwig <[email protected]>
Signed-off-by: Hans Holmberg <[email protected]>
Signed-off-by: Jens Axboe <[email protected]>
  • Loading branch information
@axboe
Hans Holmberg authored and axboe committed Aug 6, 2019
1 parent 98d87f7 commit 48e5da7
Show file tree
Hide file tree
Showing 8 changed files with 54 additions and 190 deletions.
16 changes: 9 additions & 7 deletions drivers/lightnvm/core.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -731,7 +731,7 @@ static int nvm_set_flags(struct nvm_geo *geo, struct nvm_rq *rqd)
return flags;
}

int nvm_submit_io(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd)
int nvm_submit_io(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd, void *buf)
{
struct nvm_dev *dev = tgt_dev->parent;
int ret;
Expand All @@ -745,7 +745,7 @@ int nvm_submit_io(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd)
rqd->flags = nvm_set_flags(&tgt_dev->geo, rqd);

/* In case of error, fail with right address format */
ret = dev->ops->submit_io(dev, rqd);
ret = dev->ops->submit_io(dev, rqd, buf);
if (ret)
nvm_rq_dev_to_tgt(tgt_dev, rqd);
return ret;
Expand All @@ -759,15 +759,16 @@ static void nvm_sync_end_io(struct nvm_rq *rqd)
complete(waiting);
}

static int nvm_submit_io_wait(struct nvm_dev *dev, struct nvm_rq *rqd)
static int nvm_submit_io_wait(struct nvm_dev *dev, struct nvm_rq *rqd,
void *buf)
{
DECLARE_COMPLETION_ONSTACK(wait);
int ret = 0;

rqd->end_io = nvm_sync_end_io;
rqd->private = &wait;

ret = dev->ops->submit_io(dev, rqd);
ret = dev->ops->submit_io(dev, rqd, buf);
if (ret)
return ret;

Expand All @@ -776,7 +777,8 @@ static int nvm_submit_io_wait(struct nvm_dev *dev, struct nvm_rq *rqd)
return 0;
}

int nvm_submit_io_sync(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd)
int nvm_submit_io_sync(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd,
void *buf)
{
struct nvm_dev *dev = tgt_dev->parent;
int ret;
Expand All @@ -789,7 +791,7 @@ int nvm_submit_io_sync(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd)
rqd->dev = tgt_dev;
rqd->flags = nvm_set_flags(&tgt_dev->geo, rqd);

ret = nvm_submit_io_wait(dev, rqd);
ret = nvm_submit_io_wait(dev, rqd, buf);

return ret;
}
Expand All @@ -816,7 +818,7 @@ static int nvm_submit_io_sync_raw(struct nvm_dev *dev, struct nvm_rq *rqd)
rqd->dev = NULL;
rqd->flags = nvm_set_flags(&dev->geo, rqd);

return nvm_submit_io_wait(dev, rqd);
return nvm_submit_io_wait(dev, rqd, NULL);
}

static int nvm_bb_chunk_sense(struct nvm_dev *dev, struct ppa_addr ppa)
Expand Down
113 changes: 12 additions & 101 deletions drivers/lightnvm/pblk-core.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -507,7 +507,7 @@ void pblk_set_sec_per_write(struct pblk *pblk, int sec_per_write)
pblk->sec_per_write = sec_per_write;
}

int pblk_submit_io(struct pblk *pblk, struct nvm_rq *rqd)
int pblk_submit_io(struct pblk *pblk, struct nvm_rq *rqd, void *buf)
{
struct nvm_tgt_dev *dev = pblk->dev;

Expand All @@ -518,7 +518,7 @@ int pblk_submit_io(struct pblk *pblk, struct nvm_rq *rqd)
return NVM_IO_ERR;
#endif

return nvm_submit_io(dev, rqd);
return nvm_submit_io(dev, rqd, buf);
}

void pblk_check_chunk_state_update(struct pblk *pblk, struct nvm_rq *rqd)
Expand All @@ -541,7 +541,7 @@ void pblk_check_chunk_state_update(struct pblk *pblk, struct nvm_rq *rqd)
}
}

int pblk_submit_io_sync(struct pblk *pblk, struct nvm_rq *rqd)
int pblk_submit_io_sync(struct pblk *pblk, struct nvm_rq *rqd, void *buf)
{
struct nvm_tgt_dev *dev = pblk->dev;
int ret;
Expand All @@ -553,7 +553,7 @@ int pblk_submit_io_sync(struct pblk *pblk, struct nvm_rq *rqd)
return NVM_IO_ERR;
#endif

ret = nvm_submit_io_sync(dev, rqd);
ret = nvm_submit_io_sync(dev, rqd, buf);

if (trace_pblk_chunk_state_enabled() && !ret &&
rqd->opcode == NVM_OP_PWRITE)
Expand All @@ -562,65 +562,19 @@ int pblk_submit_io_sync(struct pblk *pblk, struct nvm_rq *rqd)
return ret;
}

int pblk_submit_io_sync_sem(struct pblk *pblk, struct nvm_rq *rqd)
static int pblk_submit_io_sync_sem(struct pblk *pblk, struct nvm_rq *rqd,
void *buf)
{
struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd);
int ret;

pblk_down_chunk(pblk, ppa_list[0]);
ret = pblk_submit_io_sync(pblk, rqd);
ret = pblk_submit_io_sync(pblk, rqd, buf);
pblk_up_chunk(pblk, ppa_list[0]);

return ret;
}

static void pblk_bio_map_addr_endio(struct bio *bio)
{
bio_put(bio);
}

struct bio *pblk_bio_map_addr(struct pblk *pblk, void *data,
unsigned int nr_secs, unsigned int len,
int alloc_type, gfp_t gfp_mask)
{
struct nvm_tgt_dev *dev = pblk->dev;
void *kaddr = data;
struct page *page;
struct bio *bio;
int i, ret;

if (alloc_type == PBLK_KMALLOC_META)
return bio_map_kern(dev->q, kaddr, len, gfp_mask);

bio = bio_kmalloc(gfp_mask, nr_secs);
if (!bio)
return ERR_PTR(-ENOMEM);

for (i = 0; i < nr_secs; i++) {
page = vmalloc_to_page(kaddr);
if (!page) {
pblk_err(pblk, "could not map vmalloc bio\n");
bio_put(bio);
bio = ERR_PTR(-ENOMEM);
goto out;
}

ret = bio_add_pc_page(dev->q, bio, page, PAGE_SIZE, 0);
if (ret != PAGE_SIZE) {
pblk_err(pblk, "could not add page to bio\n");
bio_put(bio);
bio = ERR_PTR(-ENOMEM);
goto out;
}

kaddr += PAGE_SIZE;
}

bio->bi_end_io = pblk_bio_map_addr_endio;
out:
return bio;
}

int pblk_calc_secs(struct pblk *pblk, unsigned long secs_avail,
unsigned long secs_to_flush, bool skip_meta)
{
Expand Down Expand Up @@ -722,9 +676,7 @@ u64 pblk_line_smeta_start(struct pblk *pblk, struct pblk_line *line)

int pblk_line_smeta_read(struct pblk *pblk, struct pblk_line *line)
{
struct nvm_tgt_dev *dev = pblk->dev;
struct pblk_line_meta *lm = &pblk->lm;
struct bio *bio;
struct ppa_addr *ppa_list;
struct nvm_rq rqd;
u64 paddr = pblk_line_smeta_start(pblk, line);
Expand All @@ -736,16 +688,6 @@ int pblk_line_smeta_read(struct pblk *pblk, struct pblk_line *line)
if (ret)
return ret;

bio = bio_map_kern(dev->q, line->smeta, lm->smeta_len, GFP_KERNEL);
if (IS_ERR(bio)) {
ret = PTR_ERR(bio);
goto clear_rqd;
}

bio->bi_iter.bi_sector = 0; /* internal bio */
bio_set_op_attrs(bio, REQ_OP_READ, 0);

rqd.bio = bio;
rqd.opcode = NVM_OP_PREAD;
rqd.nr_ppas = lm->smeta_sec;
rqd.is_seq = 1;
Expand All @@ -754,10 +696,9 @@ int pblk_line_smeta_read(struct pblk *pblk, struct pblk_line *line)
for (i = 0; i < lm->smeta_sec; i++, paddr++)
ppa_list[i] = addr_to_gen_ppa(pblk, paddr, line->id);

ret = pblk_submit_io_sync(pblk, &rqd);
ret = pblk_submit_io_sync(pblk, &rqd, line->smeta);
if (ret) {
pblk_err(pblk, "smeta I/O submission failed: %d\n", ret);
bio_put(bio);
goto clear_rqd;
}

Expand All @@ -776,9 +717,7 @@ int pblk_line_smeta_read(struct pblk *pblk, struct pblk_line *line)
static int pblk_line_smeta_write(struct pblk *pblk, struct pblk_line *line,
u64 paddr)
{
struct nvm_tgt_dev *dev = pblk->dev;
struct pblk_line_meta *lm = &pblk->lm;
struct bio *bio;
struct ppa_addr *ppa_list;
struct nvm_rq rqd;
__le64 *lba_list = emeta_to_lbas(pblk, line->emeta->buf);
Expand All @@ -791,16 +730,6 @@ static int pblk_line_smeta_write(struct pblk *pblk, struct pblk_line *line,
if (ret)
return ret;

bio = bio_map_kern(dev->q, line->smeta, lm->smeta_len, GFP_KERNEL);
if (IS_ERR(bio)) {
ret = PTR_ERR(bio);
goto clear_rqd;
}

bio->bi_iter.bi_sector = 0; /* internal bio */
bio_set_op_attrs(bio, REQ_OP_WRITE, 0);

rqd.bio = bio;
rqd.opcode = NVM_OP_PWRITE;
rqd.nr_ppas = lm->smeta_sec;
rqd.is_seq = 1;
Expand All @@ -814,10 +743,9 @@ static int pblk_line_smeta_write(struct pblk *pblk, struct pblk_line *line,
meta->lba = lba_list[paddr] = addr_empty;
}

ret = pblk_submit_io_sync_sem(pblk, &rqd);
ret = pblk_submit_io_sync_sem(pblk, &rqd, line->smeta);
if (ret) {
pblk_err(pblk, "smeta I/O submission failed: %d\n", ret);
bio_put(bio);
goto clear_rqd;
}

Expand All @@ -838,10 +766,8 @@ int pblk_line_emeta_read(struct pblk *pblk, struct pblk_line *line,
{
struct nvm_tgt_dev *dev = pblk->dev;
struct nvm_geo *geo = &dev->geo;
struct pblk_line_mgmt *l_mg = &pblk->l_mg;
struct pblk_line_meta *lm = &pblk->lm;
void *ppa_list_buf, *meta_list;
struct bio *bio;
struct ppa_addr *ppa_list;
struct nvm_rq rqd;
u64 paddr = line->emeta_ssec;
Expand All @@ -867,17 +793,6 @@ int pblk_line_emeta_read(struct pblk *pblk, struct pblk_line *line,
rq_ppas = pblk_calc_secs(pblk, left_ppas, 0, false);
rq_len = rq_ppas * geo->csecs;

bio = pblk_bio_map_addr(pblk, emeta_buf, rq_ppas, rq_len,
l_mg->emeta_alloc_type, GFP_KERNEL);
if (IS_ERR(bio)) {
ret = PTR_ERR(bio);
goto free_rqd_dma;
}

bio->bi_iter.bi_sector = 0; /* internal bio */
bio_set_op_attrs(bio, REQ_OP_READ, 0);

rqd.bio = bio;
rqd.meta_list = meta_list;
rqd.ppa_list = ppa_list_buf;
rqd.dma_meta_list = dma_meta_list;
Expand All @@ -896,7 +811,6 @@ int pblk_line_emeta_read(struct pblk *pblk, struct pblk_line *line,
while (test_bit(pos, line->blk_bitmap)) {
paddr += min;
if (pblk_boundary_paddr_checks(pblk, paddr)) {
bio_put(bio);
ret = -EINTR;
goto free_rqd_dma;
}
Expand All @@ -906,7 +820,6 @@ int pblk_line_emeta_read(struct pblk *pblk, struct pblk_line *line,
}

if (pblk_boundary_paddr_checks(pblk, paddr + min)) {
bio_put(bio);
ret = -EINTR;
goto free_rqd_dma;
}
Expand All @@ -915,10 +828,9 @@ int pblk_line_emeta_read(struct pblk *pblk, struct pblk_line *line,
ppa_list[i] = addr_to_gen_ppa(pblk, paddr, line_id);
}

ret = pblk_submit_io_sync(pblk, &rqd);
ret = pblk_submit_io_sync(pblk, &rqd, emeta_buf);
if (ret) {
pblk_err(pblk, "emeta I/O submission failed: %d\n", ret);
bio_put(bio);
goto free_rqd_dma;
}

Expand Down Expand Up @@ -963,7 +875,7 @@ static int pblk_blk_erase_sync(struct pblk *pblk, struct ppa_addr ppa)
/* The write thread schedules erases so that it minimizes disturbances
* with writes. Thus, there is no need to take the LUN semaphore.
*/
ret = pblk_submit_io_sync(pblk, &rqd);
ret = pblk_submit_io_sync(pblk, &rqd, NULL);
rqd.private = pblk;
__pblk_end_io_erase(pblk, &rqd);

Expand Down Expand Up @@ -1792,7 +1704,7 @@ int pblk_blk_erase_async(struct pblk *pblk, struct ppa_addr ppa)
/* The write thread schedules erases so that it minimizes disturbances
* with writes. Thus, there is no need to take the LUN semaphore.
*/
err = pblk_submit_io(pblk, rqd);
err = pblk_submit_io(pblk, rqd, NULL);
if (err) {
struct nvm_tgt_dev *dev = pblk->dev;
struct nvm_geo *geo = &dev->geo;
Expand Down Expand Up @@ -1923,7 +1835,6 @@ void pblk_line_close_meta(struct pblk *pblk, struct pblk_line *line)
static void pblk_save_lba_list(struct pblk *pblk, struct pblk_line *line)
{
struct pblk_line_meta *lm = &pblk->lm;
struct pblk_line_mgmt *l_mg = &pblk->l_mg;
unsigned int lba_list_size = lm->emeta_len[2];
struct pblk_w_err_gc *w_err_gc = line->w_err_gc;
struct pblk_emeta *emeta = line->emeta;
Expand Down
22 changes: 3 additions & 19 deletions drivers/lightnvm/pblk-read.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -342,7 +342,7 @@ void pblk_submit_read(struct pblk *pblk, struct bio *bio)
bio_put(int_bio);
int_bio = bio_clone_fast(bio, GFP_KERNEL, &pblk_bio_set);
goto split_retry;
} else if (pblk_submit_io(pblk, rqd)) {
} else if (pblk_submit_io(pblk, rqd, NULL)) {
/* Submitting IO to drive failed, let's report an error */
rqd->error = -ENODEV;
pblk_end_io_read(rqd);
Expand Down Expand Up @@ -419,7 +419,6 @@ int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq)
{
struct nvm_tgt_dev *dev = pblk->dev;
struct nvm_geo *geo = &dev->geo;
struct bio *bio;
struct nvm_rq rqd;
int data_len;
int ret = NVM_IO_OK;
Expand Down Expand Up @@ -447,25 +446,12 @@ int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq)
goto out;

data_len = (gc_rq->secs_to_gc) * geo->csecs;
bio = pblk_bio_map_addr(pblk, gc_rq->data, gc_rq->secs_to_gc, data_len,
PBLK_VMALLOC_META, GFP_KERNEL);
if (IS_ERR(bio)) {
pblk_err(pblk, "could not allocate GC bio (%lu)\n",
PTR_ERR(bio));
ret = PTR_ERR(bio);
goto err_free_dma;
}

bio->bi_iter.bi_sector = 0; /* internal bio */
bio_set_op_attrs(bio, REQ_OP_READ, 0);

rqd.opcode = NVM_OP_PREAD;
rqd.nr_ppas = gc_rq->secs_to_gc;
rqd.bio = bio;

if (pblk_submit_io_sync(pblk, &rqd)) {
if (pblk_submit_io_sync(pblk, &rqd, gc_rq->data)) {
ret = -EIO;
goto err_free_bio;
goto err_free_dma;
}

pblk_read_check_rand(pblk, &rqd, gc_rq->lba_list, gc_rq->nr_secs);
Expand All @@ -489,8 +475,6 @@ int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq)
pblk_free_rqd_meta(pblk, &rqd);
return ret;

err_free_bio:
bio_put(bio);
err_free_dma:
pblk_free_rqd_meta(pblk, &rqd);
return ret;
Expand Down
Loading

0 comments on commit 48e5da7

Please sign in to comment.