Skip to content

Commit

Permalink
block: pass iov_iter to the BLOCK_PC mapping functions
Browse files Browse the repository at this point in the history 
Make use of a new interface provided by iov_iter, backed by
scatter-gather list of iovec, instead of the old interface based on
sg_iovec. Also use iov_iter_advance() instead of manual iteration.

This commit should contain only literal replacements, without
functional changes.

Cc: Christoph Hellwig <[email protected]>
Cc: Jens Axboe <[email protected]>
Cc: Doug Gilbert <[email protected]>
Cc: "James E.J. Bottomley" <[email protected]>
Signed-off-by: Kent Overstreet <[email protected]>
[dpark: add more description in commit message]
Signed-off-by: Dongsu Park <[email protected]>
[hch: fixed to do a deep clone of the iov_iter, and to properly use
      the iov_iter direction]
Signed-off-by: Christoph Hellwig <[email protected]>
Reviewed-by: Ming Lei <[email protected]>
Signed-off-by: Jens Axboe <[email protected]>
  • Loading branch information
@axboe
Kent Overstreet authored and axboe committed Feb 5, 2015
1 parent 1dfa0f6 commit 26e49cf
Show file tree
Hide file tree
Showing 6 changed files with 101 additions and 126 deletions.
146 changes: 67 additions & 79 deletions block/bio.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
#include <linux/mempool.h>
#include <linux/workqueue.h>
#include <linux/cgroup.h>
#include <scsi/sg.h> /* for struct sg_iovec */

#include <trace/events/block.h>

Expand Down Expand Up @@ -1022,58 +1021,45 @@ void bio_copy_data(struct bio *dst, struct bio *src)
EXPORT_SYMBOL(bio_copy_data);

struct bio_map_data {
int nr_sgvecs;
int is_our_pages;
struct sg_iovec sgvecs[];
struct iov_iter iter;
struct iovec iov[];
};

static void bio_set_map_data(struct bio_map_data *bmd, struct bio *bio,
const struct sg_iovec *iov, int iov_count,
int is_our_pages)
{
memcpy(bmd->sgvecs, iov, sizeof(struct sg_iovec) * iov_count);
bmd->nr_sgvecs = iov_count;
bmd->is_our_pages = is_our_pages;
bio->bi_private = bmd;
}

static struct bio_map_data *bio_alloc_map_data(unsigned int iov_count,
gfp_t gfp_mask)
{
if (iov_count > UIO_MAXIOV)
return NULL;

return kmalloc(sizeof(struct bio_map_data) +
sizeof(struct sg_iovec) * iov_count, gfp_mask);
sizeof(struct iovec) * iov_count, gfp_mask);
}

static int __bio_copy_iov(struct bio *bio, const struct sg_iovec *iov, int iov_count,
int to_user, int from_user)
static int __bio_copy_iov(struct bio *bio, const struct iov_iter *iter,
int to_user, int from_user)
{
int ret = 0, i;
struct bio_vec *bvec;
int iov_idx = 0;
unsigned int iov_off = 0;
struct iov_iter iov_iter = *iter;

bio_for_each_segment_all(bvec, bio, i) {
char *bv_addr = page_address(bvec->bv_page);
unsigned int bv_len = bvec->bv_len;

while (bv_len && iov_idx < iov_count) {
unsigned int bytes;
char __user *iov_addr;

bytes = min_t(unsigned int,
iov[iov_idx].iov_len - iov_off, bv_len);
iov_addr = iov[iov_idx].iov_base + iov_off;
while (bv_len && iov_iter.count) {
struct iovec iov = iov_iter_iovec(&iov_iter);
unsigned int bytes = min_t(unsigned int, bv_len,
iov.iov_len);

if (!ret) {
if (to_user)
ret = copy_to_user(iov_addr, bv_addr,
bytes);
ret = copy_to_user(iov.iov_base,
bv_addr, bytes);

if (from_user)
ret = copy_from_user(bv_addr, iov_addr,
ret = copy_from_user(bv_addr,
iov.iov_base,
bytes);

if (ret)
Expand All @@ -1082,13 +1068,7 @@ static int __bio_copy_iov(struct bio *bio, const struct sg_iovec *iov, int iov_c

bv_len -= bytes;
bv_addr += bytes;
iov_addr += bytes;
iov_off += bytes;

if (iov[iov_idx].iov_len == iov_off) {
iov_idx++;
iov_off = 0;
}
iov_iter_advance(&iov_iter, bytes);
}
}

Expand Down Expand Up @@ -1122,7 +1102,7 @@ int bio_uncopy_user(struct bio *bio)
* don't copy into a random user address space, just free.
*/
if (current->mm)
ret = __bio_copy_iov(bio, bmd->sgvecs, bmd->nr_sgvecs,
ret = __bio_copy_iov(bio, &bmd->iter,
bio_data_dir(bio) == READ, 0);
if (bmd->is_our_pages)
bio_free_pages(bio);
Expand All @@ -1135,37 +1115,36 @@ EXPORT_SYMBOL(bio_uncopy_user);

/**
* bio_copy_user_iov - copy user data to bio
* @q: destination block queue
* @map_data: pointer to the rq_map_data holding pages (if necessary)
* @iov: the iovec.
* @iov_count: number of elements in the iovec
* @write_to_vm: bool indicating writing to pages or not
* @gfp_mask: memory allocation flags
* @q: destination block queue
* @map_data: pointer to the rq_map_data holding pages (if necessary)
* @iter: iovec iterator
* @gfp_mask: memory allocation flags
*
* Prepares and returns a bio for indirect user io, bouncing data
* to/from kernel pages as necessary. Must be paired with
* call bio_uncopy_user() on io completion.
*/
struct bio *bio_copy_user_iov(struct request_queue *q,
struct rq_map_data *map_data,
const struct sg_iovec *iov, int iov_count,
int write_to_vm, gfp_t gfp_mask)
const struct iov_iter *iter,
gfp_t gfp_mask)
{
struct bio_map_data *bmd;
struct page *page;
struct bio *bio;
int i, ret;
int nr_pages = 0;
unsigned int len = 0;
unsigned int len = iter->count;
unsigned int offset = map_data ? map_data->offset & ~PAGE_MASK : 0;

for (i = 0; i < iov_count; i++) {
for (i = 0; i < iter->nr_segs; i++) {
unsigned long uaddr;
unsigned long end;
unsigned long start;

uaddr = (unsigned long)iov[i].iov_base;
end = (uaddr + iov[i].iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
uaddr = (unsigned long) iter->iov[i].iov_base;
end = (uaddr + iter->iov[i].iov_len + PAGE_SIZE - 1)
>> PAGE_SHIFT;
start = uaddr >> PAGE_SHIFT;

/*
Expand All @@ -1175,22 +1154,31 @@ struct bio *bio_copy_user_iov(struct request_queue *q,
return ERR_PTR(-EINVAL);

nr_pages += end - start;
len += iov[i].iov_len;
}

if (offset)
nr_pages++;

bmd = bio_alloc_map_data(iov_count, gfp_mask);
bmd = bio_alloc_map_data(iter->nr_segs, gfp_mask);
if (!bmd)
return ERR_PTR(-ENOMEM);

/*
* We need to do a deep copy of the iov_iter including the iovecs.
* The caller provided iov might point to an on-stack or otherwise
* shortlived one.
*/
bmd->is_our_pages = map_data ? 0 : 1;
memcpy(bmd->iov, iter->iov, sizeof(struct iovec) * iter->nr_segs);
iov_iter_init(&bmd->iter, iter->type, bmd->iov,
iter->nr_segs, iter->count);

ret = -ENOMEM;
bio = bio_kmalloc(gfp_mask, nr_pages);
if (!bio)
goto out_bmd;

if (!write_to_vm)
if (iter->type & WRITE)
bio->bi_rw |= REQ_WRITE;

ret = 0;
Expand Down Expand Up @@ -1238,14 +1226,14 @@ struct bio *bio_copy_user_iov(struct request_queue *q,
/*
* success
*/
if ((!write_to_vm && (!map_data || !map_data->null_mapped)) ||
if (((iter->type & WRITE) && (!map_data || !map_data->null_mapped)) ||
(map_data && map_data->from_user)) {
ret = __bio_copy_iov(bio, iov, iov_count, 0, 1);
ret = __bio_copy_iov(bio, iter, 0, 1);
if (ret)
goto cleanup;
}

bio_set_map_data(bmd, bio, iov, iov_count, map_data ? 0 : 1);
bio->bi_private = bmd;
return bio;
cleanup:
if (!map_data)
Expand All @@ -1258,19 +1246,21 @@ struct bio *bio_copy_user_iov(struct request_queue *q,

static struct bio *__bio_map_user_iov(struct request_queue *q,
struct block_device *bdev,
const struct sg_iovec *iov, int iov_count,
int write_to_vm, gfp_t gfp_mask)
const struct iov_iter *iter,
gfp_t gfp_mask)
{
int i, j;
int j;
int nr_pages = 0;
struct page **pages;
struct bio *bio;
int cur_page = 0;
int ret, offset;
struct iov_iter i;
struct iovec iov;

for (i = 0; i < iov_count; i++) {
unsigned long uaddr = (unsigned long)iov[i].iov_base;
unsigned long len = iov[i].iov_len;
iov_for_each(iov, i, *iter) {
unsigned long uaddr = (unsigned long) iov.iov_base;
unsigned long len = iov.iov_len;
unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
unsigned long start = uaddr >> PAGE_SHIFT;

Expand Down Expand Up @@ -1300,16 +1290,17 @@ static struct bio *__bio_map_user_iov(struct request_queue *q,
if (!pages)
goto out;

for (i = 0; i < iov_count; i++) {
unsigned long uaddr = (unsigned long)iov[i].iov_base;
unsigned long len = iov[i].iov_len;
iov_for_each(iov, i, *iter) {
unsigned long uaddr = (unsigned long) iov.iov_base;
unsigned long len = iov.iov_len;
unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
unsigned long start = uaddr >> PAGE_SHIFT;
const int local_nr_pages = end - start;
const int page_limit = cur_page + local_nr_pages;

ret = get_user_pages_fast(uaddr, local_nr_pages,
write_to_vm, &pages[cur_page]);
(iter->type & WRITE) != WRITE,
&pages[cur_page]);
if (ret < local_nr_pages) {
ret = -EFAULT;
goto out_unmap;
Expand Down Expand Up @@ -1349,18 +1340,18 @@ static struct bio *__bio_map_user_iov(struct request_queue *q,
/*
* set data direction, and check if mapped pages need bouncing
*/
if (!write_to_vm)
if (iter->type & WRITE)
bio->bi_rw |= REQ_WRITE;

bio->bi_bdev = bdev;
bio->bi_flags |= (1 << BIO_USER_MAPPED);
return bio;

out_unmap:
for (i = 0; i < nr_pages; i++) {
if(!pages[i])
for (j = 0; j < nr_pages; j++) {
if (!pages[j])
break;
page_cache_release(pages[i]);
page_cache_release(pages[j]);
}
out:
kfree(pages);
Expand All @@ -1369,25 +1360,22 @@ static struct bio *__bio_map_user_iov(struct request_queue *q,
}

/**
* bio_map_user_iov - map user sg_iovec table into bio
* @q: the struct request_queue for the bio
* @bdev: destination block device
* @iov: the iovec.
* @iov_count: number of elements in the iovec
* @write_to_vm: bool indicating writing to pages or not
* @gfp_mask: memory allocation flags
* bio_map_user_iov - map user iovec into bio
* @q: the struct request_queue for the bio
* @bdev: destination block device
* @iter: iovec iterator
* @gfp_mask: memory allocation flags
*
* Map the user space address into a bio suitable for io to a block
* device. Returns an error pointer in case of error.
*/
struct bio *bio_map_user_iov(struct request_queue *q, struct block_device *bdev,
const struct sg_iovec *iov, int iov_count,
int write_to_vm, gfp_t gfp_mask)
const struct iov_iter *iter,
gfp_t gfp_mask)
{
struct bio *bio;

bio = __bio_map_user_iov(q, bdev, iov, iov_count, write_to_vm,
gfp_mask);
bio = __bio_map_user_iov(q, bdev, iter, gfp_mask);
if (IS_ERR(bio))
return bio;

Expand Down
Loading

0 comments on commit 26e49cf

Please sign in to comment.