Skip to content

Commit

Permalink
Merge tag 'io_uring-5.9-2020-08-28' of git://git.kernel.dk/linux-block
Browse files Browse the repository at this point in the history 
Pull io_uring fixes from Jens Axboe:
 "A few fixes in here, all based on reports and test cases from folks
  using it. Most of it is stable material as well:

   - Hashed work cancelation fix (Pavel)

   - poll wakeup signalfd fix

   - memlock accounting fix

   - nonblocking poll retry fix

   - ensure we never return -ERESTARTSYS for reads

   - ensure offset == -1 is consistent with preadv2() as documented

   - IOPOLL -EAGAIN handling fixes

   - remove useless task_work bounce for block based -EAGAIN retry"

* tag 'io_uring-5.9-2020-08-28' of git://git.kernel.dk/linux-block:
  io_uring: don't bounce block based -EAGAIN retry off task_work
  io_uring: fix IOPOLL -EAGAIN retries
  io_uring: clear req->result on IOPOLL re-issue
  io_uring: make offset == -1 consistent with preadv2/pwritev2
  io_uring: ensure read requests go through -ERESTART* transformation
  io_uring: don't use poll handler if file can't be nonblocking read/written
  io_uring: fix imbalanced sqo_mm accounting
  io_uring: revert consumed iov_iter bytes on error
  io-wq: fix hang after cancelling pending hashed work
  io_uring: don't recurse on tsk->sighand->siglock with signalfd
  • Loading branch information
@torvalds
torvalds committed Aug 28, 2020
2 parents 005c534 + fdee946 commit 24148d8
Show file tree
Hide file tree
Showing 2 changed files with 76 additions and 44 deletions.
21 changes: 19 additions & 2 deletions fs/io-wq.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -925,6 +925,24 @@ static bool io_wq_worker_cancel(struct io_worker *worker, void *data)
return match->nr_running && !match->cancel_all;
}

static inline void io_wqe_remove_pending(struct io_wqe *wqe,
struct io_wq_work *work,
struct io_wq_work_node *prev)
{
unsigned int hash = io_get_work_hash(work);
struct io_wq_work *prev_work = NULL;

if (io_wq_is_hashed(work) && work == wqe->hash_tail[hash]) {
if (prev)
prev_work = container_of(prev, struct io_wq_work, list);
if (prev_work && io_get_work_hash(prev_work) == hash)
wqe->hash_tail[hash] = prev_work;
else
wqe->hash_tail[hash] = NULL;
}
wq_list_del(&wqe->work_list, &work->list, prev);
}

static void io_wqe_cancel_pending_work(struct io_wqe *wqe,
struct io_cb_cancel_data *match)
{
Expand All @@ -938,8 +956,7 @@ static void io_wqe_cancel_pending_work(struct io_wqe *wqe,
work = container_of(node, struct io_wq_work, list);
if (!match->fn(work, match->data))
continue;

wq_list_del(&wqe->work_list, node, prev);
io_wqe_remove_pending(wqe, work, prev);
spin_unlock_irqrestore(&wqe->lock, flags);
io_run_cancel(work, wqe);
match->nr_pending++;
Expand Down
99 changes: 57 additions & 42 deletions fs/io_uring.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1150,7 +1150,7 @@ static void io_prep_async_work(struct io_kiocb *req)
io_req_init_async(req);

if (req->flags & REQ_F_ISREG) {
if (def->hash_reg_file)
if (def->hash_reg_file || (req->ctx->flags & IORING_SETUP_IOPOLL))
io_wq_hash_work(&req->work, file_inode(req->file));
} else {
if (def->unbound_nonreg_file)
Expand Down Expand Up @@ -1746,7 +1746,8 @@ static struct io_kiocb *io_req_find_next(struct io_kiocb *req)
return __io_req_find_next(req);
}

static int io_req_task_work_add(struct io_kiocb *req, struct callback_head *cb)
static int io_req_task_work_add(struct io_kiocb *req, struct callback_head *cb,
bool twa_signal_ok)
{
struct task_struct *tsk = req->task;
struct io_ring_ctx *ctx = req->ctx;
Expand All @@ -1759,7 +1760,7 @@ static int io_req_task_work_add(struct io_kiocb *req, struct callback_head *cb)
* will do the job.
*/
notify = 0;
if (!(ctx->flags & IORING_SETUP_SQPOLL))
if (!(ctx->flags & IORING_SETUP_SQPOLL) && twa_signal_ok)
notify = TWA_SIGNAL;

ret = task_work_add(tsk, cb, notify);
Expand Down Expand Up @@ -1819,7 +1820,7 @@ static void io_req_task_queue(struct io_kiocb *req)
init_task_work(&req->task_work, io_req_task_submit);
percpu_ref_get(&req->ctx->refs);

ret = io_req_task_work_add(req, &req->task_work);
ret = io_req_task_work_add(req, &req->task_work, true);
if (unlikely(ret)) {
struct task_struct *tsk;

Expand Down Expand Up @@ -2048,6 +2049,7 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,

req = list_first_entry(done, struct io_kiocb, inflight_entry);
if (READ_ONCE(req->result) == -EAGAIN) {
req->result = 0;
req->iopoll_completed = 0;
list_move_tail(&req->inflight_entry, &again);
continue;
Expand Down Expand Up @@ -2293,22 +2295,6 @@ static bool io_resubmit_prep(struct io_kiocb *req, int error)
io_req_complete(req, ret);
return false;
}

static void io_rw_resubmit(struct callback_head *cb)
{
struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
struct io_ring_ctx *ctx = req->ctx;
int err;

err = io_sq_thread_acquire_mm(ctx, req);

if (io_resubmit_prep(req, err)) {
refcount_inc(&req->refs);
io_queue_async_work(req);
}

percpu_ref_put(&ctx->refs);
}
#endif

static bool io_rw_reissue(struct io_kiocb *req, long res)
Expand All @@ -2319,12 +2305,14 @@ static bool io_rw_reissue(struct io_kiocb *req, long res)
if ((res != -EAGAIN && res != -EOPNOTSUPP) || io_wq_current_is_worker())
return false;

init_task_work(&req->task_work, io_rw_resubmit);
percpu_ref_get(&req->ctx->refs);
ret = io_sq_thread_acquire_mm(req->ctx, req);

ret = io_req_task_work_add(req, &req->task_work);
if (!ret)
if (io_resubmit_prep(req, ret)) {
refcount_inc(&req->refs);
io_queue_async_work(req);
return true;
}

#endif
return false;
}
Expand Down Expand Up @@ -2865,6 +2853,11 @@ static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
return iov_iter_count(&req->io->rw.iter);
}

static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)
{
return kiocb->ki_filp->f_mode & FMODE_STREAM ? NULL : &kiocb->ki_pos;
}

/*
* For files that don't have ->read_iter() and ->write_iter(), handle them
* by looping over ->read() or ->write() manually.
Expand Down Expand Up @@ -2900,10 +2893,10 @@ static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,

if (rw == READ) {
nr = file->f_op->read(file, iovec.iov_base,
iovec.iov_len, &kiocb->ki_pos);
iovec.iov_len, io_kiocb_ppos(kiocb));
} else {
nr = file->f_op->write(file, iovec.iov_base,
iovec.iov_len, &kiocb->ki_pos);
iovec.iov_len, io_kiocb_ppos(kiocb));
}

if (iov_iter_is_bvec(iter))
Expand Down Expand Up @@ -3044,7 +3037,7 @@ static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode,

/* submit ref gets dropped, acquire a new one */
refcount_inc(&req->refs);
ret = io_req_task_work_add(req, &req->task_work);
ret = io_req_task_work_add(req, &req->task_work, true);
if (unlikely(ret)) {
struct task_struct *tsk;

Expand Down Expand Up @@ -3125,6 +3118,7 @@ static int io_read(struct io_kiocb *req, bool force_nonblock,
ret = io_import_iovec(READ, req, &iovec, iter, !force_nonblock);
if (ret < 0)
return ret;
iov_count = iov_iter_count(iter);
io_size = ret;
req->result = io_size;
ret = 0;
Expand All @@ -3137,8 +3131,7 @@ static int io_read(struct io_kiocb *req, bool force_nonblock,
if (force_nonblock && !io_file_supports_async(req->file, READ))
goto copy_iov;

iov_count = iov_iter_count(iter);
ret = rw_verify_area(READ, req->file, &kiocb->ki_pos, iov_count);
ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), iov_count);
if (unlikely(ret))
goto out_free;

Expand All @@ -3150,14 +3143,18 @@ static int io_read(struct io_kiocb *req, bool force_nonblock,
ret = 0;
goto out_free;
} else if (ret == -EAGAIN) {
if (!force_nonblock)
/* IOPOLL retry should happen for io-wq threads */
if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL))
goto done;
/* some cases will consume bytes even on error returns */
iov_iter_revert(iter, iov_count - iov_iter_count(iter));
ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false);
if (ret)
goto out_free;
return -EAGAIN;
} else if (ret < 0) {
goto out_free;
/* make sure -ERESTARTSYS -> -EINTR is done */
goto done;
}

/* read it all, or we did blocking attempt. no retry. */
Expand Down Expand Up @@ -3241,6 +3238,7 @@ static int io_write(struct io_kiocb *req, bool force_nonblock,
ret = io_import_iovec(WRITE, req, &iovec, iter, !force_nonblock);
if (ret < 0)
return ret;
iov_count = iov_iter_count(iter);
io_size = ret;
req->result = io_size;

Expand All @@ -3257,8 +3255,7 @@ static int io_write(struct io_kiocb *req, bool force_nonblock,
(req->flags & REQ_F_ISREG))
goto copy_iov;

iov_count = iov_iter_count(iter);
ret = rw_verify_area(WRITE, req->file, &kiocb->ki_pos, iov_count);
ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), iov_count);
if (unlikely(ret))
goto out_free;

Expand Down Expand Up @@ -3291,9 +3288,14 @@ static int io_write(struct io_kiocb *req, bool force_nonblock,
if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
ret2 = -EAGAIN;
if (!force_nonblock || ret2 != -EAGAIN) {
/* IOPOLL retry should happen for io-wq threads */
if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN)
goto copy_iov;
kiocb_done(kiocb, ret2, cs);
} else {
copy_iov:
/* some cases will consume bytes even on error returns */
iov_iter_revert(iter, iov_count - iov_iter_count(iter));
ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false);
if (!ret)
return -EAGAIN;
Expand Down Expand Up @@ -4566,6 +4568,7 @@ struct io_poll_table {
static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
__poll_t mask, task_work_func_t func)
{
bool twa_signal_ok;
int ret;

/* for instances that support it check for an event match first: */
Expand All @@ -4580,13 +4583,21 @@ static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
init_task_work(&req->task_work, func);
percpu_ref_get(&req->ctx->refs);

/*
* If we using the signalfd wait_queue_head for this wakeup, then
* it's not safe to use TWA_SIGNAL as we could be recursing on the
* tsk->sighand->siglock on doing the wakeup. Should not be needed
* either, as the normal wakeup will suffice.
*/
twa_signal_ok = (poll->head != &req->task->sighand->signalfd_wqh);

/*
* If this fails, then the task is exiting. When a task exits, the
* work gets canceled, so just cancel this request as well instead
* of executing it. We can't safely execute it anyway, as we may not
* have the needed state needed for it anyway.
*/
ret = io_req_task_work_add(req, &req->task_work);
ret = io_req_task_work_add(req, &req->task_work, twa_signal_ok);
if (unlikely(ret)) {
struct task_struct *tsk;

Expand Down Expand Up @@ -4875,12 +4886,20 @@ static bool io_arm_poll_handler(struct io_kiocb *req)
struct async_poll *apoll;
struct io_poll_table ipt;
__poll_t mask, ret;
int rw;

if (!req->file || !file_can_poll(req->file))
return false;
if (req->flags & REQ_F_POLLED)
return false;
if (!def->pollin && !def->pollout)
if (def->pollin)
rw = READ;
else if (def->pollout)
rw = WRITE;
else
return false;
/* if we can't nonblock try, then no point in arming a poll handler */
if (!io_file_supports_async(req->file, rw))
return false;

apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
Expand Down Expand Up @@ -7433,9 +7452,6 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx,
{
int ret;

mmgrab(current->mm);
ctx->sqo_mm = current->mm;

if (ctx->flags & IORING_SETUP_SQPOLL) {
ret = -EPERM;
if (!capable(CAP_SYS_ADMIN))
Expand Down Expand Up @@ -7480,10 +7496,6 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx,
return 0;
err:
io_finish_async(ctx);
if (ctx->sqo_mm) {
mmdrop(ctx->sqo_mm);
ctx->sqo_mm = NULL;
}
return ret;
}

Expand Down Expand Up @@ -8533,6 +8545,9 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p,
ctx->user = user;
ctx->creds = get_current_cred();

mmgrab(current->mm);
ctx->sqo_mm = current->mm;

/*
* Account memory _before_ installing the file descriptor. Once
* the descriptor is installed, it can get closed at any time. Also
Expand Down

0 comments on commit 24148d8

Please sign in to comment.