Skip to content

Commit

Permalink
Merge tag 'io_uring-5.11-2021-01-24' of git://git.kernel.dk/linux-block
Browse files Browse the repository at this point in the history 
Pull io_uring fixes from Jens Axboe:
 "Still need a final cancelation fix that isn't quite done done,
  expected in the next day or two. That said, this contains:

   - Wakeup fix for IOPOLL requests

   - SQPOLL split close op handling fix

   - Ensure that any use of io_uring fd itself is marked as inflight

   - Short non-regular file read fix (Pavel)

   - Fix up bad false positive warning (Pavel)

   - SQPOLL fixes (Pavel)

   - In-flight removal fix (Pavel)"

* tag 'io_uring-5.11-2021-01-24' of git://git.kernel.dk/linux-block:
  io_uring: account io_uring internal files as REQ_F_INFLIGHT
  io_uring: fix sleeping under spin in __io_clean_op
  io_uring: fix short read retries for non-reg files
  io_uring: fix SQPOLL IORING_OP_CLOSE cancelation state
  io_uring: fix skipping disabling sqo on exec
  io_uring: fix uring_flush in exit_files() warning
  io_uring: fix false positive sqo warning on flush
  io_uring: iopoll requests should also wake task ->in_idle state
  • Loading branch information
@torvalds
torvalds committed Jan 24, 2021
2 parents a692a61 + 02a1367 commit ef7b1a0
Showing 1 changed file with 47 additions and 20 deletions.
67 changes: 47 additions & 20 deletions fs/io_uring.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1025,6 +1025,7 @@ static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,
const struct iovec *fast_iov,
struct iov_iter *iter, bool force);
static void io_req_drop_files(struct io_kiocb *req);

static struct kmem_cache *req_cachep;

Expand All @@ -1048,8 +1049,7 @@ EXPORT_SYMBOL(io_uring_get_socket);

static inline void io_clean_op(struct io_kiocb *req)
{
if (req->flags & (REQ_F_NEED_CLEANUP | REQ_F_BUFFER_SELECTED |
REQ_F_INFLIGHT))
if (req->flags & (REQ_F_NEED_CLEANUP | REQ_F_BUFFER_SELECTED))
__io_clean_op(req);
}

Expand All @@ -1075,8 +1075,11 @@ static bool io_match_task(struct io_kiocb *head,
return true;

io_for_each_link(req, head) {
if ((req->flags & REQ_F_WORK_INITIALIZED) &&
(req->work.flags & IO_WQ_WORK_FILES) &&
if (!(req->flags & REQ_F_WORK_INITIALIZED))
continue;
if (req->file && req->file->f_op == &io_uring_fops)
return true;
if ((req->work.flags & IO_WQ_WORK_FILES) &&
req->work.identity->files == files)
return true;
}
Expand Down Expand Up @@ -1394,6 +1397,8 @@ static void io_req_clean_work(struct io_kiocb *req)
free_fs_struct(fs);
req->work.flags &= ~IO_WQ_WORK_FS;
}
if (req->flags & REQ_F_INFLIGHT)
io_req_drop_files(req);

io_put_identity(req->task->io_uring, req);
}
Expand Down Expand Up @@ -1503,11 +1508,14 @@ static bool io_grab_identity(struct io_kiocb *req)
return false;
atomic_inc(&id->files->count);
get_nsproxy(id->nsproxy);
req->flags |= REQ_F_INFLIGHT;

spin_lock_irq(&ctx->inflight_lock);
list_add(&req->inflight_entry, &ctx->inflight_list);
spin_unlock_irq(&ctx->inflight_lock);
if (!(req->flags & REQ_F_INFLIGHT)) {
req->flags |= REQ_F_INFLIGHT;

spin_lock_irq(&ctx->inflight_lock);
list_add(&req->inflight_entry, &ctx->inflight_list);
spin_unlock_irq(&ctx->inflight_lock);
}
req->work.flags |= IO_WQ_WORK_FILES;
}
if (!(req->work.flags & IO_WQ_WORK_MM) &&
Expand Down Expand Up @@ -2270,6 +2278,8 @@ static void io_req_free_batch_finish(struct io_ring_ctx *ctx,
struct io_uring_task *tctx = rb->task->io_uring;

percpu_counter_sub(&tctx->inflight, rb->task_refs);
if (atomic_read(&tctx->in_idle))
wake_up(&tctx->wait);
put_task_struct_many(rb->task, rb->task_refs);
rb->task = NULL;
}
Expand All @@ -2288,6 +2298,8 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req)
struct io_uring_task *tctx = rb->task->io_uring;

percpu_counter_sub(&tctx->inflight, rb->task_refs);
if (atomic_read(&tctx->in_idle))
wake_up(&tctx->wait);
put_task_struct_many(rb->task, rb->task_refs);
}
rb->task = req->task;
Expand Down Expand Up @@ -3548,7 +3560,7 @@ static int io_read(struct io_kiocb *req, bool force_nonblock,

/* read it all, or we did blocking attempt. no retry. */
if (!iov_iter_count(iter) || !force_nonblock ||
(req->file->f_flags & O_NONBLOCK))
(req->file->f_flags & O_NONBLOCK) || !(req->flags & REQ_F_ISREG))
goto done;

io_size -= ret;
Expand Down Expand Up @@ -4468,7 +4480,6 @@ static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
* io_wq_work.flags, so initialize io_wq_work firstly.
*/
io_req_init_async(req);
req->work.flags |= IO_WQ_WORK_NO_CANCEL;

if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
return -EINVAL;
Expand Down Expand Up @@ -4501,6 +4512,8 @@ static int io_close(struct io_kiocb *req, bool force_nonblock,

/* if the file has a flush method, be safe and punt to async */
if (close->put_file->f_op->flush && force_nonblock) {
/* not safe to cancel at this point */
req->work.flags |= IO_WQ_WORK_NO_CANCEL;
/* was never set, but play safe */
req->flags &= ~REQ_F_NOWAIT;
/* avoid grabbing files - we don't need the files */
Expand Down Expand Up @@ -6157,8 +6170,10 @@ static void io_req_drop_files(struct io_kiocb *req)
struct io_uring_task *tctx = req->task->io_uring;
unsigned long flags;

put_files_struct(req->work.identity->files);
put_nsproxy(req->work.identity->nsproxy);
if (req->work.flags & IO_WQ_WORK_FILES) {
put_files_struct(req->work.identity->files);
put_nsproxy(req->work.identity->nsproxy);
}
spin_lock_irqsave(&ctx->inflight_lock, flags);
list_del(&req->inflight_entry);
spin_unlock_irqrestore(&ctx->inflight_lock, flags);
Expand Down Expand Up @@ -6225,9 +6240,6 @@ static void __io_clean_op(struct io_kiocb *req)
}
req->flags &= ~REQ_F_NEED_CLEANUP;
}

if (req->flags & REQ_F_INFLIGHT)
io_req_drop_files(req);
}

static int io_issue_sqe(struct io_kiocb *req, bool force_nonblock,
Expand Down Expand Up @@ -6446,6 +6458,15 @@ static struct file *io_file_get(struct io_submit_state *state,
file = __io_file_get(state, fd);
}

if (file && file->f_op == &io_uring_fops) {
io_req_init_async(req);
req->flags |= REQ_F_INFLIGHT;

spin_lock_irq(&ctx->inflight_lock);
list_add(&req->inflight_entry, &ctx->inflight_list);
spin_unlock_irq(&ctx->inflight_lock);
}

return file;
}

Expand Down Expand Up @@ -8856,8 +8877,7 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx,

spin_lock_irq(&ctx->inflight_lock);
list_for_each_entry(req, &ctx->inflight_list, inflight_entry) {
if (req->task != task ||
req->work.identity->files != files)
if (!io_match_task(req, task, files))
continue;
found = true;
break;
Expand All @@ -8874,6 +8894,7 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx,
io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, &cancel, true);
io_poll_remove_all(ctx, task, files);
io_kill_timeouts(ctx, task, files);
io_cqring_overflow_flush(ctx, true, task, files);
/* cancellations _may_ trigger task work */
io_run_task_work();
schedule();
Expand Down Expand Up @@ -8914,8 +8935,6 @@ static void __io_uring_cancel_task_requests(struct io_ring_ctx *ctx,

static void io_disable_sqo_submit(struct io_ring_ctx *ctx)
{
WARN_ON_ONCE(ctx->sqo_task != current);

mutex_lock(&ctx->uring_lock);
ctx->sqo_dead = 1;
mutex_unlock(&ctx->uring_lock);
Expand All @@ -8937,6 +8956,7 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx,

if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sq_data) {
/* for SQPOLL only sqo_task has task notes */
WARN_ON_ONCE(ctx->sqo_task != current);
io_disable_sqo_submit(ctx);
task = ctx->sq_data->thread;
atomic_inc(&task->io_uring->in_idle);
Expand Down Expand Up @@ -9082,6 +9102,10 @@ void __io_uring_task_cancel(void)
/* make sure overflow events are dropped */
atomic_inc(&tctx->in_idle);

/* trigger io_disable_sqo_submit() */
if (tctx->sqpoll)
__io_uring_files_cancel(NULL);

do {
/* read completions before cancelations */
inflight = tctx_inflight(tctx);
Expand Down Expand Up @@ -9128,7 +9152,10 @@ static int io_uring_flush(struct file *file, void *data)

if (ctx->flags & IORING_SETUP_SQPOLL) {
/* there is only one file note, which is owned by sqo_task */
WARN_ON_ONCE((ctx->sqo_task == current) ==
WARN_ON_ONCE(ctx->sqo_task != current &&
xa_load(&tctx->xa, (unsigned long)file));
/* sqo_dead check is for when this happens after cancellation */
WARN_ON_ONCE(ctx->sqo_task == current && !ctx->sqo_dead &&
!xa_load(&tctx->xa, (unsigned long)file));

io_disable_sqo_submit(ctx);
Expand Down

0 comments on commit ef7b1a0

Please sign in to comment.