From fe1cdd558619546f76643878e7aa521c32d52131 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 17 Feb 2021 21:02:36 +0000 Subject: [PATCH 01/25] io_uring: fix read memory leak Don't forget to free iovec read inline completion and bunch of other cases that do "goto done" before setting up an async context. Fixes: 5ea5dd45844d ("io_uring: inline io_read()'s iovec freeing") Reported-by: Jens Axboe Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 58dd104811062f..4352bcea3d9d1c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3602,10 +3602,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) ret = io_iter_do_read(req, iter); if (ret == -EIOCBQUEUED) { - /* it's faster to check here then delegate to kfree */ - if (iovec) - kfree(iovec); - return 0; + goto out_free; } else if (ret == -EAGAIN) { /* IOPOLL retry should happen for io-wq threads */ if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) @@ -3626,6 +3623,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) if (ret2) return ret2; + iovec = NULL; rw = req->async_data; /* now use our persistent iterator, if we aren't already */ iter = &rw->iter; @@ -3652,6 +3650,10 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) } while (ret > 0 && ret < io_size); done: kiocb_done(kiocb, ret, issue_flags); +out_free: + /* it's faster to check here then delegate to kfree */ + if (iovec) + kfree(iovec); return 0; } From 46c4e16a8625f7afdd8eee1ac8c3b3e592cba974 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:37 +0000 Subject: [PATCH 02/25] io_uring: kill fictitious submit iteration index @i and @submitted are very much coupled together, and there is no need to keep them both. Remove @i, it doesn't change generated binary but helps to keep a single source of truth. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 4352bcea3d9d1c..32a6c89e69b161 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6884,7 +6884,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) { struct io_submit_link link; - int i, submitted = 0; + int submitted = 0; /* if we have a backlog and couldn't flush it all, return BUSY */ if (test_bit(0, &ctx->sq_check_overflow)) { @@ -6904,7 +6904,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) io_submit_state_start(&ctx->submit_state, nr); link.head = NULL; - for (i = 0; i < nr; i++) { + while (submitted < nr) { const struct io_uring_sqe *sqe; struct io_kiocb *req; int err; From 1155c76a248364dd182bde90fea6f5682a6a766f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:38 +0000 Subject: [PATCH 03/25] io_uring: keep io_*_prep() naming consistent Follow io_*_prep() naming pattern, there are only fsync and sfr that don't do that. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 32a6c89e69b161..adb5cd4b760d7e 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -4008,7 +4008,7 @@ static int io_nop(struct io_kiocb *req, unsigned int issue_flags) return 0; } -static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe) +static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_ring_ctx *ctx = req->ctx; @@ -4595,7 +4595,7 @@ static int io_close(struct io_kiocb *req, unsigned int issue_flags) return 0; } -static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe) +static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_ring_ctx *ctx = req->ctx; @@ -6081,9 +6081,9 @@ static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) case IORING_OP_POLL_REMOVE: return io_poll_remove_prep(req, sqe); case IORING_OP_FSYNC: - return io_prep_fsync(req, sqe); + return io_fsync_prep(req, sqe); case IORING_OP_SYNC_FILE_RANGE: - return io_prep_sfr(req, sqe); + return io_sfr_prep(req, sqe); case IORING_OP_SENDMSG: case IORING_OP_SEND: return io_sendmsg_prep(req, sqe); From 441960f3b9b8ee6aeea847e3e67093e0840e7059 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:39 +0000 Subject: [PATCH 04/25] io_uring: don't duplicate ->file check in sfr IORING_OP_SYNC_FILE_RANGE is marked as .needs_file, so the common path will take care of assigning and validating req->file, no need to duplicate it in io_sfr_prep(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index adb5cd4b760d7e..db6680bb02d3a8 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -4599,9 +4599,6 @@ static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_ring_ctx *ctx = req->ctx; - if (!req->file) - return -EBADF; - if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) return -EINVAL; if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index)) From b16fed66bc7dca1a5dfd0af8991e9f58b5ef8d5f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:40 +0000 Subject: [PATCH 05/25] io_uring: move io_init_req()'s definition A preparation patch, symbol to symbol move io_init_req() + io_check_restriction() a bit up. The submission path is pretty settled down, so don't worry about backports and move the functions instead of relying on forward declarations in the future. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 214 +++++++++++++++++++++++++------------------------- 1 file changed, 107 insertions(+), 107 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index db6680bb02d3a8..1563853caac5f3 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -104,6 +104,10 @@ #define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \ IORING_REGISTER_LAST + IORING_OP_LAST) +#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \ + IOSQE_IO_HARDLINK | IOSQE_ASYNC | \ + IOSQE_BUFFER_SELECT) + struct io_uring { u32 head ____cacheline_aligned_in_smp; u32 tail ____cacheline_aligned_in_smp; @@ -6639,6 +6643,109 @@ static inline void io_queue_link_head(struct io_kiocb *req) io_queue_sqe(req, NULL); } +/* + * Check SQE restrictions (opcode and flags). + * + * Returns 'true' if SQE is allowed, 'false' otherwise. + */ +static inline bool io_check_restriction(struct io_ring_ctx *ctx, + struct io_kiocb *req, + unsigned int sqe_flags) +{ + if (!ctx->restricted) + return true; + + if (!test_bit(req->opcode, ctx->restrictions.sqe_op)) + return false; + + if ((sqe_flags & ctx->restrictions.sqe_flags_required) != + ctx->restrictions.sqe_flags_required) + return false; + + if (sqe_flags & ~(ctx->restrictions.sqe_flags_allowed | + ctx->restrictions.sqe_flags_required)) + return false; + + return true; +} + +static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, + const struct io_uring_sqe *sqe) +{ + struct io_submit_state *state; + unsigned int sqe_flags; + int id, ret = 0; + + req->opcode = READ_ONCE(sqe->opcode); + /* same numerical values with corresponding REQ_F_*, safe to copy */ + req->flags = sqe_flags = READ_ONCE(sqe->flags); + req->user_data = READ_ONCE(sqe->user_data); + req->async_data = NULL; + req->file = NULL; + req->ctx = ctx; + req->link = NULL; + req->fixed_rsrc_refs = NULL; + /* one is dropped after submission, the other at completion */ + refcount_set(&req->refs, 2); + req->task = current; + req->result = 0; + + /* enforce forwards compatibility on users */ + if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) + return -EINVAL; + + if (unlikely(req->opcode >= IORING_OP_LAST)) + return -EINVAL; + + if (unlikely(io_sq_thread_acquire_mm_files(ctx, req))) + return -EFAULT; + + if (unlikely(!io_check_restriction(ctx, req, sqe_flags))) + return -EACCES; + + if ((sqe_flags & IOSQE_BUFFER_SELECT) && + !io_op_defs[req->opcode].buffer_select) + return -EOPNOTSUPP; + + id = READ_ONCE(sqe->personality); + if (id) { + struct io_identity *iod; + + iod = idr_find(&ctx->personality_idr, id); + if (unlikely(!iod)) + return -EINVAL; + refcount_inc(&iod->count); + + __io_req_init_async(req); + get_cred(iod->creds); + req->work.identity = iod; + req->work.flags |= IO_WQ_WORK_CREDS; + } + + state = &ctx->submit_state; + + /* + * Plug now if we have more than 1 IO left after this, and the target + * is potentially a read/write to block based storage. + */ + if (!state->plug_started && state->ios_left > 1 && + io_op_defs[req->opcode].plug) { + blk_start_plug(&state->plug); + state->plug_started = true; + } + + if (io_op_defs[req->opcode].needs_file) { + bool fixed = req->flags & REQ_F_FIXED_FILE; + + req->file = io_file_get(state, req, READ_ONCE(sqe->fd), fixed); + if (unlikely(!req->file)) + ret = -EBADF; + } + + state->ios_left--; + return ret; +} + struct io_submit_link { struct io_kiocb *head; struct io_kiocb *last; @@ -6771,113 +6878,6 @@ static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) return NULL; } -/* - * Check SQE restrictions (opcode and flags). - * - * Returns 'true' if SQE is allowed, 'false' otherwise. - */ -static inline bool io_check_restriction(struct io_ring_ctx *ctx, - struct io_kiocb *req, - unsigned int sqe_flags) -{ - if (!ctx->restricted) - return true; - - if (!test_bit(req->opcode, ctx->restrictions.sqe_op)) - return false; - - if ((sqe_flags & ctx->restrictions.sqe_flags_required) != - ctx->restrictions.sqe_flags_required) - return false; - - if (sqe_flags & ~(ctx->restrictions.sqe_flags_allowed | - ctx->restrictions.sqe_flags_required)) - return false; - - return true; -} - -#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \ - IOSQE_IO_HARDLINK | IOSQE_ASYNC | \ - IOSQE_BUFFER_SELECT) - -static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, - const struct io_uring_sqe *sqe) -{ - struct io_submit_state *state; - unsigned int sqe_flags; - int id, ret = 0; - - req->opcode = READ_ONCE(sqe->opcode); - /* same numerical values with corresponding REQ_F_*, safe to copy */ - req->flags = sqe_flags = READ_ONCE(sqe->flags); - req->user_data = READ_ONCE(sqe->user_data); - req->async_data = NULL; - req->file = NULL; - req->ctx = ctx; - req->link = NULL; - req->fixed_rsrc_refs = NULL; - /* one is dropped after submission, the other at completion */ - refcount_set(&req->refs, 2); - req->task = current; - req->result = 0; - - /* enforce forwards compatibility on users */ - if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) - return -EINVAL; - - if (unlikely(req->opcode >= IORING_OP_LAST)) - return -EINVAL; - - if (unlikely(io_sq_thread_acquire_mm_files(ctx, req))) - return -EFAULT; - - if (unlikely(!io_check_restriction(ctx, req, sqe_flags))) - return -EACCES; - - if ((sqe_flags & IOSQE_BUFFER_SELECT) && - !io_op_defs[req->opcode].buffer_select) - return -EOPNOTSUPP; - - id = READ_ONCE(sqe->personality); - if (id) { - struct io_identity *iod; - - iod = idr_find(&ctx->personality_idr, id); - if (unlikely(!iod)) - return -EINVAL; - refcount_inc(&iod->count); - - __io_req_init_async(req); - get_cred(iod->creds); - req->work.identity = iod; - req->work.flags |= IO_WQ_WORK_CREDS; - } - - state = &ctx->submit_state; - - /* - * Plug now if we have more than 1 IO left after this, and the target - * is potentially a read/write to block based storage. - */ - if (!state->plug_started && state->ios_left > 1 && - io_op_defs[req->opcode].plug) { - blk_start_plug(&state->plug); - state->plug_started = true; - } - - if (io_op_defs[req->opcode].needs_file) { - bool fixed = req->flags & REQ_F_FIXED_FILE; - - req->file = io_file_get(state, req, READ_ONCE(sqe->fd), fixed); - if (unlikely(!req->file)) - ret = -EBADF; - } - - state->ios_left--; - return ret; -} - static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) { struct io_submit_link link; From a6b8cadcea86da0fe92de5c2e6e82824cb6fb57c Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:41 +0000 Subject: [PATCH 06/25] io_uring: move io_init_req() into io_submit_sqe() Behaves identically, just move io_init_req() call into the beginning of io_submit_sqes(). That looks better unloads io_submit_sqes(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 1563853caac5f3..5c9b3b9ff92fb7 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6751,12 +6751,23 @@ struct io_submit_link { struct io_kiocb *last; }; -static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, +static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, + const struct io_uring_sqe *sqe, struct io_submit_link *link) { - struct io_ring_ctx *ctx = req->ctx; int ret; + ret = io_init_req(ctx, req, sqe); + if (unlikely(ret)) { +fail_req: + io_put_req(req); + io_req_complete(req, ret); + return ret; + } + + trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data, + true, ctx->flags & IORING_SETUP_SQPOLL); + /* * If we already have a head request, queue this one for async * submittal once the head completes. If we don't have a head but @@ -6782,7 +6793,7 @@ static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, if (unlikely(ret)) { /* fail even hard links since we don't submit */ head->flags |= REQ_F_FAIL_LINK; - return ret; + goto fail_req; } trace_io_uring_link(ctx, req, head); link->last->link = req; @@ -6904,7 +6915,6 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) while (submitted < nr) { const struct io_uring_sqe *sqe; struct io_kiocb *req; - int err; req = io_alloc_req(ctx); if (unlikely(!req)) { @@ -6919,20 +6929,8 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) } /* will complete beyond this point, count as submitted */ submitted++; - - err = io_init_req(ctx, req, sqe); - if (unlikely(err)) { -fail_req: - io_put_req(req); - io_req_complete(req, err); + if (io_submit_sqe(ctx, req, sqe, &link)) break; - } - - trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data, - true, ctx->flags & IORING_SETUP_SQPOLL); - err = io_submit_sqe(req, sqe, &link); - if (err) - goto fail_req; } if (unlikely(submitted != nr)) { From a1ab7b35db8f262cd74edff62b47b4d90f84f997 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:42 +0000 Subject: [PATCH 07/25] io_uring: move req link into submit_state Move struct io_submit_link into submit_state, which is a part of a submission state and so belongs to it. It saves us from explicitly passing it, and init/deinit is now nicely hidden in io_submit_state_[start,end]. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 5c9b3b9ff92fb7..fe2379179b006c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -283,8 +283,14 @@ struct io_comp_state { struct list_head locked_free_list; }; +struct io_submit_link { + struct io_kiocb *head; + struct io_kiocb *last; +}; + struct io_submit_state { struct blk_plug plug; + struct io_submit_link link; /* * io_kiocb alloc cache @@ -6746,15 +6752,10 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, return ret; } -struct io_submit_link { - struct io_kiocb *head; - struct io_kiocb *last; -}; - static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, - const struct io_uring_sqe *sqe, - struct io_submit_link *link) + const struct io_uring_sqe *sqe) { + struct io_submit_link *link = &ctx->submit_state.link; int ret; ret = io_init_req(ctx, req, sqe); @@ -6829,6 +6830,8 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, static void io_submit_state_end(struct io_submit_state *state, struct io_ring_ctx *ctx) { + if (state->link.head) + io_queue_link_head(state->link.head); if (state->comp.nr) io_submit_flush_completions(&state->comp, ctx); if (state->plug_started) @@ -6844,6 +6847,8 @@ static void io_submit_state_start(struct io_submit_state *state, { state->plug_started = false; state->ios_left = max_ios; + /* set only head, no need to init link_last in advance */ + state->link.head = NULL; } static void io_commit_sqring(struct io_ring_ctx *ctx) @@ -6891,7 +6896,6 @@ static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) { - struct io_submit_link link; int submitted = 0; /* if we have a backlog and couldn't flush it all, return BUSY */ @@ -6908,9 +6912,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) percpu_counter_add(¤t->io_uring->inflight, nr); refcount_add(nr, ¤t->usage); - io_submit_state_start(&ctx->submit_state, nr); - link.head = NULL; while (submitted < nr) { const struct io_uring_sqe *sqe; @@ -6929,7 +6931,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) } /* will complete beyond this point, count as submitted */ submitted++; - if (io_submit_sqe(ctx, req, sqe, &link)) + if (io_submit_sqe(ctx, req, sqe)) break; } @@ -6942,10 +6944,8 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) percpu_counter_sub(&tctx->inflight, unused); put_task_struct_many(current, unused); } - if (link.head) - io_queue_link_head(link.head); - io_submit_state_end(&ctx->submit_state, ctx); + io_submit_state_end(&ctx->submit_state, ctx); /* Commit SQ ring head once we've consumed and submitted all SQEs */ io_commit_sqring(ctx); From cf109604265156bb22c45e0c2aa62f53a697a3f4 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:43 +0000 Subject: [PATCH 08/25] io_uring: don't submit link on error If we get an error in io_init_req() for a request that would have been linked, we break the submission but still issue a partially composed link, that's nasty, fail it instead. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index fe2379179b006c..62688866357cf7 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6763,6 +6763,9 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, fail_req: io_put_req(req); io_req_complete(req, ret); + /* fail even hard links since we don't submit */ + if (link->head) + link->head->flags |= REQ_F_FAIL_LINK; return ret; } @@ -6791,11 +6794,8 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, ctx->drain_next = 1; } ret = io_req_defer_prep(req, sqe); - if (unlikely(ret)) { - /* fail even hard links since we don't submit */ - head->flags |= REQ_F_FAIL_LINK; + if (unlikely(ret)) goto fail_req; - } trace_io_uring_link(ctx, req, head); link->last->link = req; link->last = req; From 93642ef8843445f72a1e6b0c68914746c7aa5b9c Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:44 +0000 Subject: [PATCH 09/25] io_uring: split sqe-prep and async setup There are two kinds of opcode-specific preparations we do. The first is just initialising req with what is always needed for an opcode and reading all non-generic SQE fields. And the second is copying some of the stuff like iovec preparing to punt a request to somewhere async, e.g. to io-wq or for draining. For requests that have tried an inline execution but still needing to be punted, the second prep type is done by the opcode handler itself. Currently, we don't explicitly split those preparation steps, but combining both of them into io_*_prep(), altering the behaviour by allocating ->async_data. That's pretty messy and hard to follow and also gets in the way of some optimisations. Split the steps, leave the first type as where it is now, and put the second into a new io_req_prep_async() helper. It may make us to do opcode switch twice, but it's worth it. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 120 +++++++++++++++++++++++++++++--------------------- 1 file changed, 70 insertions(+), 50 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 62688866357cf7..987cfd8db2134b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3472,19 +3472,9 @@ static inline int io_rw_prep_async(struct io_kiocb *req, int rw) static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - ssize_t ret; - - ret = io_prep_rw(req, sqe); - if (ret) - return ret; - if (unlikely(!(req->file->f_mode & FMODE_READ))) return -EBADF; - - /* either don't need iovec imported or already have it */ - if (!req->async_data) - return 0; - return io_rw_prep_async(req, READ); + return io_prep_rw(req, sqe); } /* @@ -3669,19 +3659,9 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - ssize_t ret; - - ret = io_prep_rw(req, sqe); - if (ret) - return ret; - if (unlikely(!(req->file->f_mode & FMODE_WRITE))) return -EBADF; - - /* either don't need iovec imported or already have it */ - if (!req->async_data) - return 0; - return io_rw_prep_async(req, WRITE); + return io_prep_rw(req, sqe); } static int io_write(struct io_kiocb *req, unsigned int issue_flags) @@ -4668,11 +4648,21 @@ static int io_sendmsg_copy_hdr(struct io_kiocb *req, req->sr_msg.msg_flags, &iomsg->free_iov); } +static int io_sendmsg_prep_async(struct io_kiocb *req) +{ + int ret; + + if (!io_op_defs[req->opcode].needs_async_data) + return 0; + ret = io_sendmsg_copy_hdr(req, req->async_data); + if (!ret) + req->flags |= REQ_F_NEED_CLEANUP; + return ret; +} + static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_async_msghdr *async_msg = req->async_data; struct io_sr_msg *sr = &req->sr_msg; - int ret; if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) return -EINVAL; @@ -4685,13 +4675,7 @@ static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (req->ctx->compat) sr->msg_flags |= MSG_CMSG_COMPAT; #endif - - if (!async_msg || !io_op_defs[req->opcode].needs_async_data) - return 0; - ret = io_sendmsg_copy_hdr(req, async_msg); - if (!ret) - req->flags |= REQ_F_NEED_CLEANUP; - return ret; + return 0; } static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) @@ -4885,13 +4869,22 @@ static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req) return io_put_kbuf(req, req->sr_msg.kbuf); } -static int io_recvmsg_prep(struct io_kiocb *req, - const struct io_uring_sqe *sqe) +static int io_recvmsg_prep_async(struct io_kiocb *req) { - struct io_async_msghdr *async_msg = req->async_data; - struct io_sr_msg *sr = &req->sr_msg; int ret; + if (!io_op_defs[req->opcode].needs_async_data) + return 0; + ret = io_recvmsg_copy_hdr(req, req->async_data); + if (!ret) + req->flags |= REQ_F_NEED_CLEANUP; + return ret; +} + +static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +{ + struct io_sr_msg *sr = &req->sr_msg; + if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) return -EINVAL; @@ -4904,13 +4897,7 @@ static int io_recvmsg_prep(struct io_kiocb *req, if (req->ctx->compat) sr->msg_flags |= MSG_CMSG_COMPAT; #endif - - if (!async_msg || !io_op_defs[req->opcode].needs_async_data) - return 0; - ret = io_recvmsg_copy_hdr(req, async_msg); - if (!ret) - req->flags |= REQ_F_NEED_CLEANUP; - return ret; + return 0; } static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) @@ -5063,10 +5050,17 @@ static int io_accept(struct io_kiocb *req, unsigned int issue_flags) return 0; } +static int io_connect_prep_async(struct io_kiocb *req) +{ + struct io_async_connect *io = req->async_data; + struct io_connect *conn = &req->connect; + + return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address); +} + static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_connect *conn = &req->connect; - struct io_async_connect *io = req->async_data; if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) return -EINVAL; @@ -5075,12 +5069,7 @@ static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); conn->addr_len = READ_ONCE(sqe->addr2); - - if (!io) - return 0; - - return move_addr_to_kernel(conn->addr, conn->addr_len, - &io->address); + return 0; } static int io_connect(struct io_kiocb *req, unsigned int issue_flags) @@ -6148,14 +6137,45 @@ static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return-EINVAL; } +static int io_req_prep_async(struct io_kiocb *req) +{ + switch (req->opcode) { + case IORING_OP_READV: + case IORING_OP_READ_FIXED: + case IORING_OP_READ: + return io_rw_prep_async(req, READ); + case IORING_OP_WRITEV: + case IORING_OP_WRITE_FIXED: + case IORING_OP_WRITE: + return io_rw_prep_async(req, WRITE); + case IORING_OP_SENDMSG: + case IORING_OP_SEND: + return io_sendmsg_prep_async(req); + case IORING_OP_RECVMSG: + case IORING_OP_RECV: + return io_recvmsg_prep_async(req); + case IORING_OP_CONNECT: + return io_connect_prep_async(req); + } + return 0; +} + static int io_req_defer_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { + int ret; + if (!sqe) return 0; if (io_alloc_async_data(req)) return -EAGAIN; - return io_req_prep(req, sqe); + ret = io_req_prep(req, sqe); + if (ret) + return ret; + if (req->async_data) + return io_req_prep_async(req); + return 0; + } static u32 io_get_sequence(struct io_kiocb *req) From be7053b7d028dc891857ca3e23b401a901257789 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:45 +0000 Subject: [PATCH 10/25] io_uring: do io_*_prep() early in io_submit_sqe() Now as preparations are split from async setup, we can do the first one pretty early not spilling it across multiple call sites. And after it's done SQE is not needed anymore and we can save on passing it deeply into the submission stack. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 59 +++++++++++++++++++++------------------------------ 1 file changed, 24 insertions(+), 35 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 987cfd8db2134b..7d54b0abbb8283 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6160,22 +6160,16 @@ static int io_req_prep_async(struct io_kiocb *req) return 0; } -static int io_req_defer_prep(struct io_kiocb *req, - const struct io_uring_sqe *sqe) +static int io_req_defer_prep(struct io_kiocb *req) { - int ret; - - if (!sqe) + if (!io_op_defs[req->opcode].needs_async_data) return 0; - if (io_alloc_async_data(req)) - return -EAGAIN; - ret = io_req_prep(req, sqe); - if (ret) - return ret; + /* some opcodes init it during the inital prep */ if (req->async_data) - return io_req_prep_async(req); - return 0; - + return 0; + if (__io_alloc_async_data(req)) + return -EAGAIN; + return io_req_prep_async(req); } static u32 io_get_sequence(struct io_kiocb *req) @@ -6191,7 +6185,7 @@ static u32 io_get_sequence(struct io_kiocb *req) return total_submitted - nr_reqs; } -static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe) +static int io_req_defer(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx; struct io_defer_entry *de; @@ -6208,11 +6202,9 @@ static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list)) return 0; - if (!req->async_data) { - ret = io_req_defer_prep(req, sqe); - if (ret) - return ret; - } + ret = io_req_defer_prep(req); + if (ret) + return ret; io_prep_async_link(req); de = kmalloc(sizeof(*de), GFP_KERNEL); if (!de) @@ -6631,11 +6623,11 @@ static void __io_queue_sqe(struct io_kiocb *req) io_queue_linked_timeout(linked_timeout); } -static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe) +static void io_queue_sqe(struct io_kiocb *req) { int ret; - ret = io_req_defer(req, sqe); + ret = io_req_defer(req); if (ret) { if (ret != -EIOCBQUEUED) { fail_req: @@ -6644,18 +6636,11 @@ static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe) io_req_complete(req, ret); } } else if (req->flags & REQ_F_FORCE_ASYNC) { - if (!req->async_data) { - ret = io_req_defer_prep(req, sqe); - if (unlikely(ret)) - goto fail_req; - } + ret = io_req_defer_prep(req); + if (unlikely(ret)) + goto fail_req; io_queue_async_work(req); } else { - if (sqe) { - ret = io_req_prep(req, sqe); - if (unlikely(ret)) - goto fail_req; - } __io_queue_sqe(req); } } @@ -6666,7 +6651,7 @@ static inline void io_queue_link_head(struct io_kiocb *req) io_put_req(req); io_req_complete(req, -ECANCELED); } else - io_queue_sqe(req, NULL); + io_queue_sqe(req); } /* @@ -6788,7 +6773,11 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, link->head->flags |= REQ_F_FAIL_LINK; return ret; } + ret = io_req_prep(req, sqe); + if (unlikely(ret)) + goto fail_req; + /* don't need @sqe from now on */ trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data, true, ctx->flags & IORING_SETUP_SQPOLL); @@ -6813,7 +6802,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, head->flags |= REQ_F_IO_DRAIN; ctx->drain_next = 1; } - ret = io_req_defer_prep(req, sqe); + ret = io_req_defer_prep(req); if (unlikely(ret)) goto fail_req; trace_io_uring_link(ctx, req, head); @@ -6831,13 +6820,13 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, ctx->drain_next = 0; } if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) { - ret = io_req_defer_prep(req, sqe); + ret = io_req_defer_prep(req); if (unlikely(ret)) req->flags |= REQ_F_FAIL_LINK; link->head = req; link->last = req; } else { - io_queue_sqe(req, sqe); + io_queue_sqe(req); } } From 1ee43ba8d267b5e6729c45b8756263f69c2978cc Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:46 +0000 Subject: [PATCH 11/25] io_uring: don't do async setup for links' heads Now, as we can do async setup without holding an SQE, we can skip doing io_req_defer_prep() for link heads, it will be tried to be executed inline and follows all the rules of the non-linked requests. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 7d54b0abbb8283..45f78fd25ce29f 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6820,9 +6820,6 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, ctx->drain_next = 0; } if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) { - ret = io_req_defer_prep(req); - if (unlikely(ret)) - req->flags |= REQ_F_FAIL_LINK; link->head = req; link->last = req; } else { From de59bc104c24f2e8637464a9e3ebbd8fd4c0f115 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:47 +0000 Subject: [PATCH 12/25] io_uring: fail links more in io_submit_sqe() Instead of marking a link with REQ_F_FAIL_LINK on an error and delaying its failing to the caller, do it eagerly right when after getting an error in io_submit_sqe(). This renders FAIL_LINK checks in io_queue_link_head() useless and we can skip it. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 45f78fd25ce29f..2fdfe5fa00b0c4 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6645,15 +6645,6 @@ static void io_queue_sqe(struct io_kiocb *req) } } -static inline void io_queue_link_head(struct io_kiocb *req) -{ - if (unlikely(req->flags & REQ_F_FAIL_LINK)) { - io_put_req(req); - io_req_complete(req, -ECANCELED); - } else - io_queue_sqe(req); -} - /* * Check SQE restrictions (opcode and flags). * @@ -6768,9 +6759,13 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, fail_req: io_put_req(req); io_req_complete(req, ret); - /* fail even hard links since we don't submit */ - if (link->head) + if (link->head) { + /* fail even hard links since we don't submit */ link->head->flags |= REQ_F_FAIL_LINK; + io_put_req(link->head); + io_req_complete(link->head, -ECANCELED); + link->head = NULL; + } return ret; } ret = io_req_prep(req, sqe); @@ -6811,7 +6806,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, /* last request of a link, enqueue the link */ if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) { - io_queue_link_head(head); + io_queue_sqe(head); link->head = NULL; } } else { @@ -6837,7 +6832,7 @@ static void io_submit_state_end(struct io_submit_state *state, struct io_ring_ctx *ctx) { if (state->link.head) - io_queue_link_head(state->link.head); + io_queue_sqe(state->link.head); if (state->comp.nr) io_submit_flush_completions(&state->comp, ctx); if (state->plug_started) From 792bb6eb862333658bf1bd2260133f0507e2da8d Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 22:32:51 +0000 Subject: [PATCH 13/25] io_uring: don't take uring_lock during iowq cancel [ 97.866748] a.out/2890 is trying to acquire lock: [ 97.867829] ffff8881046763e8 (&ctx->uring_lock){+.+.}-{3:3}, at: io_wq_submit_work+0x155/0x240 [ 97.869735] [ 97.869735] but task is already holding lock: [ 97.871033] ffff88810dfe0be8 (&ctx->uring_lock){+.+.}-{3:3}, at: __x64_sys_io_uring_enter+0x3f0/0x5b0 [ 97.873074] [ 97.873074] other info that might help us debug this: [ 97.874520] Possible unsafe locking scenario: [ 97.874520] [ 97.875845] CPU0 [ 97.876440] ---- [ 97.877048] lock(&ctx->uring_lock); [ 97.877961] lock(&ctx->uring_lock); [ 97.878881] [ 97.878881] *** DEADLOCK *** [ 97.878881] [ 97.880341] May be due to missing lock nesting notation [ 97.880341] [ 97.881952] 1 lock held by a.out/2890: [ 97.882873] #0: ffff88810dfe0be8 (&ctx->uring_lock){+.+.}-{3:3}, at: __x64_sys_io_uring_enter+0x3f0/0x5b0 [ 97.885108] [ 97.885108] stack backtrace: [ 97.890457] Call Trace: [ 97.891121] dump_stack+0xac/0xe3 [ 97.891972] __lock_acquire+0xab6/0x13a0 [ 97.892940] lock_acquire+0x2c3/0x390 [ 97.894894] __mutex_lock+0xae/0x9f0 [ 97.901101] io_wq_submit_work+0x155/0x240 [ 97.902112] io_wq_cancel_cb+0x162/0x490 [ 97.904126] io_async_find_and_cancel+0x3b/0x140 [ 97.905247] io_issue_sqe+0x86d/0x13e0 [ 97.909122] __io_queue_sqe+0x10b/0x550 [ 97.913971] io_queue_sqe+0x235/0x470 [ 97.914894] io_submit_sqes+0xcce/0xf10 [ 97.917872] __x64_sys_io_uring_enter+0x3fb/0x5b0 [ 97.921424] do_syscall_64+0x2d/0x40 [ 97.922329] entry_SYSCALL_64_after_hwframe+0x44/0xa9 While holding uring_lock, e.g. from inline execution, async cancel request may attempt cancellations through io_wq_submit_work, which may try to grab a lock. Delay it to task_work, so we do it from a clean context and don't have to worry about locking. Cc: # 5.5+ Fixes: c07e6719511e ("io_uring: hold uring_lock while completing failed polled io in io_wq_submit_work()") Reported-by: Abaci Reported-by: Hao Xu Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 2fdfe5fa00b0c4..8dab07f42b34b8 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2337,7 +2337,9 @@ static void io_req_task_cancel(struct callback_head *cb) struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work); struct io_ring_ctx *ctx = req->ctx; + mutex_lock(&ctx->uring_lock); __io_req_task_cancel(req, -ECANCELED); + mutex_unlock(&ctx->uring_lock); percpu_ref_put(&ctx->refs); } @@ -6426,8 +6428,13 @@ static void io_wq_submit_work(struct io_wq_work *work) if (timeout) io_queue_linked_timeout(timeout); - if (work->flags & IO_WQ_WORK_CANCEL) - ret = -ECANCELED; + if (work->flags & IO_WQ_WORK_CANCEL) { + /* io-wq is going to take down one */ + refcount_inc(&req->refs); + percpu_ref_get(&req->ctx->refs); + io_req_task_work_add_fallback(req, io_req_task_cancel); + return; + } if (!ret) { do { From a3df769899c0bdc224c94d1d8cc9cbb3f3a72553 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 22:32:52 +0000 Subject: [PATCH 14/25] io_uring: fail io-wq submission from a task_work In case of failure io_wq_submit_work() needs to post an CQE and so potentially take uring_lock. The safest way to deal with it is to do that from under task_work where we can safely take the lock. Also, as io_iopoll_check() holds the lock tight and releases it reluctantly, it will play nicer in the furuter with notifying an iopolling task about new such pending failed requests. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 48 ++++++++++++++++++------------------------------ 1 file changed, 18 insertions(+), 30 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 8dab07f42b34b8..582306b1dfd114 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2338,7 +2338,7 @@ static void io_req_task_cancel(struct callback_head *cb) struct io_ring_ctx *ctx = req->ctx; mutex_lock(&ctx->uring_lock); - __io_req_task_cancel(req, -ECANCELED); + __io_req_task_cancel(req, req->result); mutex_unlock(&ctx->uring_lock); percpu_ref_put(&ctx->refs); } @@ -2371,11 +2371,22 @@ static void io_req_task_queue(struct io_kiocb *req) req->task_work.func = io_req_task_submit; ret = io_req_task_work_add(req); if (unlikely(ret)) { + req->result = -ECANCELED; percpu_ref_get(&req->ctx->refs); io_req_task_work_add_fallback(req, io_req_task_cancel); } } +static void io_req_task_queue_fail(struct io_kiocb *req, int ret) +{ + percpu_ref_get(&req->ctx->refs); + req->result = ret; + req->task_work.func = io_req_task_cancel; + + if (unlikely(io_req_task_work_add(req))) + io_req_task_work_add_fallback(req, io_req_task_cancel); +} + static inline void io_queue_next(struct io_kiocb *req) { struct io_kiocb *nxt = io_req_find_next(req); @@ -6428,13 +6439,8 @@ static void io_wq_submit_work(struct io_wq_work *work) if (timeout) io_queue_linked_timeout(timeout); - if (work->flags & IO_WQ_WORK_CANCEL) { - /* io-wq is going to take down one */ - refcount_inc(&req->refs); - percpu_ref_get(&req->ctx->refs); - io_req_task_work_add_fallback(req, io_req_task_cancel); - return; - } + if (work->flags & IO_WQ_WORK_CANCEL) + ret = -ECANCELED; if (!ret) { do { @@ -6450,29 +6456,11 @@ static void io_wq_submit_work(struct io_wq_work *work) } while (1); } + /* avoid locking problems by failing it from a clean context */ if (ret) { - struct io_ring_ctx *lock_ctx = NULL; - - if (req->ctx->flags & IORING_SETUP_IOPOLL) - lock_ctx = req->ctx; - - /* - * io_iopoll_complete() does not hold completion_lock to - * complete polled io, so here for polled io, we can not call - * io_req_complete() directly, otherwise there maybe concurrent - * access to cqring, defer_list, etc, which is not safe. Given - * that io_iopoll_complete() is always called under uring_lock, - * so here for polled io, we also get uring_lock to complete - * it. - */ - if (lock_ctx) - mutex_lock(&lock_ctx->uring_lock); - - req_set_fail_links(req); - io_req_complete(req, ret); - - if (lock_ctx) - mutex_unlock(&lock_ctx->uring_lock); + /* io-wq is going to take one down */ + refcount_inc(&req->refs); + io_req_task_queue_fail(req, ret); } } From 8bad28d8a305b0e5ae444c8c3051e8744f5a4296 Mon Sep 17 00:00:00 2001 From: Hao Xu Date: Fri, 19 Feb 2021 17:19:36 +0800 Subject: [PATCH 15/25] io_uring: don't hold uring_lock when calling io_run_task_work* Abaci reported the below issue: [ 141.400455] hrtimer: interrupt took 205853 ns [ 189.869316] process 'usr/local/ilogtail/ilogtail_0.16.26' started with executable stack [ 250.188042] [ 250.188327] ============================================ [ 250.189015] WARNING: possible recursive locking detected [ 250.189732] 5.11.0-rc4 #1 Not tainted [ 250.190267] -------------------------------------------- [ 250.190917] a.out/7363 is trying to acquire lock: [ 250.191506] ffff888114dbcbe8 (&ctx->uring_lock){+.+.}-{3:3}, at: __io_req_task_submit+0x29/0xa0 [ 250.192599] [ 250.192599] but task is already holding lock: [ 250.193309] ffff888114dbfbe8 (&ctx->uring_lock){+.+.}-{3:3}, at: __x64_sys_io_uring_register+0xad/0x210 [ 250.194426] [ 250.194426] other info that might help us debug this: [ 250.195238] Possible unsafe locking scenario: [ 250.195238] [ 250.196019] CPU0 [ 250.196411] ---- [ 250.196803] lock(&ctx->uring_lock); [ 250.197420] lock(&ctx->uring_lock); [ 250.197966] [ 250.197966] *** DEADLOCK *** [ 250.197966] [ 250.198837] May be due to missing lock nesting notation [ 250.198837] [ 250.199780] 1 lock held by a.out/7363: [ 250.200373] #0: ffff888114dbfbe8 (&ctx->uring_lock){+.+.}-{3:3}, at: __x64_sys_io_uring_register+0xad/0x210 [ 250.201645] [ 250.201645] stack backtrace: [ 250.202298] CPU: 0 PID: 7363 Comm: a.out Not tainted 5.11.0-rc4 #1 [ 250.203144] Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011 [ 250.203887] Call Trace: [ 250.204302] dump_stack+0xac/0xe3 [ 250.204804] __lock_acquire+0xab6/0x13a0 [ 250.205392] lock_acquire+0x2c3/0x390 [ 250.205928] ? __io_req_task_submit+0x29/0xa0 [ 250.206541] __mutex_lock+0xae/0x9f0 [ 250.207071] ? __io_req_task_submit+0x29/0xa0 [ 250.207745] ? 0xffffffffa0006083 [ 250.208248] ? __io_req_task_submit+0x29/0xa0 [ 250.208845] ? __io_req_task_submit+0x29/0xa0 [ 250.209452] ? __io_req_task_submit+0x5/0xa0 [ 250.210083] __io_req_task_submit+0x29/0xa0 [ 250.210687] io_async_task_func+0x23d/0x4c0 [ 250.211278] task_work_run+0x89/0xd0 [ 250.211884] io_run_task_work_sig+0x50/0xc0 [ 250.212464] io_sqe_files_unregister+0xb2/0x1f0 [ 250.213109] __io_uring_register+0x115a/0x1750 [ 250.213718] ? __x64_sys_io_uring_register+0xad/0x210 [ 250.214395] ? __fget_files+0x15a/0x260 [ 250.214956] __x64_sys_io_uring_register+0xbe/0x210 [ 250.215620] ? trace_hardirqs_on+0x46/0x110 [ 250.216205] do_syscall_64+0x2d/0x40 [ 250.216731] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [ 250.217455] RIP: 0033:0x7f0fa17e5239 [ 250.218034] Code: 01 00 48 81 c4 80 00 00 00 e9 f1 fe ff ff 0f 1f 00 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 3d 01 f0 ff ff 73 01 c3 48 8b 0d 27 ec 2c 00 f7 d8 64 89 01 48 [ 250.220343] RSP: 002b:00007f0fa1eeac48 EFLAGS: 00000246 ORIG_RAX: 00000000000001ab [ 250.221360] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f0fa17e5239 [ 250.222272] RDX: 0000000000000000 RSI: 0000000000000003 RDI: 0000000000000008 [ 250.223185] RBP: 00007f0fa1eeae20 R08: 0000000000000000 R09: 0000000000000000 [ 250.224091] R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000000 [ 250.224999] R13: 0000000000021000 R14: 0000000000000000 R15: 00007f0fa1eeb700 This is caused by calling io_run_task_work_sig() to do work under uring_lock while the caller io_sqe_files_unregister() already held uring_lock. To fix this issue, briefly drop uring_lock when calling io_run_task_work_sig(), and there are two things to concern: - hold uring_lock in io_ring_ctx_free() around io_sqe_files_unregister() this is for consistency of lock/unlock. - add new fixed rsrc ref node before dropping uring_lock it's not safe to do io_uring_enter-->percpu_ref_get() with a dying one. - check if rsrc_data->refs is dying to avoid parallel io_sqe_files_unregister Reported-by: Abaci Fixes: 1ffc54220c44 ("io_uring: fix io_sqe_files_unregister() hangs") Suggested-by: Pavel Begunkov Signed-off-by: Hao Xu [axboe: fixes from Pavel folded in] Signed-off-by: Jens Axboe --- fs/io_uring.c | 61 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 582306b1dfd114..7956c6751a67d4 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -236,6 +236,7 @@ struct fixed_rsrc_data { struct fixed_rsrc_ref_node *node; struct percpu_ref refs; struct completion done; + bool quiesce; }; struct io_buffer { @@ -7316,38 +7317,57 @@ static void io_sqe_rsrc_set_node(struct io_ring_ctx *ctx, percpu_ref_get(&rsrc_data->refs); } -static int io_rsrc_ref_quiesce(struct fixed_rsrc_data *data, - struct io_ring_ctx *ctx, - struct fixed_rsrc_ref_node *backup_node) +static void io_sqe_rsrc_kill_node(struct io_ring_ctx *ctx, struct fixed_rsrc_data *data) { - struct fixed_rsrc_ref_node *ref_node; - int ret; + struct fixed_rsrc_ref_node *ref_node = NULL; io_rsrc_ref_lock(ctx); ref_node = data->node; io_rsrc_ref_unlock(ctx); if (ref_node) percpu_ref_kill(&ref_node->refs); +} + +static int io_rsrc_ref_quiesce(struct fixed_rsrc_data *data, + struct io_ring_ctx *ctx, + struct fixed_rsrc_ref_node *backup_node) +{ + int ret; - percpu_ref_kill(&data->refs); + if (data->quiesce) + return -ENXIO; - /* wait for all refs nodes to complete */ - flush_delayed_work(&ctx->rsrc_put_work); + data->quiesce = true; do { + io_sqe_rsrc_kill_node(ctx, data); + percpu_ref_kill(&data->refs); + flush_delayed_work(&ctx->rsrc_put_work); + ret = wait_for_completion_interruptible(&data->done); if (!ret) break; + + percpu_ref_resurrect(&data->refs); + io_sqe_rsrc_set_node(ctx, data, backup_node); + backup_node = NULL; + reinit_completion(&data->done); + mutex_unlock(&ctx->uring_lock); ret = io_run_task_work_sig(); - if (ret < 0) { - percpu_ref_resurrect(&data->refs); - reinit_completion(&data->done); - io_sqe_rsrc_set_node(ctx, data, backup_node); - return ret; - } + mutex_lock(&ctx->uring_lock); + + if (ret < 0) + break; + backup_node = alloc_fixed_rsrc_ref_node(ctx); + ret = -ENOMEM; + if (!backup_node) + break; + init_fixed_file_ref_node(ctx, backup_node); } while (1); + data->quiesce = false; - destroy_fixed_rsrc_ref_node(backup_node); - return 0; + if (backup_node) + destroy_fixed_rsrc_ref_node(backup_node); + return ret; } static struct fixed_rsrc_data *alloc_fixed_rsrc_data(struct io_ring_ctx *ctx) @@ -7382,7 +7402,12 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) unsigned nr_tables, i; int ret; - if (!data) + /* + * percpu_ref_is_dying() is to stop parallel files unregister + * Since we possibly drop uring lock later in this function to + * run task work. + */ + if (!data || percpu_ref_is_dying(&data->refs)) return -ENXIO; backup_node = alloc_fixed_rsrc_ref_node(ctx); if (!backup_node) @@ -8731,7 +8756,9 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) css_put(ctx->sqo_blkcg_css); #endif + mutex_lock(&ctx->uring_lock); io_sqe_files_unregister(ctx); + mutex_unlock(&ctx->uring_lock); io_eventfd_unregister(ctx); io_destroy_buffers(ctx); idr_destroy(&ctx->personality_idr); From 99a10081647168022745859bb2f1c28b2f70dc83 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 19 Feb 2021 09:35:19 -0700 Subject: [PATCH 16/25] io_uring: make the !CONFIG_NET helpers a bit more robust With the prep and prep async split, we now have potentially 3 helpers that need to be defined for !CONFIG_NET. Add some helpers to do just that. Fixes the following compile error on !CONFIG_NET: fs/io_uring.c:6171:10: error: implicit declaration of function 'io_sendmsg_prep_async'; did you mean 'io_req_prep_async'? [-Werror=implicit-function-declaration] return io_sendmsg_prep_async(req); ^~~~~~~~~~~~~~~~~~~~~ io_req_prep_async Fixes: 93642ef88434 ("io_uring: split sqe-prep and async setup") Reported-by: Naresh Kamboju Signed-off-by: Jens Axboe --- fs/io_uring.c | 76 ++++++++++++++++++--------------------------------- 1 file changed, 26 insertions(+), 50 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 7956c6751a67d4..cef80106b30592 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5128,56 +5128,32 @@ static int io_connect(struct io_kiocb *req, unsigned int issue_flags) return 0; } #else /* !CONFIG_NET */ -static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -{ - return -EOPNOTSUPP; -} - -static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) -{ - return -EOPNOTSUPP; -} - -static int io_send(struct io_kiocb *req, unsigned int issue_flags) -{ - return -EOPNOTSUPP; -} - -static int io_recvmsg_prep(struct io_kiocb *req, - const struct io_uring_sqe *sqe) -{ - return -EOPNOTSUPP; -} - -static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) -{ - return -EOPNOTSUPP; -} - -static int io_recv(struct io_kiocb *req, unsigned int issue_flags) -{ - return -EOPNOTSUPP; -} - -static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -{ - return -EOPNOTSUPP; -} - -static int io_accept(struct io_kiocb *req, unsigned int issue_flags) -{ - return -EOPNOTSUPP; -} - -static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -{ - return -EOPNOTSUPP; -} - -static int io_connect(struct io_kiocb *req, unsigned int issue_flags) -{ - return -EOPNOTSUPP; -} +#define IO_NETOP_FN(op) \ +static int io_##op(struct io_kiocb *req, unsigned int issue_flags) \ +{ \ + return -EOPNOTSUPP; \ +} + +#define IO_NETOP_PREP(op) \ +IO_NETOP_FN(op) \ +static int io_##op##_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) \ +{ \ + return -EOPNOTSUPP; \ +} \ + +#define IO_NETOP_PREP_ASYNC(op) \ +IO_NETOP_PREP(op) \ +static int io_##op##_prep_async(struct io_kiocb *req) \ +{ \ + return -EOPNOTSUPP; \ +} + +IO_NETOP_PREP_ASYNC(sendmsg); +IO_NETOP_PREP_ASYNC(recvmsg); +IO_NETOP_PREP_ASYNC(connect); +IO_NETOP_PREP(accept); +IO_NETOP_FN(send); +IO_NETOP_FN(recv); #endif /* CONFIG_NET */ struct io_poll_table { From e6cb007c45dedada0a847eaa486c49509d63b1e8 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 20 Feb 2021 18:03:47 +0000 Subject: [PATCH 17/25] io_uring: zero ref_node after killing it After a rsrc/files reference node's refs are killed, it must never be used. And that's how it works, it either assigns a new node or kills the whole data table. Let's explicitly NULL it, that shouldn't be necessary, but if something would go wrong I'd rather catch a NULL dereference to using a dangling pointer. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/io_uring.c b/fs/io_uring.c index cef80106b30592..5215d32c4f8c2f 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7299,6 +7299,7 @@ static void io_sqe_rsrc_kill_node(struct io_ring_ctx *ctx, struct fixed_rsrc_dat io_rsrc_ref_lock(ctx); ref_node = data->node; + data->node = NULL; io_rsrc_ref_unlock(ctx); if (ref_node) percpu_ref_kill(&ref_node->refs); From f2303b1f8244d88ffca28d3be6166ce4835cc27a Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 20 Feb 2021 18:03:49 +0000 Subject: [PATCH 18/25] io_uring: keep generic rsrc infra generic io_rsrc_ref_quiesce() is a generic resource function, though now it was wired to allocate and initialise ref nodes with file-specific callbacks/etc. Keep it sane by passing in as a parameters everything we need for initialisations, otherwise it will hurt us badly one day. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 5215d32c4f8c2f..c98b673f0bb1e8 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1037,8 +1037,7 @@ static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, static void destroy_fixed_rsrc_ref_node(struct fixed_rsrc_ref_node *ref_node); static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node( struct io_ring_ctx *ctx); -static void init_fixed_file_ref_node(struct io_ring_ctx *ctx, - struct fixed_rsrc_ref_node *ref_node); +static void io_ring_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc); static bool io_rw_reissue(struct io_kiocb *req); static void io_cqring_fill_event(struct io_kiocb *req, long res); @@ -7307,8 +7306,10 @@ static void io_sqe_rsrc_kill_node(struct io_ring_ctx *ctx, struct fixed_rsrc_dat static int io_rsrc_ref_quiesce(struct fixed_rsrc_data *data, struct io_ring_ctx *ctx, - struct fixed_rsrc_ref_node *backup_node) + void (*rsrc_put)(struct io_ring_ctx *ctx, + struct io_rsrc_put *prsrc)) { + struct fixed_rsrc_ref_node *backup_node; int ret; if (data->quiesce) @@ -7316,6 +7317,13 @@ static int io_rsrc_ref_quiesce(struct fixed_rsrc_data *data, data->quiesce = true; do { + ret = -ENOMEM; + backup_node = alloc_fixed_rsrc_ref_node(ctx); + if (!backup_node) + break; + backup_node->rsrc_data = data; + backup_node->rsrc_put = rsrc_put; + io_sqe_rsrc_kill_node(ctx, data); percpu_ref_kill(&data->refs); flush_delayed_work(&ctx->rsrc_put_work); @@ -7331,15 +7339,7 @@ static int io_rsrc_ref_quiesce(struct fixed_rsrc_data *data, mutex_unlock(&ctx->uring_lock); ret = io_run_task_work_sig(); mutex_lock(&ctx->uring_lock); - - if (ret < 0) - break; - backup_node = alloc_fixed_rsrc_ref_node(ctx); - ret = -ENOMEM; - if (!backup_node) - break; - init_fixed_file_ref_node(ctx, backup_node); - } while (1); + } while (ret >= 0); data->quiesce = false; if (backup_node) @@ -7375,7 +7375,6 @@ static void free_fixed_rsrc_data(struct fixed_rsrc_data *data) static int io_sqe_files_unregister(struct io_ring_ctx *ctx) { struct fixed_rsrc_data *data = ctx->file_data; - struct fixed_rsrc_ref_node *backup_node; unsigned nr_tables, i; int ret; @@ -7386,12 +7385,7 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) */ if (!data || percpu_ref_is_dying(&data->refs)) return -ENXIO; - backup_node = alloc_fixed_rsrc_ref_node(ctx); - if (!backup_node) - return -ENOMEM; - init_fixed_file_ref_node(ctx, backup_node); - - ret = io_rsrc_ref_quiesce(data, ctx, backup_node); + ret = io_rsrc_ref_quiesce(data, ctx, io_ring_file_put); if (ret) return ret; From 88f171ab7798a1ed0b9e39867ee16f307466e870 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 20 Feb 2021 18:03:50 +0000 Subject: [PATCH 19/25] io_uring: wait potential ->release() on resurrect There is a short window where percpu_refs are already turned zero, but we try to do resurrect(). Play nicer and wait for ->release() to happen in this case and proceed as everything is ok. One downside for ctx refs is that we can ignore signal_pending() on a rare occasion, but someone else should check for it later if needed. Cc: # 5.5+ Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index c98b673f0bb1e8..5cc02226bb38e0 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1104,6 +1104,21 @@ static inline void io_set_resource_node(struct io_kiocb *req) } } +static bool io_refs_resurrect(struct percpu_ref *ref, struct completion *compl) +{ + if (!percpu_ref_tryget(ref)) { + /* already at zero, wait for ->release() */ + if (!try_wait_for_completion(compl)) + synchronize_rcu(); + return false; + } + + percpu_ref_resurrect(ref); + reinit_completion(compl); + percpu_ref_put(ref); + return true; +} + static bool io_match_task(struct io_kiocb *head, struct task_struct *task, struct files_struct *files) @@ -7329,13 +7344,11 @@ static int io_rsrc_ref_quiesce(struct fixed_rsrc_data *data, flush_delayed_work(&ctx->rsrc_put_work); ret = wait_for_completion_interruptible(&data->done); - if (!ret) + if (!ret || !io_refs_resurrect(&data->refs, &data->done)) break; - percpu_ref_resurrect(&data->refs); io_sqe_rsrc_set_node(ctx, data, backup_node); backup_node = NULL; - reinit_completion(&data->done); mutex_unlock(&ctx->uring_lock); ret = io_run_task_work_sig(); mutex_lock(&ctx->uring_lock); @@ -10070,10 +10083,8 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, mutex_lock(&ctx->uring_lock); - if (ret) { - percpu_ref_resurrect(&ctx->refs); - goto out_quiesce; - } + if (ret && io_refs_resurrect(&ctx->refs, &ctx->ref_comp)) + return ret; } if (ctx->restricted) { @@ -10165,7 +10176,6 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, if (io_register_op_must_quiesce(opcode)) { /* bring the ctx back to life */ percpu_ref_reinit(&ctx->refs); -out_quiesce: reinit_completion(&ctx->ref_comp); } return ret; From ebf4a5db690a47e71056381ead8a134de7202694 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 20 Feb 2021 01:39:53 +0000 Subject: [PATCH 20/25] io_uring: fix leaving invalid req->flags sqe->flags are subset of req flags, so incorrectly copied may span into in-kernel flags and wreck havoc, e.g. by setting REQ_F_INFLIGHT. Fixes: 5be9ad1e4287e ("io_uring: optimise io_init_req() flags setting") Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 5cc02226bb38e0..1501f20fde8447 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6679,8 +6679,10 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, req->result = 0; /* enforce forwards compatibility on users */ - if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) + if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) { + req->flags = 0; return -EINVAL; + } if (unlikely(req->opcode >= IORING_OP_LAST)) return -EINVAL; From b6c23dd5a483174f386e4c2e1711d9532e090c00 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 20 Feb 2021 15:17:18 +0000 Subject: [PATCH 21/25] io_uring: run task_work on io_uring_register() Do run task_work before io_uring_register(), that might make a first quiesce round much nicer. We generally do that for any syscall invocation to avoid spurious -EINTR/-ERESTARTSYS, for task_work that we generate. This patch brings io_uring_register() inline with the two other io_uring syscalls. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/io_uring.c b/fs/io_uring.c index 1501f20fde8447..3ecc3c08bf1224 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -10200,6 +10200,8 @@ SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, ctx = f.file->private_data; + io_run_task_work(); + mutex_lock(&ctx->uring_lock); ret = __io_uring_register(ctx, opcode, arg, nr_args); mutex_unlock(&ctx->uring_lock); From 8e5c66c485a8af3f39a8b0358e9e09f002016d92 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 22 Feb 2021 11:45:55 +0000 Subject: [PATCH 22/25] io_uring: clear request count when freeing caches BUG: KASAN: double-free or invalid-free in io_req_caches_free.constprop.0+0x3ce/0x530 fs/io_uring.c:8709 Workqueue: events_unbound io_ring_exit_work Call Trace: [...] __cache_free mm/slab.c:3424 [inline] kmem_cache_free_bulk+0x4b/0x1b0 mm/slab.c:3744 io_req_caches_free.constprop.0+0x3ce/0x530 fs/io_uring.c:8709 io_ring_ctx_free fs/io_uring.c:8764 [inline] io_ring_exit_work+0x518/0x6b0 fs/io_uring.c:8846 process_one_work+0x98d/0x1600 kernel/workqueue.c:2275 worker_thread+0x64c/0x1120 kernel/workqueue.c:2421 kthread+0x3b1/0x4a0 kernel/kthread.c:292 ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:294 Freed by task 11900: [...] kmem_cache_free_bulk+0x4b/0x1b0 mm/slab.c:3744 io_req_caches_free.constprop.0+0x3ce/0x530 fs/io_uring.c:8709 io_uring_flush+0x483/0x6e0 fs/io_uring.c:9237 filp_close+0xb4/0x170 fs/open.c:1286 close_files fs/file.c:403 [inline] put_files_struct fs/file.c:418 [inline] put_files_struct+0x1d0/0x350 fs/file.c:415 exit_files+0x7e/0xa0 fs/file.c:435 do_exit+0xc27/0x2ae0 kernel/exit.c:820 do_group_exit+0x125/0x310 kernel/exit.c:922 [...] io_req_caches_free() doesn't zero submit_state->free_reqs, so io_uring considers just freed requests to be good and sound and will reuse or double free them. Zero the counter. Reported-by: syzbot+30b4936dcdb3aafa4fb4@syzkaller.appspotmail.com Fixes: 41be53e94fb04 ("io_uring: kill cached requests from exiting task closing the ring") Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 3ecc3c08bf1224..bf9ad810c62190 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8704,9 +8704,11 @@ static void io_req_caches_free(struct io_ring_ctx *ctx, struct task_struct *tsk) mutex_lock(&ctx->uring_lock); - if (submit_state->free_reqs) + if (submit_state->free_reqs) { kmem_cache_free_bulk(req_cachep, submit_state->free_reqs, submit_state->reqs); + submit_state->free_reqs = 0; + } io_req_cache_free(&submit_state->comp.free_list, NULL); From 7c977a58dc83366e488c217fd88b1469d242bee5 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 23 Feb 2021 19:17:35 -0700 Subject: [PATCH 23/25] io_uring: don't attempt IO reissue from the ring exit path If we're exiting the ring, just let the IO fail with -EAGAIN as nobody will care anyway. It's not the right context to reissue from. Cc: stable@vger.kernel.org Signed-off-by: Jens Axboe --- fs/io_uring.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fs/io_uring.c b/fs/io_uring.c index bf9ad810c62190..275ad84e8227b2 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2839,6 +2839,13 @@ static bool io_rw_reissue(struct io_kiocb *req) return false; if ((req->flags & REQ_F_NOWAIT) || io_wq_current_is_worker()) return false; + /* + * If ref is dying, we might be running poll reap from the exit work. + * Don't attempt to reissue from that path, just let it fail with + * -EAGAIN. + */ + if (percpu_ref_is_dying(&req->ctx->refs)) + return false; lockdep_assert_held(&req->ctx->uring_lock); From e5547d2c5eb363bfac7632ba789ca834fa829650 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 23 Feb 2021 22:17:20 +0000 Subject: [PATCH 24/25] io_uring: fix locked_free_list caches_free() Don't forget to zero locked_free_nr, it's not a disaster but makes it attempting to flush it with extra locking when there is nothing in the list. Also, don't traverse a potentially long list freeing requests under spinlock, splice the list and do it afterwards. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 275ad84e8227b2..5c8e24274acfa8 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8708,6 +8708,7 @@ static void io_req_cache_free(struct list_head *list, struct task_struct *tsk) static void io_req_caches_free(struct io_ring_ctx *ctx, struct task_struct *tsk) { struct io_submit_state *submit_state = &ctx->submit_state; + struct io_comp_state *cs = &ctx->submit_state.comp; mutex_lock(&ctx->uring_lock); @@ -8717,12 +8718,13 @@ static void io_req_caches_free(struct io_ring_ctx *ctx, struct task_struct *tsk) submit_state->free_reqs = 0; } - io_req_cache_free(&submit_state->comp.free_list, NULL); - spin_lock_irq(&ctx->completion_lock); - io_req_cache_free(&submit_state->comp.locked_free_list, NULL); + list_splice_init(&cs->locked_free_list, &cs->free_list); + cs->locked_free_nr = 0; spin_unlock_irq(&ctx->completion_lock); + io_req_cache_free(&cs->free_list, NULL); + mutex_unlock(&ctx->uring_lock); } From cb5e1b81304e089ee3ca948db4d29f71902eb575 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 25 Feb 2021 07:37:35 -0700 Subject: [PATCH 25/25] Revert "io_uring: wait potential ->release() on resurrect" This reverts commit 88f171ab7798a1ed0b9e39867ee16f307466e870. I ran into a case where the ref resurrect now spins, so revert this change for now until we can further investigate why it's broken. The bug seems to indicate spinning on the lock itself, likely there's some ABBA deadlock involved: [<0>] __percpu_ref_switch_mode+0x45/0x180 [<0>] percpu_ref_resurrect+0x46/0x70 [<0>] io_refs_resurrect+0x25/0xa0 [<0>] __io_uring_register+0x135/0x10c0 [<0>] __x64_sys_io_uring_register+0xc2/0x1a0 [<0>] do_syscall_64+0x42/0x110 [<0>] entry_SYSCALL_64_after_hwframe+0x44/0xa9 Signed-off-by: Jens Axboe --- fs/io_uring.c | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 5c8e24274acfa8..442337b40c9d4c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1104,21 +1104,6 @@ static inline void io_set_resource_node(struct io_kiocb *req) } } -static bool io_refs_resurrect(struct percpu_ref *ref, struct completion *compl) -{ - if (!percpu_ref_tryget(ref)) { - /* already at zero, wait for ->release() */ - if (!try_wait_for_completion(compl)) - synchronize_rcu(); - return false; - } - - percpu_ref_resurrect(ref); - reinit_completion(compl); - percpu_ref_put(ref); - return true; -} - static bool io_match_task(struct io_kiocb *head, struct task_struct *task, struct files_struct *files) @@ -7353,11 +7338,13 @@ static int io_rsrc_ref_quiesce(struct fixed_rsrc_data *data, flush_delayed_work(&ctx->rsrc_put_work); ret = wait_for_completion_interruptible(&data->done); - if (!ret || !io_refs_resurrect(&data->refs, &data->done)) + if (!ret) break; + percpu_ref_resurrect(&data->refs); io_sqe_rsrc_set_node(ctx, data, backup_node); backup_node = NULL; + reinit_completion(&data->done); mutex_unlock(&ctx->uring_lock); ret = io_run_task_work_sig(); mutex_lock(&ctx->uring_lock); @@ -10096,8 +10083,10 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, mutex_lock(&ctx->uring_lock); - if (ret && io_refs_resurrect(&ctx->refs, &ctx->ref_comp)) - return ret; + if (ret) { + percpu_ref_resurrect(&ctx->refs); + goto out_quiesce; + } } if (ctx->restricted) { @@ -10189,6 +10178,7 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, if (io_register_op_must_quiesce(opcode)) { /* bring the ctx back to life */ percpu_ref_reinit(&ctx->refs); +out_quiesce: reinit_completion(&ctx->ref_comp); } return ret;