io_uring/net: fix notif cqe reordering

send zc is not restricted to !IO_URING_F_UNLOCKED anymore and so we can't use task-tw ordering trick to order notification cqes with requests completions. In this case leave it alone and let io_send_zc_cleanup() flush it. Cc: [email protected] Fixes: 53bdc88 ("io_uring/notif: order notif vs send CQEs") Signed-off-by: Pavel Begunkov <[email protected]> Link: https://lore.kernel.org/r/0031f3a00d492e814a4a0935a2029a46d9c9ba06.1664486545.git.asml.silence@gmail.com Signed-off-by: Jens Axboe <[email protected]>
niw · Sep 29, 2022 · 108893d · 108893d
1 parent 6f10ae8
commit 108893d
Showing 1 changed file with 19 additions and 5 deletions.
diff --git a/io_uring/net.c b/io_uring/net.c
@@ -1127,8 +1127,14 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags)
 	else if (zc->done_io)
 		ret = zc->done_io;
 
-	io_notif_flush(zc->notif);
-	req->flags &= ~REQ_F_NEED_CLEANUP;
+	/*
+	 * If we're in io-wq we can't rely on tw ordering guarantees, defer
+	 * flushing notif to io_send_zc_cleanup()
+	 */
+	if (!(issue_flags & IO_URING_F_UNLOCKED)) {
+		io_notif_flush(zc->notif);
+		req->flags &= ~REQ_F_NEED_CLEANUP;
+	}
 	io_req_set_res(req, ret, IORING_CQE_F_MORE);
 	return IOU_OK;
 }
@@ -1182,17 +1188,25 @@ int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags)
 		req_set_fail(req);
 	}
 	/* fast path, check for non-NULL to avoid function call */
-	if (kmsg->free_iov)
+	if (kmsg->free_iov) {
 		kfree(kmsg->free_iov);
+		kmsg->free_iov = NULL;
+	}
 
 	io_netmsg_recycle(req, issue_flags);
 	if (ret >= 0)
 		ret += sr->done_io;
 	else if (sr->done_io)
 		ret = sr->done_io;
 
-	io_notif_flush(sr->notif);
-	req->flags &= ~REQ_F_NEED_CLEANUP;
+	/*
+	 * If we're in io-wq we can't rely on tw ordering guarantees, defer
+	 * flushing notif to io_send_zc_cleanup()
+	 */
+	if (!(issue_flags & IO_URING_F_UNLOCKED)) {
+		io_notif_flush(sr->notif);
+		req->flags &= ~REQ_F_NEED_CLEANUP;
+	}
 	io_req_set_res(req, ret, IORING_CQE_F_MORE);
 	return IOU_OK;
 }