summaryrefslogtreecommitdiff
path: root/io_uring
diff options
context:
space:
mode:
authorJens Axboe <axboe@kernel.dk>2024-02-14 12:59:36 -0700
committerJens Axboe <axboe@kernel.dk>2024-02-14 13:01:25 -0700
commit428f13826855e3eea44bf13cedbf33f382ef8794 (patch)
treef6d779f7ed0c23be2cf59ff90387b68f204be8c7 /io_uring
parent3fb1764c6b57808ddab7fe7c242fa04c2479ef0a (diff)
io_uring/napi: ensure napi polling is aborted when work is available
While testing io_uring NAPI with DEFER_TASKRUN, I ran into slowdowns and stalls in packet delivery. Turns out that while io_napi_busy_loop_should_end() aborts appropriately on regular task_work, it does not abort if we have local task_work pending. Move io_has_work() into the private io_uring.h header, and gate whether we should continue polling on that as well. This makes NAPI polling on send/receive work as designed with IORING_SETUP_DEFER_TASKRUN as well. Fixes: 8d0c12a80cde ("io-uring: add napi busy poll support") Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'io_uring')
-rw-r--r--io_uring/io_uring.c11
-rw-r--r--io_uring/io_uring.h11
-rw-r--r--io_uring/napi.c2
3 files changed, 12 insertions, 12 deletions
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index b25c2217c322..844a7524ed91 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -122,11 +122,6 @@
#define IO_COMPL_BATCH 32
#define IO_REQ_ALLOC_BATCH 8
-enum {
- IO_CHECK_CQ_OVERFLOW_BIT,
- IO_CHECK_CQ_DROPPED_BIT,
-};
-
struct io_defer_entry {
struct list_head list;
struct io_kiocb *req;
@@ -2479,12 +2474,6 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
return ret;
}
-static inline bool io_has_work(struct io_ring_ctx *ctx)
-{
- return test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq) ||
- !llist_empty(&ctx->work_llist);
-}
-
static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
int wake_flags, void *key)
{
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 1ca99522811b..6426ee382276 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -448,4 +448,15 @@ static inline bool io_file_can_poll(struct io_kiocb *req)
}
return false;
}
+
+enum {
+ IO_CHECK_CQ_OVERFLOW_BIT,
+ IO_CHECK_CQ_DROPPED_BIT,
+};
+
+static inline bool io_has_work(struct io_ring_ctx *ctx)
+{
+ return test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq) ||
+ !llist_empty(&ctx->work_llist);
+}
#endif
diff --git a/io_uring/napi.c b/io_uring/napi.c
index 8ec016899539..b234adda7dfd 100644
--- a/io_uring/napi.c
+++ b/io_uring/napi.c
@@ -122,7 +122,7 @@ static bool io_napi_busy_loop_should_end(void *data,
if (signal_pending(current))
return true;
- if (io_should_wake(iowq))
+ if (io_should_wake(iowq) || io_has_work(iowq->ctx))
return true;
if (io_napi_busy_loop_timeout(start_time, iowq->napi_busy_poll_to))
return true;