Skip to content

Commit b4bc35c

Browse files
isilenceaxboe
authored andcommitted
io_uring: combine cq_wait_nr checks
Instead of explicitly checking ->cq_wait_nr for whether there are waiting, which is currently represented by 0, we can store there a large value and the nr_tw will automatically filter out those cases. Add a named constant for that and for the wake up bias value. Signed-off-by: Pavel Begunkov <asml.silence@gmail.com> Link: https://lore.kernel.org/r/38def30282654d980673976cd42fde9bab19b297.1705438669.git.asml.silence@gmail.com Signed-off-by: Jens Axboe <axboe@kernel.dk>
1 parent e8c4077 commit b4bc35c

File tree

1 file changed

+27
-7
lines changed

1 file changed

+27
-7
lines changed

io_uring/io_uring.c

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,14 @@ struct io_defer_entry {
137137
#define IO_DISARM_MASK (REQ_F_ARM_LTIMEOUT | REQ_F_LINK_TIMEOUT | REQ_F_FAIL)
138138
#define IO_REQ_LINK_FLAGS (REQ_F_LINK | REQ_F_HARDLINK)
139139

140+
/*
141+
* No waiters. It's larger than any valid value of the tw counter
142+
* so that tests against ->cq_wait_nr would fail and skip wake_up().
143+
*/
144+
#define IO_CQ_WAKE_INIT (-1U)
145+
/* Forced wake up if there is a waiter regardless of ->cq_wait_nr */
146+
#define IO_CQ_WAKE_FORCE (IO_CQ_WAKE_INIT >> 1)
147+
140148
static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
141149
struct task_struct *task,
142150
bool cancel_all);
@@ -303,6 +311,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
303311
goto err;
304312

305313
ctx->flags = p->flags;
314+
atomic_set(&ctx->cq_wait_nr, IO_CQ_WAKE_INIT);
306315
init_waitqueue_head(&ctx->sqo_sq_wait);
307316
INIT_LIST_HEAD(&ctx->sqd_list);
308317
INIT_LIST_HEAD(&ctx->cq_overflow_list);
@@ -1306,6 +1315,13 @@ static inline void io_req_local_work_add(struct io_kiocb *req, unsigned flags)
13061315
unsigned nr_wait, nr_tw, nr_tw_prev;
13071316
struct llist_node *head;
13081317

1318+
/* See comment above IO_CQ_WAKE_INIT */
1319+
BUILD_BUG_ON(IO_CQ_WAKE_FORCE <= IORING_MAX_CQ_ENTRIES);
1320+
1321+
/*
1322+
* We don't know how many reuqests is there in the link and whether
1323+
* they can even be queued lazily, fall back to non-lazy.
1324+
*/
13091325
if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK))
13101326
flags &= ~IOU_F_TWQ_LAZY_WAKE;
13111327

@@ -1322,10 +1338,14 @@ static inline void io_req_local_work_add(struct io_kiocb *req, unsigned flags)
13221338
*/
13231339
nr_tw_prev = READ_ONCE(first_req->nr_tw);
13241340
}
1341+
1342+
/*
1343+
* Theoretically, it can overflow, but that's fine as one of
1344+
* previous adds should've tried to wake the task.
1345+
*/
13251346
nr_tw = nr_tw_prev + 1;
1326-
/* Large enough to fail the nr_wait comparison below */
13271347
if (!(flags & IOU_F_TWQ_LAZY_WAKE))
1328-
nr_tw = INT_MAX;
1348+
nr_tw = IO_CQ_WAKE_FORCE;
13291349

13301350
req->nr_tw = nr_tw;
13311351
req->io_task_work.node.next = head;
@@ -1348,11 +1368,11 @@ static inline void io_req_local_work_add(struct io_kiocb *req, unsigned flags)
13481368
}
13491369

13501370
nr_wait = atomic_read(&ctx->cq_wait_nr);
1351-
/* no one is waiting */
1352-
if (!nr_wait)
1371+
/* not enough or no one is waiting */
1372+
if (nr_tw < nr_wait)
13531373
return;
1354-
/* either not enough or the previous add has already woken it up */
1355-
if (nr_wait > nr_tw || nr_tw_prev >= nr_wait)
1374+
/* the previous add has already woken it up */
1375+
if (nr_tw_prev >= nr_wait)
13561376
return;
13571377
wake_up_state(ctx->submitter_task, TASK_INTERRUPTIBLE);
13581378
}
@@ -2620,7 +2640,7 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
26202640

26212641
ret = io_cqring_wait_schedule(ctx, &iowq);
26222642
__set_current_state(TASK_RUNNING);
2623-
atomic_set(&ctx->cq_wait_nr, 0);
2643+
atomic_set(&ctx->cq_wait_nr, IO_CQ_WAKE_INIT);
26242644

26252645
/*
26262646
* Run task_work after scheduling and before io_should_wake().

0 commit comments

Comments
 (0)