@@ -137,6 +137,14 @@ struct io_defer_entry {
137
137
#define IO_DISARM_MASK (REQ_F_ARM_LTIMEOUT | REQ_F_LINK_TIMEOUT | REQ_F_FAIL)
138
138
#define IO_REQ_LINK_FLAGS (REQ_F_LINK | REQ_F_HARDLINK)
139
139
140
+ /*
141
+ * No waiters. It's larger than any valid value of the tw counter
142
+ * so that tests against ->cq_wait_nr would fail and skip wake_up().
143
+ */
144
+ #define IO_CQ_WAKE_INIT (-1U)
145
+ /* Forced wake up if there is a waiter regardless of ->cq_wait_nr */
146
+ #define IO_CQ_WAKE_FORCE (IO_CQ_WAKE_INIT >> 1)
147
+
140
148
static bool io_uring_try_cancel_requests (struct io_ring_ctx * ctx ,
141
149
struct task_struct * task ,
142
150
bool cancel_all );
@@ -303,6 +311,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
303
311
goto err ;
304
312
305
313
ctx -> flags = p -> flags ;
314
+ atomic_set (& ctx -> cq_wait_nr , IO_CQ_WAKE_INIT );
306
315
init_waitqueue_head (& ctx -> sqo_sq_wait );
307
316
INIT_LIST_HEAD (& ctx -> sqd_list );
308
317
INIT_LIST_HEAD (& ctx -> cq_overflow_list );
@@ -1306,6 +1315,13 @@ static inline void io_req_local_work_add(struct io_kiocb *req, unsigned flags)
1306
1315
unsigned nr_wait , nr_tw , nr_tw_prev ;
1307
1316
struct llist_node * head ;
1308
1317
1318
+ /* See comment above IO_CQ_WAKE_INIT */
1319
+ BUILD_BUG_ON (IO_CQ_WAKE_FORCE <= IORING_MAX_CQ_ENTRIES );
1320
+
1321
+ /*
1322
+ * We don't know how many reuqests is there in the link and whether
1323
+ * they can even be queued lazily, fall back to non-lazy.
1324
+ */
1309
1325
if (req -> flags & (REQ_F_LINK | REQ_F_HARDLINK ))
1310
1326
flags &= ~IOU_F_TWQ_LAZY_WAKE ;
1311
1327
@@ -1322,10 +1338,14 @@ static inline void io_req_local_work_add(struct io_kiocb *req, unsigned flags)
1322
1338
*/
1323
1339
nr_tw_prev = READ_ONCE (first_req -> nr_tw );
1324
1340
}
1341
+
1342
+ /*
1343
+ * Theoretically, it can overflow, but that's fine as one of
1344
+ * previous adds should've tried to wake the task.
1345
+ */
1325
1346
nr_tw = nr_tw_prev + 1 ;
1326
- /* Large enough to fail the nr_wait comparison below */
1327
1347
if (!(flags & IOU_F_TWQ_LAZY_WAKE ))
1328
- nr_tw = INT_MAX ;
1348
+ nr_tw = IO_CQ_WAKE_FORCE ;
1329
1349
1330
1350
req -> nr_tw = nr_tw ;
1331
1351
req -> io_task_work .node .next = head ;
@@ -1348,11 +1368,11 @@ static inline void io_req_local_work_add(struct io_kiocb *req, unsigned flags)
1348
1368
}
1349
1369
1350
1370
nr_wait = atomic_read (& ctx -> cq_wait_nr );
1351
- /* no one is waiting */
1352
- if (! nr_wait )
1371
+ /* not enough or no one is waiting */
1372
+ if (nr_tw < nr_wait )
1353
1373
return ;
1354
- /* either not enough or the previous add has already woken it up */
1355
- if (nr_wait > nr_tw || nr_tw_prev >= nr_wait )
1374
+ /* the previous add has already woken it up */
1375
+ if (nr_tw_prev >= nr_wait )
1356
1376
return ;
1357
1377
wake_up_state (ctx -> submitter_task , TASK_INTERRUPTIBLE );
1358
1378
}
@@ -2620,7 +2640,7 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
2620
2640
2621
2641
ret = io_cqring_wait_schedule (ctx , & iowq );
2622
2642
__set_current_state (TASK_RUNNING );
2623
- atomic_set (& ctx -> cq_wait_nr , 0 );
2643
+ atomic_set (& ctx -> cq_wait_nr , IO_CQ_WAKE_INIT );
2624
2644
2625
2645
/*
2626
2646
* Run task_work after scheduling and before io_should_wake().
0 commit comments