Skip to content

Commit c92fcfc

Browse files
committed
io_uring: avoid indirect function calls for the hottest task_work
We use task_work for a variety of reasons, but doing completions or triggering rety after poll are by far the hottest two. Use the indirect funtion call wrappers to avoid the indirect function call if CONFIG_RETPOLINE is set. Signed-off-by: Jens Axboe <axboe@kernel.dk>
1 parent f026be0 commit c92fcfc

File tree

5 files changed

+12
-4
lines changed

5 files changed

+12
-4
lines changed

io_uring/io_uring.c

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@
9595

9696
#include "timeout.h"
9797
#include "poll.h"
98+
#include "rw.h"
9899
#include "alloc_cache.h"
99100

100101
#define IORING_MAX_ENTRIES 32768
@@ -1205,7 +1206,9 @@ static unsigned int handle_tw_list(struct llist_node *node,
12051206
ts->locked = mutex_trylock(&(*ctx)->uring_lock);
12061207
percpu_ref_get(&(*ctx)->refs);
12071208
}
1208-
req->io_task_work.func(req, ts);
1209+
INDIRECT_CALL_2(req->io_task_work.func,
1210+
io_poll_task_func, io_req_rw_complete,
1211+
req, ts);
12091212
node = next;
12101213
count++;
12111214
if (unlikely(need_resched())) {
@@ -1415,7 +1418,9 @@ static int __io_run_local_work(struct io_ring_ctx *ctx, struct io_tw_state *ts)
14151418
struct io_kiocb *req = container_of(node, struct io_kiocb,
14161419
io_task_work.node);
14171420
prefetch(container_of(next, struct io_kiocb, io_task_work.node));
1418-
req->io_task_work.func(req, ts);
1421+
INDIRECT_CALL_2(req->io_task_work.func,
1422+
io_poll_task_func, io_req_rw_complete,
1423+
req, ts);
14191424
ret++;
14201425
node = next;
14211426
}

io_uring/poll.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,7 @@ static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts)
326326
return IOU_POLL_NO_ACTION;
327327
}
328328

329-
static void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts)
329+
void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts)
330330
{
331331
int ret;
332332

io_uring/poll.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,5 @@ bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,
3838
bool cancel_all);
3939

4040
void io_apoll_cache_free(struct io_cache_entry *entry);
41+
42+
void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts);

io_uring/rw.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -283,7 +283,7 @@ static inline int io_fixup_rw_res(struct io_kiocb *req, long res)
283283
return res;
284284
}
285285

286-
static void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts)
286+
void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts)
287287
{
288288
io_req_io_end(req);
289289

io_uring/rw.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,4 @@ int io_write(struct io_kiocb *req, unsigned int issue_flags);
2222
int io_writev_prep_async(struct io_kiocb *req);
2323
void io_readv_writev_cleanup(struct io_kiocb *req);
2424
void io_rw_fail(struct io_kiocb *req);
25+
void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts);

0 commit comments

Comments
 (0)