Skip to content

Commit 0aa7aa5

Browse files
isilenceaxboe
authored andcommitted
io_uring: move multishot cqe cache in ctx
We cache multishot CQEs before flushing them to the CQ in submit_state.cqe. It's a 16 entry cache totalling 256 bytes in the middle of the io_submit_state structure. Move it out of there, it should help with CPU caches for the submission state, and shouldn't affect cached CQEs. Signed-off-by: Pavel Begunkov <asml.silence@gmail.com> Link: https://lore.kernel.org/r/dbe1f39c043ee23da918836be44fcec252ce6711.1692916914.git.asml.silence@gmail.com Signed-off-by: Jens Axboe <axboe@kernel.dk>
1 parent c9def23 commit 0aa7aa5

File tree

2 files changed

+5
-4
lines changed

2 files changed

+5
-4
lines changed

include/linux/io_uring_types.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,6 @@ struct io_submit_state {
176176
unsigned short submit_nr;
177177
unsigned int cqes_count;
178178
struct blk_plug plug;
179-
struct io_uring_cqe cqes[16];
180179
};
181180

182181
struct io_ev_fd {
@@ -307,6 +306,8 @@ struct io_ring_ctx {
307306
unsigned cq_last_tm_flush;
308307
} ____cacheline_aligned_in_smp;
309308

309+
struct io_uring_cqe completion_cqes[16];
310+
310311
/* IRQ completion list, under ->completion_lock */
311312
struct io_wq_work_list locked_free_list;
312313
unsigned int locked_free_nr;

io_uring/io_uring.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -880,7 +880,7 @@ static void __io_flush_post_cqes(struct io_ring_ctx *ctx)
880880

881881
lockdep_assert_held(&ctx->uring_lock);
882882
for (i = 0; i < state->cqes_count; i++) {
883-
struct io_uring_cqe *cqe = &state->cqes[i];
883+
struct io_uring_cqe *cqe = &ctx->completion_cqes[i];
884884

885885
if (!io_fill_cqe_aux(ctx, cqe->user_data, cqe->res, cqe->flags)) {
886886
if (ctx->task_complete) {
@@ -931,7 +931,7 @@ bool io_fill_cqe_req_aux(struct io_kiocb *req, bool defer, s32 res, u32 cflags)
931931

932932
lockdep_assert_held(&ctx->uring_lock);
933933

934-
if (ctx->submit_state.cqes_count == ARRAY_SIZE(ctx->submit_state.cqes)) {
934+
if (ctx->submit_state.cqes_count == ARRAY_SIZE(ctx->completion_cqes)) {
935935
__io_cq_lock(ctx);
936936
__io_flush_post_cqes(ctx);
937937
/* no need to flush - flush is deferred */
@@ -945,7 +945,7 @@ bool io_fill_cqe_req_aux(struct io_kiocb *req, bool defer, s32 res, u32 cflags)
945945
if (test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq))
946946
return false;
947947

948-
cqe = &ctx->submit_state.cqes[ctx->submit_state.cqes_count++];
948+
cqe = &ctx->completion_cqes[ctx->submit_state.cqes_count++];
949949
cqe->user_data = user_data;
950950
cqe->res = res;
951951
cqe->flags = cflags;

0 commit comments

Comments
 (0)