Skip to content

Commit 82982fd

Browse files
samitolvanenpalmer-dabbelt
authored andcommitted
riscv: Deduplicate IRQ stack switching
With CONFIG_IRQ_STACKS, we switch to a separate per-CPU IRQ stack before calling handle_riscv_irq or __do_softirq. We currently have duplicate inline assembly snippets for stack switching in both code paths. Now that we can access per-CPU variables in assembly, implement call_on_irq_stack in assembly, and use that instead of redundant inline assembly. Signed-off-by: Sami Tolvanen <samitolvanen@google.com> Tested-by: Nathan Chancellor <nathan@kernel.org> Reviewed-by: Guo Ren <guoren@kernel.org> Link: https://lore.kernel.org/r/20230927224757.1154247-10-samitolvanen@google.com Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
1 parent be97d0d commit 82982fd

File tree

6 files changed

+55
-55
lines changed

6 files changed

+55
-55
lines changed

arch/riscv/include/asm/asm.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,11 @@
104104
.endm
105105
#endif /* CONFIG_SMP */
106106

107+
.macro load_per_cpu dst ptr tmp
108+
asm_per_cpu \dst \ptr \tmp
109+
REG_L \dst, 0(\dst)
110+
.endm
111+
107112
/* save all GPs except x1 ~ x5 */
108113
.macro save_from_x6_to_x31
109114
REG_S x6, PT_T1(sp)

arch/riscv/include/asm/irq_stack.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@
1212

1313
DECLARE_PER_CPU(ulong *, irq_stack_ptr);
1414

15+
asmlinkage void call_on_irq_stack(struct pt_regs *regs,
16+
void (*func)(struct pt_regs *));
17+
1518
#ifdef CONFIG_VMAP_STACK
1619
/*
1720
* To ensure that VMAP'd stack overflow detection works correctly, all VMAP'd

arch/riscv/kernel/asm-offsets.c

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include <asm/thread_info.h>
1515
#include <asm/ptrace.h>
1616
#include <asm/cpu_ops_sbi.h>
17+
#include <asm/stacktrace.h>
1718
#include <asm/suspend.h>
1819

1920
void asm_offsets(void);
@@ -480,4 +481,8 @@ void asm_offsets(void)
480481
OFFSET(KERNEL_MAP_VIRT_ADDR, kernel_mapping, virt_addr);
481482
OFFSET(SBI_HART_BOOT_TASK_PTR_OFFSET, sbi_hart_boot_data, task_ptr);
482483
OFFSET(SBI_HART_BOOT_STACK_PTR_OFFSET, sbi_hart_boot_data, stack_ptr);
484+
485+
DEFINE(STACKFRAME_SIZE_ON_STACK, ALIGN(sizeof(struct stackframe), STACK_ALIGN));
486+
OFFSET(STACKFRAME_FP, stackframe, fp);
487+
OFFSET(STACKFRAME_RA, stackframe, ra);
483488
}

arch/riscv/kernel/entry.S

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,36 @@ SYM_CODE_START(ret_from_fork)
218218
tail syscall_exit_to_user_mode
219219
SYM_CODE_END(ret_from_fork)
220220

221+
#ifdef CONFIG_IRQ_STACKS
222+
/*
223+
* void call_on_irq_stack(struct pt_regs *regs,
224+
* void (*func)(struct pt_regs *));
225+
*
226+
* Calls func(regs) using the per-CPU IRQ stack.
227+
*/
228+
SYM_FUNC_START(call_on_irq_stack)
229+
/* Create a frame record to save ra and s0 (fp) */
230+
addi sp, sp, -STACKFRAME_SIZE_ON_STACK
231+
REG_S ra, STACKFRAME_RA(sp)
232+
REG_S s0, STACKFRAME_FP(sp)
233+
addi s0, sp, STACKFRAME_SIZE_ON_STACK
234+
235+
/* Switch to the per-CPU IRQ stack and call the handler */
236+
load_per_cpu t0, irq_stack_ptr, t1
237+
li t1, IRQ_STACK_SIZE
238+
add sp, t0, t1
239+
jalr a1
240+
241+
/* Switch back to the thread stack and restore ra and s0 */
242+
addi sp, s0, -STACKFRAME_SIZE_ON_STACK
243+
REG_L ra, STACKFRAME_RA(sp)
244+
REG_L s0, STACKFRAME_FP(sp)
245+
addi sp, sp, STACKFRAME_SIZE_ON_STACK
246+
247+
ret
248+
SYM_FUNC_END(call_on_irq_stack)
249+
#endif /* CONFIG_IRQ_STACKS */
250+
221251
/*
222252
* Integer register context switch
223253
* The callee-saved registers must be saved and restored.

arch/riscv/kernel/irq.c

Lines changed: 8 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -61,35 +61,16 @@ static void init_irq_stacks(void)
6161
#endif /* CONFIG_VMAP_STACK */
6262

6363
#ifdef CONFIG_HAVE_SOFTIRQ_ON_OWN_STACK
64+
static void ___do_softirq(struct pt_regs *regs)
65+
{
66+
__do_softirq();
67+
}
68+
6469
void do_softirq_own_stack(void)
6570
{
66-
#ifdef CONFIG_IRQ_STACKS
67-
if (on_thread_stack()) {
68-
ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id())
69-
+ IRQ_STACK_SIZE/sizeof(ulong);
70-
__asm__ __volatile(
71-
"addi sp, sp, -"RISCV_SZPTR "\n"
72-
REG_S" ra, (sp) \n"
73-
"addi sp, sp, -"RISCV_SZPTR "\n"
74-
REG_S" s0, (sp) \n"
75-
"addi s0, sp, 2*"RISCV_SZPTR "\n"
76-
"move sp, %[sp] \n"
77-
"call __do_softirq \n"
78-
"addi sp, s0, -2*"RISCV_SZPTR"\n"
79-
REG_L" s0, (sp) \n"
80-
"addi sp, sp, "RISCV_SZPTR "\n"
81-
REG_L" ra, (sp) \n"
82-
"addi sp, sp, "RISCV_SZPTR "\n"
83-
:
84-
: [sp] "r" (sp)
85-
: "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7",
86-
"t0", "t1", "t2", "t3", "t4", "t5", "t6",
87-
#ifndef CONFIG_FRAME_POINTER
88-
"s0",
89-
#endif
90-
"memory");
91-
} else
92-
#endif
71+
if (on_thread_stack())
72+
call_on_irq_stack(NULL, ___do_softirq);
73+
else
9374
__do_softirq();
9475
}
9576
#endif /* CONFIG_HAVE_SOFTIRQ_ON_OWN_STACK */

arch/riscv/kernel/traps.c

Lines changed: 4 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -352,34 +352,10 @@ static void noinstr handle_riscv_irq(struct pt_regs *regs)
352352
asmlinkage void noinstr do_irq(struct pt_regs *regs)
353353
{
354354
irqentry_state_t state = irqentry_enter(regs);
355-
#ifdef CONFIG_IRQ_STACKS
356-
if (on_thread_stack()) {
357-
ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id())
358-
+ IRQ_STACK_SIZE/sizeof(ulong);
359-
__asm__ __volatile(
360-
"addi sp, sp, -"RISCV_SZPTR "\n"
361-
REG_S" ra, (sp) \n"
362-
"addi sp, sp, -"RISCV_SZPTR "\n"
363-
REG_S" s0, (sp) \n"
364-
"addi s0, sp, 2*"RISCV_SZPTR "\n"
365-
"move sp, %[sp] \n"
366-
"move a0, %[regs] \n"
367-
"call handle_riscv_irq \n"
368-
"addi sp, s0, -2*"RISCV_SZPTR"\n"
369-
REG_L" s0, (sp) \n"
370-
"addi sp, sp, "RISCV_SZPTR "\n"
371-
REG_L" ra, (sp) \n"
372-
"addi sp, sp, "RISCV_SZPTR "\n"
373-
:
374-
: [sp] "r" (sp), [regs] "r" (regs)
375-
: "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7",
376-
"t0", "t1", "t2", "t3", "t4", "t5", "t6",
377-
#ifndef CONFIG_FRAME_POINTER
378-
"s0",
379-
#endif
380-
"memory");
381-
} else
382-
#endif
355+
356+
if (IS_ENABLED(CONFIG_IRQ_STACKS) && on_thread_stack())
357+
call_on_irq_stack(regs, handle_riscv_irq);
358+
else
383359
handle_riscv_irq(regs);
384360

385361
irqentry_exit(regs, state);

0 commit comments

Comments
 (0)