Skip to content

Commit a44fb57

Browse files
charlie-rivosAlexandre Ghiti
charlie-rivos
authored and
Alexandre Ghiti
committed
riscv: Add runtime constant support
Implement the runtime constant infrastructure for riscv. Use this infrastructure to generate constants to be used by the d_hash() function. This is the riscv variant of commit 94a2bc0 ("arm64: add 'runtime constant' support") and commit e3c92e8 ("runtime constants: add x86 architecture support"). [ alex: Remove trailing whitespace ] Signed-off-by: Charlie Jenkins <charlie@rivosinc.com> Reviewed-by: Alexandre Ghiti <alexghiti@rivosinc.com> Tested-by: Alexandre Ghiti <alexghiti@rivosinc.com> Link: https://lore.kernel.org/r/20250319-runtime_const_riscv-v10-2-745b31a11d65@rivosinc.com Signed-off-by: Alexandre Ghiti <alexghiti@rivosinc.com>
1 parent afa8a93 commit a44fb57

File tree

4 files changed

+291
-0
lines changed

4 files changed

+291
-0
lines changed

‎arch/riscv/Kconfig

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -783,6 +783,28 @@ config RISCV_ISA_ZBC
783783

784784
If you don't know what to do here, say Y.
785785

786+
config TOOLCHAIN_HAS_ZBKB
787+
bool
788+
default y
789+
depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zbkb)
790+
depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zbkb)
791+
depends on LLD_VERSION >= 150000 || LD_VERSION >= 23900
792+
depends on AS_HAS_OPTION_ARCH
793+
794+
config RISCV_ISA_ZBKB
795+
bool "Zbkb extension support for bit manipulation instructions"
796+
depends on TOOLCHAIN_HAS_ZBKB
797+
depends on RISCV_ALTERNATIVE
798+
default y
799+
help
800+
Adds support to dynamically detect the presence of the ZBKB
801+
extension (bit manipulation for cryptography) and enable its usage.
802+
803+
The Zbkb extension provides instructions to accelerate a number
804+
of common cryptography operations (pack, zip, etc).
805+
806+
If you don't know what to do here, say Y.
807+
786808
config RISCV_ISA_ZICBOM
787809
bool "Zicbom extension support for non-coherent DMA operation"
788810
depends on MMU

‎arch/riscv/include/asm/asm.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#define REG_ASM __REG_SEL(.dword, .word)
2828
#define SZREG __REG_SEL(8, 4)
2929
#define LGREG __REG_SEL(3, 2)
30+
#define SRLI __REG_SEL(srliw, srli)
3031

3132
#if __SIZEOF_POINTER__ == 8
3233
#ifdef __ASSEMBLY__

‎arch/riscv/include/asm/runtime-const.h

Lines changed: 265 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,265 @@
1+
/* SPDX-License-Identifier: GPL-2.0 */
2+
#ifndef _ASM_RISCV_RUNTIME_CONST_H
3+
#define _ASM_RISCV_RUNTIME_CONST_H
4+
5+
#include <asm/asm.h>
6+
#include <asm/alternative.h>
7+
#include <asm/cacheflush.h>
8+
#include <asm/insn-def.h>
9+
#include <linux/memory.h>
10+
#include <asm/text-patching.h>
11+
12+
#include <linux/uaccess.h>
13+
14+
#ifdef CONFIG_32BIT
15+
#define runtime_const_ptr(sym) \
16+
({ \
17+
typeof(sym) __ret; \
18+
asm_inline(".option push\n\t" \
19+
".option norvc\n\t" \
20+
"1:\t" \
21+
"lui %[__ret],0x89abd\n\t" \
22+
"addi %[__ret],%[__ret],-0x211\n\t" \
23+
".option pop\n\t" \
24+
".pushsection runtime_ptr_" #sym ",\"a\"\n\t" \
25+
".long 1b - .\n\t" \
26+
".popsection" \
27+
: [__ret] "=r" (__ret)); \
28+
__ret; \
29+
})
30+
#else
31+
/*
32+
* Loading 64-bit constants into a register from immediates is a non-trivial
33+
* task on riscv64. To get it somewhat performant, load 32 bits into two
34+
* different registers and then combine the results.
35+
*
36+
* If the processor supports the Zbkb extension, we can combine the final
37+
* "slli,slli,srli,add" into the single "pack" instruction. If the processor
38+
* doesn't support Zbkb but does support the Zbb extension, we can
39+
* combine the final "slli,srli,add" into one instruction "add.uw".
40+
*/
41+
#define RISCV_RUNTIME_CONST_64_PREAMBLE \
42+
".option push\n\t" \
43+
".option norvc\n\t" \
44+
"1:\t" \
45+
"lui %[__ret],0x89abd\n\t" \
46+
"lui %[__tmp],0x1234\n\t" \
47+
"addiw %[__ret],%[__ret],-0x211\n\t" \
48+
"addiw %[__tmp],%[__tmp],0x567\n\t" \
49+
50+
#define RISCV_RUNTIME_CONST_64_BASE \
51+
"slli %[__tmp],%[__tmp],32\n\t" \
52+
"slli %[__ret],%[__ret],32\n\t" \
53+
"srli %[__ret],%[__ret],32\n\t" \
54+
"add %[__ret],%[__ret],%[__tmp]\n\t" \
55+
56+
#define RISCV_RUNTIME_CONST_64_ZBA \
57+
".option push\n\t" \
58+
".option arch,+zba\n\t" \
59+
"slli %[__tmp],%[__tmp],32\n\t" \
60+
"add.uw %[__ret],%[__ret],%[__tmp]\n\t" \
61+
"nop\n\t" \
62+
"nop\n\t" \
63+
".option pop\n\t" \
64+
65+
#define RISCV_RUNTIME_CONST_64_ZBKB \
66+
".option push\n\t" \
67+
".option arch,+zbkb\n\t" \
68+
"pack %[__ret],%[__ret],%[__tmp]\n\t" \
69+
"nop\n\t" \
70+
"nop\n\t" \
71+
"nop\n\t" \
72+
".option pop\n\t" \
73+
74+
#define RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \
75+
".option pop\n\t" \
76+
".pushsection runtime_ptr_" #sym ",\"a\"\n\t" \
77+
".long 1b - .\n\t" \
78+
".popsection" \
79+
80+
#if defined(CONFIG_RISCV_ISA_ZBA) && defined(CONFIG_RISCV_ISA_ZBKB)
81+
#define runtime_const_ptr(sym) \
82+
({ \
83+
typeof(sym) __ret, __tmp; \
84+
asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \
85+
ALTERNATIVE_2( \
86+
RISCV_RUNTIME_CONST_64_BASE, \
87+
RISCV_RUNTIME_CONST_64_ZBA, \
88+
0, RISCV_ISA_EXT_ZBA, 1, \
89+
RISCV_RUNTIME_CONST_64_ZBKB, \
90+
0, RISCV_ISA_EXT_ZBKB, 1 \
91+
) \
92+
RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \
93+
: [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \
94+
__ret; \
95+
})
96+
#elif defined(CONFIG_RISCV_ISA_ZBA)
97+
#define runtime_const_ptr(sym) \
98+
({ \
99+
typeof(sym) __ret, __tmp; \
100+
asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \
101+
ALTERNATIVE( \
102+
RISCV_RUNTIME_CONST_64_BASE, \
103+
RISCV_RUNTIME_CONST_64_ZBA, \
104+
0, RISCV_ISA_EXT_ZBA, 1 \
105+
) \
106+
RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \
107+
: [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \
108+
__ret; \
109+
})
110+
#elif defined(CONFIG_RISCV_ISA_ZBKB)
111+
#define runtime_const_ptr(sym) \
112+
({ \
113+
typeof(sym) __ret, __tmp; \
114+
asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \
115+
ALTERNATIVE( \
116+
RISCV_RUNTIME_CONST_64_BASE, \
117+
RISCV_RUNTIME_CONST_64_ZBKB, \
118+
0, RISCV_ISA_EXT_ZBKB, 1 \
119+
) \
120+
RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \
121+
: [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \
122+
__ret; \
123+
})
124+
#else
125+
#define runtime_const_ptr(sym) \
126+
({ \
127+
typeof(sym) __ret, __tmp; \
128+
asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \
129+
RISCV_RUNTIME_CONST_64_BASE \
130+
RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \
131+
: [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \
132+
__ret; \
133+
})
134+
#endif
135+
#endif
136+
137+
#define runtime_const_shift_right_32(val, sym) \
138+
({ \
139+
u32 __ret; \
140+
asm_inline(".option push\n\t" \
141+
".option norvc\n\t" \
142+
"1:\t" \
143+
SRLI " %[__ret],%[__val],12\n\t" \
144+
".option pop\n\t" \
145+
".pushsection runtime_shift_" #sym ",\"a\"\n\t" \
146+
".long 1b - .\n\t" \
147+
".popsection" \
148+
: [__ret] "=r" (__ret) \
149+
: [__val] "r" (val)); \
150+
__ret; \
151+
})
152+
153+
#define runtime_const_init(type, sym) do { \
154+
extern s32 __start_runtime_##type##_##sym[]; \
155+
extern s32 __stop_runtime_##type##_##sym[]; \
156+
\
157+
runtime_const_fixup(__runtime_fixup_##type, \
158+
(unsigned long)(sym), \
159+
__start_runtime_##type##_##sym, \
160+
__stop_runtime_##type##_##sym); \
161+
} while (0)
162+
163+
static inline void __runtime_fixup_caches(void *where, unsigned int insns)
164+
{
165+
/* On riscv there are currently only cache-wide flushes so va is ignored. */
166+
__always_unused uintptr_t va = (uintptr_t)where;
167+
168+
flush_icache_range(va, va + 4 * insns);
169+
}
170+
171+
/*
172+
* The 32-bit immediate is stored in a lui+addi pairing.
173+
* lui holds the upper 20 bits of the immediate in the first 20 bits of the instruction.
174+
* addi holds the lower 12 bits of the immediate in the first 12 bits of the instruction.
175+
*/
176+
static inline void __runtime_fixup_32(__le16 *lui_parcel, __le16 *addi_parcel, unsigned int val)
177+
{
178+
unsigned int lower_immediate, upper_immediate;
179+
u32 lui_insn, addi_insn, addi_insn_mask;
180+
__le32 lui_res, addi_res;
181+
182+
/* Mask out upper 12 bit of addi */
183+
addi_insn_mask = 0x000fffff;
184+
185+
lui_insn = (u32)le16_to_cpu(lui_parcel[0]) | (u32)le16_to_cpu(lui_parcel[1]) << 16;
186+
addi_insn = (u32)le16_to_cpu(addi_parcel[0]) | (u32)le16_to_cpu(addi_parcel[1]) << 16;
187+
188+
lower_immediate = sign_extend32(val, 11);
189+
upper_immediate = (val - lower_immediate);
190+
191+
if (upper_immediate & 0xfffff000) {
192+
/* replace upper 20 bits of lui with upper immediate */
193+
lui_insn &= 0x00000fff;
194+
lui_insn |= upper_immediate & 0xfffff000;
195+
} else {
196+
/* replace lui with nop if immediate is small enough to fit in addi */
197+
lui_insn = RISCV_INSN_NOP4;
198+
/*
199+
* lui is being skipped, so do a load instead of an add. A load
200+
* is performed by adding with the x0 register. Setting rs to
201+
* zero with the following mask will accomplish this goal.
202+
*/
203+
addi_insn_mask &= 0x07fff;
204+
}
205+
206+
if (lower_immediate & 0x00000fff) {
207+
/* replace upper 12 bits of addi with lower 12 bits of val */
208+
addi_insn &= addi_insn_mask;
209+
addi_insn |= (lower_immediate & 0x00000fff) << 20;
210+
} else {
211+
/* replace addi with nop if lower_immediate is empty */
212+
addi_insn = RISCV_INSN_NOP4;
213+
}
214+
215+
addi_res = cpu_to_le32(addi_insn);
216+
lui_res = cpu_to_le32(lui_insn);
217+
mutex_lock(&text_mutex);
218+
patch_insn_write(addi_parcel, &addi_res, sizeof(addi_res));
219+
patch_insn_write(lui_parcel, &lui_res, sizeof(lui_res));
220+
mutex_unlock(&text_mutex);
221+
}
222+
223+
static inline void __runtime_fixup_ptr(void *where, unsigned long val)
224+
{
225+
#ifdef CONFIG_32BIT
226+
__runtime_fixup_32(where, where + 4, val);
227+
__runtime_fixup_caches(where, 2);
228+
#else
229+
__runtime_fixup_32(where, where + 8, val);
230+
__runtime_fixup_32(where + 4, where + 12, val >> 32);
231+
__runtime_fixup_caches(where, 4);
232+
#endif
233+
}
234+
235+
/*
236+
* Replace the least significant 5 bits of the srli/srliw immediate that is
237+
* located at bits 20-24
238+
*/
239+
static inline void __runtime_fixup_shift(void *where, unsigned long val)
240+
{
241+
__le16 *parcel = where;
242+
__le32 res;
243+
u32 insn;
244+
245+
insn = (u32)le16_to_cpu(parcel[0]) | (u32)le16_to_cpu(parcel[1]) << 16;
246+
247+
insn &= 0xfe0fffff;
248+
insn |= (val & 0b11111) << 20;
249+
250+
res = cpu_to_le32(insn);
251+
mutex_lock(&text_mutex);
252+
patch_text_nosync(where, &res, sizeof(insn));
253+
mutex_unlock(&text_mutex);
254+
}
255+
256+
static inline void runtime_const_fixup(void (*fn)(void *, unsigned long),
257+
unsigned long val, s32 *start, s32 *end)
258+
{
259+
while (start < end) {
260+
fn(*start + (void *)start, val);
261+
start++;
262+
}
263+
}
264+
265+
#endif /* _ASM_RISCV_RUNTIME_CONST_H */

‎arch/riscv/kernel/vmlinux.lds.S

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,9 @@ SECTIONS
9797
{
9898
EXIT_DATA
9999
}
100+
101+
RUNTIME_CONST_VARIABLES
102+
100103
PERCPU_SECTION(L1_CACHE_BYTES)
101104

102105
.rel.dyn : {

0 commit comments

Comments
 (0)