Skip to content

Commit aecb09e

Browse files
jones-drewAlexandre Ghiti
authored andcommitted
riscv: Add parameter for skipping access speed tests
Allow skipping scalar and vector unaligned access speed tests. This is useful for testing alternative code paths and to skip the tests in environments where they run too slowly. All CPUs must have the same unaligned access speed. The code movement is because we now need the scalar cpu hotplug callback to always run, so we need to bring it and its supporting functions out of CONFIG_RISCV_PROBE_UNALIGNED_ACCESS. Signed-off-by: Andrew Jones <ajones@ventanamicro.com> Link: https://lore.kernel.org/r/20250304120014.143628-17-ajones@ventanamicro.com Signed-off-by: Alexandre Ghiti <alexghiti@rivosinc.com>
1 parent 2744ec4 commit aecb09e

File tree

1 file changed

+121
-66
lines changed

1 file changed

+121
-66
lines changed

arch/riscv/kernel/unaligned_access_speed.c

Lines changed: 121 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,12 @@
2424
DEFINE_PER_CPU(long, misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
2525
DEFINE_PER_CPU(long, vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
2626

27-
#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
27+
static long unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
28+
static long unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
29+
2830
static cpumask_t fast_misaligned_access;
31+
32+
#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
2933
static int check_unaligned_access(void *param)
3034
{
3135
int cpu = smp_processor_id();
@@ -130,6 +134,50 @@ static void __init check_unaligned_access_nonboot_cpu(void *param)
130134
check_unaligned_access(pages[cpu]);
131135
}
132136

137+
/* Measure unaligned access speed on all CPUs present at boot in parallel. */
138+
static void __init check_unaligned_access_speed_all_cpus(void)
139+
{
140+
unsigned int cpu;
141+
unsigned int cpu_count = num_possible_cpus();
142+
struct page **bufs = kcalloc(cpu_count, sizeof(*bufs), GFP_KERNEL);
143+
144+
if (!bufs) {
145+
pr_warn("Allocation failure, not measuring misaligned performance\n");
146+
return;
147+
}
148+
149+
/*
150+
* Allocate separate buffers for each CPU so there's no fighting over
151+
* cache lines.
152+
*/
153+
for_each_cpu(cpu, cpu_online_mask) {
154+
bufs[cpu] = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER);
155+
if (!bufs[cpu]) {
156+
pr_warn("Allocation failure, not measuring misaligned performance\n");
157+
goto out;
158+
}
159+
}
160+
161+
/* Check everybody except 0, who stays behind to tend jiffies. */
162+
on_each_cpu(check_unaligned_access_nonboot_cpu, bufs, 1);
163+
164+
/* Check core 0. */
165+
smp_call_on_cpu(0, check_unaligned_access, bufs[0], true);
166+
167+
out:
168+
for_each_cpu(cpu, cpu_online_mask) {
169+
if (bufs[cpu])
170+
__free_pages(bufs[cpu], MISALIGNED_BUFFER_ORDER);
171+
}
172+
173+
kfree(bufs);
174+
}
175+
#else /* CONFIG_RISCV_PROBE_UNALIGNED_ACCESS */
176+
static void __init check_unaligned_access_speed_all_cpus(void)
177+
{
178+
}
179+
#endif
180+
133181
DEFINE_STATIC_KEY_FALSE(fast_unaligned_access_speed_key);
134182

135183
static void modify_unaligned_access_branches(cpumask_t *mask, int weight)
@@ -188,21 +236,29 @@ arch_initcall_sync(lock_and_set_unaligned_access_static_branch);
188236

189237
static int riscv_online_cpu(unsigned int cpu)
190238
{
191-
static struct page *buf;
192-
193239
/* We are already set since the last check */
194-
if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN)
240+
if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) {
241+
goto exit;
242+
} else if (unaligned_scalar_speed_param != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) {
243+
per_cpu(misaligned_access_speed, cpu) = unaligned_scalar_speed_param;
195244
goto exit;
196-
197-
check_unaligned_access_emulated(NULL);
198-
buf = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER);
199-
if (!buf) {
200-
pr_warn("Allocation failure, not measuring misaligned performance\n");
201-
return -ENOMEM;
202245
}
203246

204-
check_unaligned_access(buf);
205-
__free_pages(buf, MISALIGNED_BUFFER_ORDER);
247+
#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
248+
{
249+
static struct page *buf;
250+
251+
check_unaligned_access_emulated(NULL);
252+
buf = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER);
253+
if (!buf) {
254+
pr_warn("Allocation failure, not measuring misaligned performance\n");
255+
return -ENOMEM;
256+
}
257+
258+
check_unaligned_access(buf);
259+
__free_pages(buf, MISALIGNED_BUFFER_ORDER);
260+
}
261+
#endif
206262

207263
exit:
208264
set_unaligned_access_static_branches();
@@ -217,50 +273,6 @@ static int riscv_offline_cpu(unsigned int cpu)
217273
return 0;
218274
}
219275

220-
/* Measure unaligned access speed on all CPUs present at boot in parallel. */
221-
static void __init check_unaligned_access_speed_all_cpus(void)
222-
{
223-
unsigned int cpu;
224-
unsigned int cpu_count = num_possible_cpus();
225-
struct page **bufs = kcalloc(cpu_count, sizeof(*bufs), GFP_KERNEL);
226-
227-
if (!bufs) {
228-
pr_warn("Allocation failure, not measuring misaligned performance\n");
229-
return;
230-
}
231-
232-
/*
233-
* Allocate separate buffers for each CPU so there's no fighting over
234-
* cache lines.
235-
*/
236-
for_each_cpu(cpu, cpu_online_mask) {
237-
bufs[cpu] = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER);
238-
if (!bufs[cpu]) {
239-
pr_warn("Allocation failure, not measuring misaligned performance\n");
240-
goto out;
241-
}
242-
}
243-
244-
/* Check everybody except 0, who stays behind to tend jiffies. */
245-
on_each_cpu(check_unaligned_access_nonboot_cpu, bufs, 1);
246-
247-
/* Check core 0. */
248-
smp_call_on_cpu(0, check_unaligned_access, bufs[0], true);
249-
250-
out:
251-
for_each_cpu(cpu, cpu_online_mask) {
252-
if (bufs[cpu])
253-
__free_pages(bufs[cpu], MISALIGNED_BUFFER_ORDER);
254-
}
255-
256-
kfree(bufs);
257-
}
258-
#else /* CONFIG_RISCV_PROBE_UNALIGNED_ACCESS */
259-
static void __init check_unaligned_access_speed_all_cpus(void)
260-
{
261-
}
262-
#endif
263-
264276
#ifdef CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS
265277
static void check_vector_unaligned_access(struct work_struct *work __always_unused)
266278
{
@@ -372,8 +384,8 @@ static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __alway
372384

373385
static int riscv_online_cpu_vec(unsigned int cpu)
374386
{
375-
if (!has_vector()) {
376-
per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
387+
if (unaligned_vector_speed_param != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) {
388+
per_cpu(vector_misaligned_access, cpu) = unaligned_vector_speed_param;
377389
return 0;
378390
}
379391

@@ -388,30 +400,73 @@ static int riscv_online_cpu_vec(unsigned int cpu)
388400
return 0;
389401
}
390402

403+
static const char * const speed_str[] __initconst = { NULL, NULL, "slow", "fast", "unsupported" };
404+
405+
static int __init set_unaligned_scalar_speed_param(char *str)
406+
{
407+
if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW]))
408+
unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW;
409+
else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_SCALAR_FAST]))
410+
unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_FAST;
411+
else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_SCALAR_UNSUPPORTED]))
412+
unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_UNSUPPORTED;
413+
else
414+
return -EINVAL;
415+
416+
return 1;
417+
}
418+
__setup("unaligned_scalar_speed=", set_unaligned_scalar_speed_param);
419+
420+
static int __init set_unaligned_vector_speed_param(char *str)
421+
{
422+
if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW]))
423+
unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW;
424+
else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_VECTOR_FAST]))
425+
unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_FAST;
426+
else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED]))
427+
unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
428+
else
429+
return -EINVAL;
430+
431+
return 1;
432+
}
433+
__setup("unaligned_vector_speed=", set_unaligned_vector_speed_param);
434+
391435
static int __init check_unaligned_access_all_cpus(void)
392436
{
393437
int cpu;
394438

395-
if (!check_unaligned_access_emulated_all_cpus())
439+
if (unaligned_scalar_speed_param == RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN &&
440+
!check_unaligned_access_emulated_all_cpus()) {
396441
check_unaligned_access_speed_all_cpus();
397-
398-
if (!has_vector()) {
442+
} else {
443+
pr_info("scalar unaligned access speed set to '%s' by command line\n",
444+
speed_str[unaligned_scalar_speed_param]);
399445
for_each_online_cpu(cpu)
400-
per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
401-
} else if (!check_vector_unaligned_access_emulated_all_cpus() &&
402-
IS_ENABLED(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS)) {
446+
per_cpu(misaligned_access_speed, cpu) = unaligned_scalar_speed_param;
447+
}
448+
449+
if (!has_vector())
450+
unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
451+
452+
if (unaligned_vector_speed_param == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN &&
453+
!check_vector_unaligned_access_emulated_all_cpus() &&
454+
IS_ENABLED(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS)) {
403455
kthread_run(vec_check_unaligned_access_speed_all_cpus,
404456
NULL, "vec_check_unaligned_access_speed_all_cpus");
457+
} else {
458+
pr_info("vector unaligned access speed set to '%s' by command line\n",
459+
speed_str[unaligned_vector_speed_param]);
460+
for_each_online_cpu(cpu)
461+
per_cpu(vector_misaligned_access, cpu) = unaligned_vector_speed_param;
405462
}
406463

407464
/*
408465
* Setup hotplug callbacks for any new CPUs that come online or go
409466
* offline.
410467
*/
411-
#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
412468
cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online",
413469
riscv_online_cpu, riscv_offline_cpu);
414-
#endif
415470
cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online",
416471
riscv_online_cpu_vec, NULL);
417472

0 commit comments

Comments
 (0)