24
24
DEFINE_PER_CPU (long , misaligned_access_speed ) = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN ;
25
25
DEFINE_PER_CPU (long , vector_misaligned_access ) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED ;
26
26
27
- #ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
27
+ static long unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN ;
28
+ static long unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN ;
29
+
28
30
static cpumask_t fast_misaligned_access ;
31
+
32
+ #ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
29
33
static int check_unaligned_access (void * param )
30
34
{
31
35
int cpu = smp_processor_id ();
@@ -130,6 +134,50 @@ static void __init check_unaligned_access_nonboot_cpu(void *param)
130
134
check_unaligned_access (pages [cpu ]);
131
135
}
132
136
137
+ /* Measure unaligned access speed on all CPUs present at boot in parallel. */
138
+ static void __init check_unaligned_access_speed_all_cpus (void )
139
+ {
140
+ unsigned int cpu ;
141
+ unsigned int cpu_count = num_possible_cpus ();
142
+ struct page * * bufs = kcalloc (cpu_count , sizeof (* bufs ), GFP_KERNEL );
143
+
144
+ if (!bufs ) {
145
+ pr_warn ("Allocation failure, not measuring misaligned performance\n" );
146
+ return ;
147
+ }
148
+
149
+ /*
150
+ * Allocate separate buffers for each CPU so there's no fighting over
151
+ * cache lines.
152
+ */
153
+ for_each_cpu (cpu , cpu_online_mask ) {
154
+ bufs [cpu ] = alloc_pages (GFP_KERNEL , MISALIGNED_BUFFER_ORDER );
155
+ if (!bufs [cpu ]) {
156
+ pr_warn ("Allocation failure, not measuring misaligned performance\n" );
157
+ goto out ;
158
+ }
159
+ }
160
+
161
+ /* Check everybody except 0, who stays behind to tend jiffies. */
162
+ on_each_cpu (check_unaligned_access_nonboot_cpu , bufs , 1 );
163
+
164
+ /* Check core 0. */
165
+ smp_call_on_cpu (0 , check_unaligned_access , bufs [0 ], true);
166
+
167
+ out :
168
+ for_each_cpu (cpu , cpu_online_mask ) {
169
+ if (bufs [cpu ])
170
+ __free_pages (bufs [cpu ], MISALIGNED_BUFFER_ORDER );
171
+ }
172
+
173
+ kfree (bufs );
174
+ }
175
+ #else /* CONFIG_RISCV_PROBE_UNALIGNED_ACCESS */
176
+ static void __init check_unaligned_access_speed_all_cpus (void )
177
+ {
178
+ }
179
+ #endif
180
+
133
181
DEFINE_STATIC_KEY_FALSE (fast_unaligned_access_speed_key );
134
182
135
183
static void modify_unaligned_access_branches (cpumask_t * mask , int weight )
@@ -188,21 +236,29 @@ arch_initcall_sync(lock_and_set_unaligned_access_static_branch);
188
236
189
237
static int riscv_online_cpu (unsigned int cpu )
190
238
{
191
- static struct page * buf ;
192
-
193
239
/* We are already set since the last check */
194
- if (per_cpu (misaligned_access_speed , cpu ) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN )
240
+ if (per_cpu (misaligned_access_speed , cpu ) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN ) {
241
+ goto exit ;
242
+ } else if (unaligned_scalar_speed_param != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN ) {
243
+ per_cpu (misaligned_access_speed , cpu ) = unaligned_scalar_speed_param ;
195
244
goto exit ;
196
-
197
- check_unaligned_access_emulated (NULL );
198
- buf = alloc_pages (GFP_KERNEL , MISALIGNED_BUFFER_ORDER );
199
- if (!buf ) {
200
- pr_warn ("Allocation failure, not measuring misaligned performance\n" );
201
- return - ENOMEM ;
202
245
}
203
246
204
- check_unaligned_access (buf );
205
- __free_pages (buf , MISALIGNED_BUFFER_ORDER );
247
+ #ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
248
+ {
249
+ static struct page * buf ;
250
+
251
+ check_unaligned_access_emulated (NULL );
252
+ buf = alloc_pages (GFP_KERNEL , MISALIGNED_BUFFER_ORDER );
253
+ if (!buf ) {
254
+ pr_warn ("Allocation failure, not measuring misaligned performance\n" );
255
+ return - ENOMEM ;
256
+ }
257
+
258
+ check_unaligned_access (buf );
259
+ __free_pages (buf , MISALIGNED_BUFFER_ORDER );
260
+ }
261
+ #endif
206
262
207
263
exit :
208
264
set_unaligned_access_static_branches ();
@@ -217,50 +273,6 @@ static int riscv_offline_cpu(unsigned int cpu)
217
273
return 0 ;
218
274
}
219
275
220
- /* Measure unaligned access speed on all CPUs present at boot in parallel. */
221
- static void __init check_unaligned_access_speed_all_cpus (void )
222
- {
223
- unsigned int cpu ;
224
- unsigned int cpu_count = num_possible_cpus ();
225
- struct page * * bufs = kcalloc (cpu_count , sizeof (* bufs ), GFP_KERNEL );
226
-
227
- if (!bufs ) {
228
- pr_warn ("Allocation failure, not measuring misaligned performance\n" );
229
- return ;
230
- }
231
-
232
- /*
233
- * Allocate separate buffers for each CPU so there's no fighting over
234
- * cache lines.
235
- */
236
- for_each_cpu (cpu , cpu_online_mask ) {
237
- bufs [cpu ] = alloc_pages (GFP_KERNEL , MISALIGNED_BUFFER_ORDER );
238
- if (!bufs [cpu ]) {
239
- pr_warn ("Allocation failure, not measuring misaligned performance\n" );
240
- goto out ;
241
- }
242
- }
243
-
244
- /* Check everybody except 0, who stays behind to tend jiffies. */
245
- on_each_cpu (check_unaligned_access_nonboot_cpu , bufs , 1 );
246
-
247
- /* Check core 0. */
248
- smp_call_on_cpu (0 , check_unaligned_access , bufs [0 ], true);
249
-
250
- out :
251
- for_each_cpu (cpu , cpu_online_mask ) {
252
- if (bufs [cpu ])
253
- __free_pages (bufs [cpu ], MISALIGNED_BUFFER_ORDER );
254
- }
255
-
256
- kfree (bufs );
257
- }
258
- #else /* CONFIG_RISCV_PROBE_UNALIGNED_ACCESS */
259
- static void __init check_unaligned_access_speed_all_cpus (void )
260
- {
261
- }
262
- #endif
263
-
264
276
#ifdef CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS
265
277
static void check_vector_unaligned_access (struct work_struct * work __always_unused )
266
278
{
@@ -372,8 +384,8 @@ static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __alway
372
384
373
385
static int riscv_online_cpu_vec (unsigned int cpu )
374
386
{
375
- if (! has_vector () ) {
376
- per_cpu (vector_misaligned_access , cpu ) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED ;
387
+ if (unaligned_vector_speed_param != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN ) {
388
+ per_cpu (vector_misaligned_access , cpu ) = unaligned_vector_speed_param ;
377
389
return 0 ;
378
390
}
379
391
@@ -388,30 +400,73 @@ static int riscv_online_cpu_vec(unsigned int cpu)
388
400
return 0 ;
389
401
}
390
402
403
+ static const char * const speed_str [] __initconst = { NULL , NULL , "slow" , "fast" , "unsupported" };
404
+
405
+ static int __init set_unaligned_scalar_speed_param (char * str )
406
+ {
407
+ if (!strcmp (str , speed_str [RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW ]))
408
+ unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW ;
409
+ else if (!strcmp (str , speed_str [RISCV_HWPROBE_MISALIGNED_SCALAR_FAST ]))
410
+ unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_FAST ;
411
+ else if (!strcmp (str , speed_str [RISCV_HWPROBE_MISALIGNED_SCALAR_UNSUPPORTED ]))
412
+ unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_UNSUPPORTED ;
413
+ else
414
+ return - EINVAL ;
415
+
416
+ return 1 ;
417
+ }
418
+ __setup ("unaligned_scalar_speed=" , set_unaligned_scalar_speed_param );
419
+
420
+ static int __init set_unaligned_vector_speed_param (char * str )
421
+ {
422
+ if (!strcmp (str , speed_str [RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW ]))
423
+ unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW ;
424
+ else if (!strcmp (str , speed_str [RISCV_HWPROBE_MISALIGNED_VECTOR_FAST ]))
425
+ unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_FAST ;
426
+ else if (!strcmp (str , speed_str [RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED ]))
427
+ unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED ;
428
+ else
429
+ return - EINVAL ;
430
+
431
+ return 1 ;
432
+ }
433
+ __setup ("unaligned_vector_speed=" , set_unaligned_vector_speed_param );
434
+
391
435
static int __init check_unaligned_access_all_cpus (void )
392
436
{
393
437
int cpu ;
394
438
395
- if (!check_unaligned_access_emulated_all_cpus ())
439
+ if (unaligned_scalar_speed_param == RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN &&
440
+ !check_unaligned_access_emulated_all_cpus ()) {
396
441
check_unaligned_access_speed_all_cpus ();
397
-
398
- if (!has_vector ()) {
442
+ } else {
443
+ pr_info ("scalar unaligned access speed set to '%s' by command line\n" ,
444
+ speed_str [unaligned_scalar_speed_param ]);
399
445
for_each_online_cpu (cpu )
400
- per_cpu (vector_misaligned_access , cpu ) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED ;
401
- } else if (!check_vector_unaligned_access_emulated_all_cpus () &&
402
- IS_ENABLED (CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS )) {
446
+ per_cpu (misaligned_access_speed , cpu ) = unaligned_scalar_speed_param ;
447
+ }
448
+
449
+ if (!has_vector ())
450
+ unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED ;
451
+
452
+ if (unaligned_vector_speed_param == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN &&
453
+ !check_vector_unaligned_access_emulated_all_cpus () &&
454
+ IS_ENABLED (CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS )) {
403
455
kthread_run (vec_check_unaligned_access_speed_all_cpus ,
404
456
NULL , "vec_check_unaligned_access_speed_all_cpus" );
457
+ } else {
458
+ pr_info ("vector unaligned access speed set to '%s' by command line\n" ,
459
+ speed_str [unaligned_vector_speed_param ]);
460
+ for_each_online_cpu (cpu )
461
+ per_cpu (vector_misaligned_access , cpu ) = unaligned_vector_speed_param ;
405
462
}
406
463
407
464
/*
408
465
* Setup hotplug callbacks for any new CPUs that come online or go
409
466
* offline.
410
467
*/
411
- #ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
412
468
cpuhp_setup_state_nocalls (CPUHP_AP_ONLINE_DYN , "riscv:online" ,
413
469
riscv_online_cpu , riscv_offline_cpu );
414
- #endif
415
470
cpuhp_setup_state_nocalls (CPUHP_AP_ONLINE_DYN , "riscv:online" ,
416
471
riscv_online_cpu_vec , NULL );
417
472
0 commit comments