Skip to content

Commit ea6b8e7

Browse files
arndbzx2c4
arndb
authored and
zx2c4
committed
compat: siphash: use _unaligned version by default
On ARM v6 and later, we define CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS because the ordinary load/store instructions (ldr, ldrh, ldrb) can tolerate any misalignment of the memory address. However, load/store double and load/store multiple instructions (ldrd, ldm) may still only be used on memory addresses that are 32-bit aligned, and so we have to use the CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS macro with care, or we may end up with a severe performance hit due to alignment traps that require fixups by the kernel. Testing shows that this currently happens with clang-13 but not gcc-11. In theory, any compiler version can produce this bug or other problems, as we are dealing with undefined behavior in C99 even on architectures that support this in hardware, see also https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100363. Fortunately, the get_unaligned() accessors do the right thing: when building for ARMv6 or later, the compiler will emit unaligned accesses using the ordinary load/store instructions (but avoid the ones that require 32-bit alignment). When building for older ARM, those accessors will emit the appropriate sequence of ldrb/mov/orr instructions. And on architectures that can truly tolerate any kind of misalignment, the get_unaligned() accessors resolve to the leXX_to_cpup accessors that operate on aligned addresses. Since the compiler will in fact emit ldrd or ldm instructions when building this code for ARM v6 or later, the solution is to use the unaligned accessors unconditionally on architectures where this is known to be fast. The _aligned version of the hash function is however still needed to get the best performance on architectures that cannot do any unaligned access in hardware. This new version avoids the undefined behavior and should produce the fastest hash on all architectures we support. Reported-by: Ard Biesheuvel <ard.biesheuvel@linaro.org> Signed-off-by: Arnd Bergmann <arnd@arndb.de> Reviewed-by: Jason A. Donenfeld <Jason@zx2c4.com> Acked-by: Ard Biesheuvel <ardb@kernel.org> Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
1 parent 5325bc8 commit ea6b8e7

File tree

2 files changed

+28
-34
lines changed

2 files changed

+28
-34
lines changed

‎src/compat/siphash/include/linux/siphash.h

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,7 @@ typedef struct {
2222
} siphash_key_t;
2323

2424
u64 __siphash_aligned(const void *data, size_t len, const siphash_key_t *key);
25-
#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
2625
u64 __siphash_unaligned(const void *data, size_t len, const siphash_key_t *key);
27-
#endif
2826

2927
u64 siphash_1u64(const u64 a, const siphash_key_t *key);
3028
u64 siphash_2u64(const u64 a, const u64 b, const siphash_key_t *key);
@@ -77,10 +75,9 @@ static inline u64 ___siphash_aligned(const __le64 *data, size_t len,
7775
static inline u64 siphash(const void *data, size_t len,
7876
const siphash_key_t *key)
7977
{
80-
#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
81-
if (!IS_ALIGNED((unsigned long)data, SIPHASH_ALIGNMENT))
78+
if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ||
79+
!IS_ALIGNED((unsigned long)data, SIPHASH_ALIGNMENT))
8280
return __siphash_unaligned(data, len, key);
83-
#endif
8481
return ___siphash_aligned(data, len, key);
8582
}
8683

@@ -91,10 +88,8 @@ typedef struct {
9188

9289
u32 __hsiphash_aligned(const void *data, size_t len,
9390
const hsiphash_key_t *key);
94-
#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
9591
u32 __hsiphash_unaligned(const void *data, size_t len,
9692
const hsiphash_key_t *key);
97-
#endif
9893

9994
u32 hsiphash_1u32(const u32 a, const hsiphash_key_t *key);
10095
u32 hsiphash_2u32(const u32 a, const u32 b, const hsiphash_key_t *key);
@@ -130,10 +125,9 @@ static inline u32 ___hsiphash_aligned(const __le32 *data, size_t len,
130125
static inline u32 hsiphash(const void *data, size_t len,
131126
const hsiphash_key_t *key)
132127
{
133-
#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
134-
if (!IS_ALIGNED((unsigned long)data, HSIPHASH_ALIGNMENT))
128+
if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ||
129+
!IS_ALIGNED((unsigned long)data, HSIPHASH_ALIGNMENT))
135130
return __hsiphash_unaligned(data, len, key);
136-
#endif
137131
return ___hsiphash_aligned(data, len, key);
138132
}
139133

‎src/compat/siphash/siphash.c

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
SIPROUND; \
5858
return (v0 ^ v1) ^ (v2 ^ v3);
5959

60+
#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
6061
u64 __siphash_aligned(const void *data, size_t len, const siphash_key_t *key)
6162
{
6263
const u8 *end = data + len - (len % sizeof(u64));
@@ -76,19 +77,19 @@ u64 __siphash_aligned(const void *data, size_t len, const siphash_key_t *key)
7677
bytemask_from_count(left)));
7778
#else
7879
switch (left) {
79-
case 7: b |= ((u64)end[6]) << 48;
80-
case 6: b |= ((u64)end[5]) << 40;
81-
case 5: b |= ((u64)end[4]) << 32;
80+
case 7: b |= ((u64)end[6]) << 48; fallthrough;
81+
case 6: b |= ((u64)end[5]) << 40; fallthrough;
82+
case 5: b |= ((u64)end[4]) << 32; fallthrough;
8283
case 4: b |= le32_to_cpup(data); break;
83-
case 3: b |= ((u64)end[2]) << 16;
84+
case 3: b |= ((u64)end[2]) << 16; fallthrough;
8485
case 2: b |= le16_to_cpup(data); break;
8586
case 1: b |= end[0];
8687
}
8788
#endif
8889
POSTAMBLE
8990
}
91+
#endif
9092

91-
#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
9293
u64 __siphash_unaligned(const void *data, size_t len, const siphash_key_t *key)
9394
{
9495
const u8 *end = data + len - (len % sizeof(u64));
@@ -108,18 +109,17 @@ u64 __siphash_unaligned(const void *data, size_t len, const siphash_key_t *key)
108109
bytemask_from_count(left)));
109110
#else
110111
switch (left) {
111-
case 7: b |= ((u64)end[6]) << 48;
112-
case 6: b |= ((u64)end[5]) << 40;
113-
case 5: b |= ((u64)end[4]) << 32;
112+
case 7: b |= ((u64)end[6]) << 48; fallthrough;
113+
case 6: b |= ((u64)end[5]) << 40; fallthrough;
114+
case 5: b |= ((u64)end[4]) << 32; fallthrough;
114115
case 4: b |= get_unaligned_le32(end); break;
115-
case 3: b |= ((u64)end[2]) << 16;
116+
case 3: b |= ((u64)end[2]) << 16; fallthrough;
116117
case 2: b |= get_unaligned_le16(end); break;
117118
case 1: b |= end[0];
118119
}
119120
#endif
120121
POSTAMBLE
121122
}
122-
#endif
123123

124124
/**
125125
* siphash_1u64 - compute 64-bit siphash PRF value of a u64
@@ -250,6 +250,7 @@ u64 siphash_3u32(const u32 first, const u32 second, const u32 third,
250250
HSIPROUND; \
251251
return (v0 ^ v1) ^ (v2 ^ v3);
252252

253+
#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
253254
u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key)
254255
{
255256
const u8 *end = data + len - (len % sizeof(u64));
@@ -268,19 +269,19 @@ u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key)
268269
bytemask_from_count(left)));
269270
#else
270271
switch (left) {
271-
case 7: b |= ((u64)end[6]) << 48;
272-
case 6: b |= ((u64)end[5]) << 40;
273-
case 5: b |= ((u64)end[4]) << 32;
272+
case 7: b |= ((u64)end[6]) << 48; fallthrough;
273+
case 6: b |= ((u64)end[5]) << 40; fallthrough;
274+
case 5: b |= ((u64)end[4]) << 32; fallthrough;
274275
case 4: b |= le32_to_cpup(data); break;
275-
case 3: b |= ((u64)end[2]) << 16;
276+
case 3: b |= ((u64)end[2]) << 16; fallthrough;
276277
case 2: b |= le16_to_cpup(data); break;
277278
case 1: b |= end[0];
278279
}
279280
#endif
280281
HPOSTAMBLE
281282
}
283+
#endif
282284

283-
#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
284285
u32 __hsiphash_unaligned(const void *data, size_t len,
285286
const hsiphash_key_t *key)
286287
{
@@ -300,18 +301,17 @@ u32 __hsiphash_unaligned(const void *data, size_t len,
300301
bytemask_from_count(left)));
301302
#else
302303
switch (left) {
303-
case 7: b |= ((u64)end[6]) << 48;
304-
case 6: b |= ((u64)end[5]) << 40;
305-
case 5: b |= ((u64)end[4]) << 32;
304+
case 7: b |= ((u64)end[6]) << 48; fallthrough;
305+
case 6: b |= ((u64)end[5]) << 40; fallthrough;
306+
case 5: b |= ((u64)end[4]) << 32; fallthrough;
306307
case 4: b |= get_unaligned_le32(end); break;
307-
case 3: b |= ((u64)end[2]) << 16;
308+
case 3: b |= ((u64)end[2]) << 16; fallthrough;
308309
case 2: b |= get_unaligned_le16(end); break;
309310
case 1: b |= end[0];
310311
}
311312
#endif
312313
HPOSTAMBLE
313314
}
314-
#endif
315315

316316
/**
317317
* hsiphash_1u32 - compute 64-bit hsiphash PRF value of a u32
@@ -412,6 +412,7 @@ u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third,
412412
HSIPROUND; \
413413
return v1 ^ v3;
414414

415+
#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
415416
u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key)
416417
{
417418
const u8 *end = data + len - (len % sizeof(u32));
@@ -425,14 +426,14 @@ u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key)
425426
v0 ^= m;
426427
}
427428
switch (left) {
428-
case 3: b |= ((u32)end[2]) << 16;
429+
case 3: b |= ((u32)end[2]) << 16; fallthrough;
429430
case 2: b |= le16_to_cpup(data); break;
430431
case 1: b |= end[0];
431432
}
432433
HPOSTAMBLE
433434
}
435+
#endif
434436

435-
#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
436437
u32 __hsiphash_unaligned(const void *data, size_t len,
437438
const hsiphash_key_t *key)
438439
{
@@ -447,13 +448,12 @@ u32 __hsiphash_unaligned(const void *data, size_t len,
447448
v0 ^= m;
448449
}
449450
switch (left) {
450-
case 3: b |= ((u32)end[2]) << 16;
451+
case 3: b |= ((u32)end[2]) << 16; fallthrough;
451452
case 2: b |= get_unaligned_le16(end); break;
452453
case 1: b |= end[0];
453454
}
454455
HPOSTAMBLE
455456
}
456-
#endif
457457

458458
/**
459459
* hsiphash_1u32 - compute 32-bit hsiphash PRF value of a u32

0 commit comments

Comments
 (0)