Skip to content

Commit 07f2c04

Browse files
palmer-dabbeltpalmer-dabbelt
committed
Merge patch series "riscv: mm: Extend mappable memory up to hint address"
Charlie Jenkins <charlie@rivosinc.com> says: On riscv, mmap currently returns an address from the largest address space that can fit entirely inside of the hint address. This makes it such that the hint address is almost never returned. This patch raises the mappable area up to and including the hint address. This allows mmap to often return the hint address, which allows a performance improvement over searching for a valid address as well as making the behavior more similar to other architectures. Note that a previous patch introduced stronger semantics compared to other architectures for riscv mmap. On riscv, mmap will not use bits in the upper bits of the virtual address depending on the hint address. On other architectures, a random address is returned in the address space requested. On all architectures the hint address will be returned if it is available. This allows riscv applications to configure how many bits in the virtual address should be left empty. This has the two benefits of being able to request address spaces that are smaller than the default and doesn't require the application to know the page table layout of riscv. * b4-shazam-merge: docs: riscv: Define behavior of mmap selftests: riscv: Generalize mm selftests riscv: mm: Use hint address in mmap if available Link: https://lore.kernel.org/r/20240130-use_mmap_hint_address-v3-0-8a655cfa8bcb@rivosinc.com Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
2 parents 099dbac + 371a3c2 commit 07f2c04

File tree

5 files changed

+83
-113
lines changed

5 files changed

+83
-113
lines changed

‎Documentation/arch/riscv/vm-layout.rst

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -144,14 +144,8 @@ passing 0 into the hint address parameter of mmap. On CPUs with an address space
144144
smaller than sv48, the CPU maximum supported address space will be the default.
145145

146146
Software can "opt-in" to receiving VAs from another VA space by providing
147-
a hint address to mmap. A hint address passed to mmap will cause the largest
148-
address space that fits entirely into the hint to be used, unless there is no
149-
space left in the address space. If there is no space available in the requested
150-
address space, an address in the next smallest available address space will be
151-
returned.
152-
153-
For example, in order to obtain 48-bit VA space, a hint address greater than
154-
:code:`1 << 47` must be provided. Note that this is 47 due to sv48 userspace
155-
ending at :code:`1 << 47` and the addresses beyond this are reserved for the
156-
kernel. Similarly, to obtain 57-bit VA space addresses, a hint address greater
157-
than or equal to :code:`1 << 56` must be provided.
147+
a hint address to mmap. When a hint address is passed to mmap, the returned
148+
address will never use more bits than the hint address. For example, if a hint
149+
address of `1 << 40` is passed to mmap, a valid returned address will never use
150+
bits 41 through 63. If no mappable addresses are available in that range, mmap
151+
will return `MAP_FAILED`.

‎arch/riscv/include/asm/processor.h

Lines changed: 11 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -14,22 +14,16 @@
1414

1515
#include <asm/ptrace.h>
1616

17-
#ifdef CONFIG_64BIT
18-
#define DEFAULT_MAP_WINDOW (UL(1) << (MMAP_VA_BITS - 1))
19-
#define STACK_TOP_MAX TASK_SIZE
20-
2117
#define arch_get_mmap_end(addr, len, flags) \
2218
({ \
2319
unsigned long mmap_end; \
2420
typeof(addr) _addr = (addr); \
25-
if ((_addr) == 0 || (IS_ENABLED(CONFIG_COMPAT) && is_compat_task())) \
21+
if ((_addr) == 0 || \
22+
(IS_ENABLED(CONFIG_COMPAT) && is_compat_task()) || \
23+
((_addr + len) > BIT(VA_BITS - 1))) \
2624
mmap_end = STACK_TOP_MAX; \
27-
else if ((_addr) >= VA_USER_SV57) \
28-
mmap_end = STACK_TOP_MAX; \
29-
else if ((((_addr) >= VA_USER_SV48)) && (VA_BITS >= VA_BITS_SV48)) \
30-
mmap_end = VA_USER_SV48; \
3125
else \
32-
mmap_end = VA_USER_SV39; \
26+
mmap_end = (_addr + len); \
3327
mmap_end; \
3428
})
3529

@@ -39,17 +33,18 @@
3933
typeof(addr) _addr = (addr); \
4034
typeof(base) _base = (base); \
4135
unsigned long rnd_gap = DEFAULT_MAP_WINDOW - (_base); \
42-
if ((_addr) == 0 || (IS_ENABLED(CONFIG_COMPAT) && is_compat_task())) \
36+
if ((_addr) == 0 || \
37+
(IS_ENABLED(CONFIG_COMPAT) && is_compat_task()) || \
38+
((_addr + len) > BIT(VA_BITS - 1))) \
4339
mmap_base = (_base); \
44-
else if (((_addr) >= VA_USER_SV57) && (VA_BITS >= VA_BITS_SV57)) \
45-
mmap_base = VA_USER_SV57 - rnd_gap; \
46-
else if ((((_addr) >= VA_USER_SV48)) && (VA_BITS >= VA_BITS_SV48)) \
47-
mmap_base = VA_USER_SV48 - rnd_gap; \
4840
else \
49-
mmap_base = VA_USER_SV39 - rnd_gap; \
41+
mmap_base = (_addr + len) - rnd_gap; \
5042
mmap_base; \
5143
})
5244

45+
#ifdef CONFIG_64BIT
46+
#define DEFAULT_MAP_WINDOW (UL(1) << (MMAP_VA_BITS - 1))
47+
#define STACK_TOP_MAX TASK_SIZE_64
5348
#else
5449
#define DEFAULT_MAP_WINDOW TASK_SIZE
5550
#define STACK_TOP_MAX TASK_SIZE

‎tools/testing/selftests/riscv/mm/mmap_bottomup.c

Lines changed: 1 addition & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -6,30 +6,9 @@
66

77
TEST(infinite_rlimit)
88
{
9-
// Only works on 64 bit
10-
#if __riscv_xlen == 64
11-
struct addresses mmap_addresses;
12-
139
EXPECT_EQ(BOTTOM_UP, memory_layout());
1410

15-
do_mmaps(&mmap_addresses);
16-
17-
EXPECT_NE(MAP_FAILED, mmap_addresses.no_hint);
18-
EXPECT_NE(MAP_FAILED, mmap_addresses.on_37_addr);
19-
EXPECT_NE(MAP_FAILED, mmap_addresses.on_38_addr);
20-
EXPECT_NE(MAP_FAILED, mmap_addresses.on_46_addr);
21-
EXPECT_NE(MAP_FAILED, mmap_addresses.on_47_addr);
22-
EXPECT_NE(MAP_FAILED, mmap_addresses.on_55_addr);
23-
EXPECT_NE(MAP_FAILED, mmap_addresses.on_56_addr);
24-
25-
EXPECT_GT(1UL << 47, (unsigned long)mmap_addresses.no_hint);
26-
EXPECT_GT(1UL << 38, (unsigned long)mmap_addresses.on_37_addr);
27-
EXPECT_GT(1UL << 38, (unsigned long)mmap_addresses.on_38_addr);
28-
EXPECT_GT(1UL << 38, (unsigned long)mmap_addresses.on_46_addr);
29-
EXPECT_GT(1UL << 47, (unsigned long)mmap_addresses.on_47_addr);
30-
EXPECT_GT(1UL << 47, (unsigned long)mmap_addresses.on_55_addr);
31-
EXPECT_GT(1UL << 56, (unsigned long)mmap_addresses.on_56_addr);
32-
#endif
11+
TEST_MMAPS;
3312
}
3413

3514
TEST_HARNESS_MAIN

‎tools/testing/selftests/riscv/mm/mmap_default.c

Lines changed: 1 addition & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -6,30 +6,9 @@
66

77
TEST(default_rlimit)
88
{
9-
// Only works on 64 bit
10-
#if __riscv_xlen == 64
11-
struct addresses mmap_addresses;
12-
139
EXPECT_EQ(TOP_DOWN, memory_layout());
1410

15-
do_mmaps(&mmap_addresses);
16-
17-
EXPECT_NE(MAP_FAILED, mmap_addresses.no_hint);
18-
EXPECT_NE(MAP_FAILED, mmap_addresses.on_37_addr);
19-
EXPECT_NE(MAP_FAILED, mmap_addresses.on_38_addr);
20-
EXPECT_NE(MAP_FAILED, mmap_addresses.on_46_addr);
21-
EXPECT_NE(MAP_FAILED, mmap_addresses.on_47_addr);
22-
EXPECT_NE(MAP_FAILED, mmap_addresses.on_55_addr);
23-
EXPECT_NE(MAP_FAILED, mmap_addresses.on_56_addr);
24-
25-
EXPECT_GT(1UL << 47, (unsigned long)mmap_addresses.no_hint);
26-
EXPECT_GT(1UL << 38, (unsigned long)mmap_addresses.on_37_addr);
27-
EXPECT_GT(1UL << 38, (unsigned long)mmap_addresses.on_38_addr);
28-
EXPECT_GT(1UL << 38, (unsigned long)mmap_addresses.on_46_addr);
29-
EXPECT_GT(1UL << 47, (unsigned long)mmap_addresses.on_47_addr);
30-
EXPECT_GT(1UL << 47, (unsigned long)mmap_addresses.on_55_addr);
31-
EXPECT_GT(1UL << 56, (unsigned long)mmap_addresses.on_56_addr);
32-
#endif
11+
TEST_MMAPS;
3312
}
3413

3514
TEST_HARNESS_MAIN

‎tools/testing/selftests/riscv/mm/mmap_test.h

Lines changed: 65 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -4,63 +4,86 @@
44
#include <sys/mman.h>
55
#include <sys/resource.h>
66
#include <stddef.h>
7+
#include <strings.h>
8+
#include "../../kselftest_harness.h"
79

810
#define TOP_DOWN 0
911
#define BOTTOM_UP 1
1012

11-
struct addresses {
12-
int *no_hint;
13-
int *on_37_addr;
14-
int *on_38_addr;
15-
int *on_46_addr;
16-
int *on_47_addr;
17-
int *on_55_addr;
18-
int *on_56_addr;
13+
#if __riscv_xlen == 64
14+
uint64_t random_addresses[] = {
15+
0x19764f0d73b3a9f0, 0x016049584cecef59, 0x3580bdd3562f4acd,
16+
0x1164219f20b17da0, 0x07d97fcb40ff2373, 0x76ec528921272ee7,
17+
0x4dd48c38a3de3f70, 0x2e11415055f6997d, 0x14b43334ac476c02,
18+
0x375a60795aff19f6, 0x47f3051725b8ee1a, 0x4e697cf240494a9f,
19+
0x456b59b5c2f9e9d1, 0x101724379d63cb96, 0x7fe9ad31619528c1,
20+
0x2f417247c495c2ea, 0x329a5a5b82943a5e, 0x06d7a9d6adcd3827,
21+
0x327b0b9ee37f62d5, 0x17c7b1851dfd9b76, 0x006ebb6456ec2cd9,
22+
0x00836cd14146a134, 0x00e5c4dcde7126db, 0x004c29feadf75753,
23+
0x00d8b20149ed930c, 0x00d71574c269387a, 0x0006ebe4a82acb7a,
24+
0x0016135df51f471b, 0x00758bdb55455160, 0x00d0bdd949b13b32,
25+
0x00ecea01e7c5f54b, 0x00e37b071b9948b1, 0x0011fdd00ff57ab3,
26+
0x00e407294b52f5ea, 0x00567748c200ed20, 0x000d073084651046,
27+
0x00ac896f4365463c, 0x00eb0d49a0b26216, 0x0066a2564a982a31,
28+
0x002e0d20237784ae, 0x0000554ff8a77a76, 0x00006ce07a54c012,
29+
0x000009570516d799, 0x00000954ca15b84d, 0x0000684f0d453379,
30+
0x00002ae5816302b5, 0x0000042403fb54bf, 0x00004bad7392bf30,
31+
0x00003e73bfa4b5e3, 0x00005442c29978e0, 0x00002803f11286b6,
32+
0x000073875d745fc6, 0x00007cede9cb8240, 0x000027df84cc6a4f,
33+
0x00006d7e0e74242a, 0x00004afd0b836e02, 0x000047d0e837cd82,
34+
0x00003b42405efeda, 0x00001531bafa4c95, 0x00007172cae34ac4,
35+
};
36+
#else
37+
uint32_t random_addresses[] = {
38+
0x8dc302e0, 0x929ab1e0, 0xb47683ba, 0xea519c73, 0xa19f1c90, 0xc49ba213,
39+
0x8f57c625, 0xadfe5137, 0x874d4d95, 0xaa20f09d, 0xcf21ebfc, 0xda7737f1,
40+
0xcedf392a, 0x83026c14, 0xccedca52, 0xc6ccf826, 0xe0cd9415, 0x997472ca,
41+
0xa21a44c1, 0xe82196f5, 0xa23fd66b, 0xc28d5590, 0xd009cdce, 0xcf0be646,
42+
0x8fc8c7ff, 0xe2a85984, 0xa3d3236b, 0x89a0619d, 0xc03db924, 0xb5d4cc1b,
43+
0xb96ee04c, 0xd191da48, 0xb432a000, 0xaa2bebbc, 0xa2fcb289, 0xb0cca89b,
44+
0xb0c18d6a, 0x88f58deb, 0xa4d42d1c, 0xe4d74e86, 0x99902b09, 0x8f786d31,
45+
0xbec5e381, 0x9a727e65, 0xa9a65040, 0xa880d789, 0x8f1b335e, 0xfc821c1e,
46+
0x97e34be4, 0xbbef84ed, 0xf447d197, 0xfd7ceee2, 0xe632348d, 0xee4590f4,
47+
0x958992a5, 0xd57e05d6, 0xfd240970, 0xc5b0dcff, 0xd96da2c2, 0xa7ae041d,
1948
};
49+
#endif
2050

2151
// Only works on 64 bit
2252
#if __riscv_xlen == 64
23-
static inline void do_mmaps(struct addresses *mmap_addresses)
24-
{
25-
/*
26-
* Place all of the hint addresses on the boundaries of mmap
27-
* sv39, sv48, sv57
28-
* User addresses end at 1<<38, 1<<47, 1<<56 respectively
29-
*/
30-
void *on_37_bits = (void *)(1UL << 37);
31-
void *on_38_bits = (void *)(1UL << 38);
32-
void *on_46_bits = (void *)(1UL << 46);
33-
void *on_47_bits = (void *)(1UL << 47);
34-
void *on_55_bits = (void *)(1UL << 55);
35-
void *on_56_bits = (void *)(1UL << 56);
53+
#define PROT (PROT_READ | PROT_WRITE)
54+
#define FLAGS (MAP_PRIVATE | MAP_ANONYMOUS)
3655

37-
int prot = PROT_READ | PROT_WRITE;
38-
int flags = MAP_PRIVATE | MAP_ANONYMOUS;
56+
/* mmap must return a value that doesn't use more bits than the hint address. */
57+
static inline unsigned long get_max_value(unsigned long input)
58+
{
59+
unsigned long max_bit = (1UL << (((sizeof(unsigned long) * 8) - 1 -
60+
__builtin_clzl(input))));
3961

40-
mmap_addresses->no_hint =
41-
mmap(NULL, 5 * sizeof(int), prot, flags, 0, 0);
42-
mmap_addresses->on_37_addr =
43-
mmap(on_37_bits, 5 * sizeof(int), prot, flags, 0, 0);
44-
mmap_addresses->on_38_addr =
45-
mmap(on_38_bits, 5 * sizeof(int), prot, flags, 0, 0);
46-
mmap_addresses->on_46_addr =
47-
mmap(on_46_bits, 5 * sizeof(int), prot, flags, 0, 0);
48-
mmap_addresses->on_47_addr =
49-
mmap(on_47_bits, 5 * sizeof(int), prot, flags, 0, 0);
50-
mmap_addresses->on_55_addr =
51-
mmap(on_55_bits, 5 * sizeof(int), prot, flags, 0, 0);
52-
mmap_addresses->on_56_addr =
53-
mmap(on_56_bits, 5 * sizeof(int), prot, flags, 0, 0);
62+
return max_bit + (max_bit - 1);
5463
}
64+
65+
#define TEST_MMAPS \
66+
({ \
67+
void *mmap_addr; \
68+
for (int i = 0; i < ARRAY_SIZE(random_addresses); i++) { \
69+
mmap_addr = mmap((void *)random_addresses[i], \
70+
5 * sizeof(int), PROT, FLAGS, 0, 0); \
71+
EXPECT_NE(MAP_FAILED, mmap_addr); \
72+
EXPECT_GE((void *)get_max_value(random_addresses[i]), \
73+
mmap_addr); \
74+
mmap_addr = mmap((void *)random_addresses[i], \
75+
5 * sizeof(int), PROT, FLAGS, 0, 0); \
76+
EXPECT_NE(MAP_FAILED, mmap_addr); \
77+
EXPECT_GE((void *)get_max_value(random_addresses[i]), \
78+
mmap_addr); \
79+
} \
80+
})
5581
#endif /* __riscv_xlen == 64 */
5682

5783
static inline int memory_layout(void)
5884
{
59-
int prot = PROT_READ | PROT_WRITE;
60-
int flags = MAP_PRIVATE | MAP_ANONYMOUS;
61-
62-
void *value1 = mmap(NULL, sizeof(int), prot, flags, 0, 0);
63-
void *value2 = mmap(NULL, sizeof(int), prot, flags, 0, 0);
85+
void *value1 = mmap(NULL, sizeof(int), PROT, FLAGS, 0, 0);
86+
void *value2 = mmap(NULL, sizeof(int), PROT, FLAGS, 0, 0);
6487

6588
return value2 > value1;
6689
}

0 commit comments

Comments
 (0)