Skip to content

Commit c545cd3

Browse files
committed
Merge tag 'x86-mm-2025-01-31' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull x86 mm updates from Ingo Molnar: - The biggest changes are the TLB flushing scalability optimizations, to update the mm_cpumask lazily and related changes. This feature has both a track record and a continued risk of performance regressions, so it was already delayed by a cycle - but it's all 100% perfect now™ (Rik van Riel) - Also miscellaneous fixes and cleanups. (Gautam Somani, Kirill Shutemov, Sebastian Andrzej Siewior) * tag 'x86-mm-2025-01-31' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: x86/mm: Remove unnecessary include of <linux/extable.h> x86/mtrr: Rename mtrr_overwrite_state() to guest_force_mtrr_state() x86/mm/selftests: Fix typo in lam.c x86/mm/tlb: Only trim the mm_cpumask once a second x86/mm/tlb: Also remove local CPU from mm_cpumask if stale x86/mm/tlb: Add tracepoint for TLB flush IPI to stale CPU x86/mm/tlb: Update mm_cpumask lazily
2 parents 626d1a1 + aa135d1 commit c545cd3

File tree

8 files changed

+57
-20
lines changed

8 files changed

+57
-20
lines changed

arch/x86/include/asm/mmu.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ typedef struct {
3737
*/
3838
atomic64_t tlb_gen;
3939

40+
unsigned long next_trim_cpumask;
41+
4042
#ifdef CONFIG_MODIFY_LDT_SYSCALL
4143
struct rw_semaphore ldt_usr_sem;
4244
struct ldt_struct *ldt;

arch/x86/include/asm/mmu_context.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ static inline int init_new_context(struct task_struct *tsk,
151151

152152
mm->context.ctx_id = atomic64_inc_return(&last_mm_ctx_id);
153153
atomic64_set(&mm->context.tlb_gen, 0);
154+
mm->context.next_trim_cpumask = jiffies + HZ;
154155

155156
#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
156157
if (cpu_feature_enabled(X86_FEATURE_OSPKE)) {

arch/x86/include/asm/tlbflush.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,7 @@ struct flush_tlb_info {
222222
unsigned int initiating_cpu;
223223
u8 stride_shift;
224224
u8 freed_tables;
225+
u8 trim_cpumask;
225226
};
226227

227228
void flush_tlb_local(void);

arch/x86/kernel/alternative.c

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1854,11 +1854,18 @@ static inline temp_mm_state_t use_temporary_mm(struct mm_struct *mm)
18541854
return temp_state;
18551855
}
18561856

1857+
__ro_after_init struct mm_struct *poking_mm;
1858+
__ro_after_init unsigned long poking_addr;
1859+
18571860
static inline void unuse_temporary_mm(temp_mm_state_t prev_state)
18581861
{
18591862
lockdep_assert_irqs_disabled();
1863+
18601864
switch_mm_irqs_off(NULL, prev_state.mm, current);
18611865

1866+
/* Clear the cpumask, to indicate no TLB flushing is needed anywhere */
1867+
cpumask_clear_cpu(raw_smp_processor_id(), mm_cpumask(poking_mm));
1868+
18621869
/*
18631870
* Restore the breakpoints if they were disabled before the temporary mm
18641871
* was loaded.
@@ -1867,9 +1874,6 @@ static inline void unuse_temporary_mm(temp_mm_state_t prev_state)
18671874
hw_breakpoint_restore();
18681875
}
18691876

1870-
__ro_after_init struct mm_struct *poking_mm;
1871-
__ro_after_init unsigned long poking_addr;
1872-
18731877
static void text_poke_memcpy(void *dst, const void *src, size_t len)
18741878
{
18751879
memcpy(dst, src, len);

arch/x86/mm/fault.c

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
#include <linux/sched.h> /* test_thread_flag(), ... */
88
#include <linux/sched/task_stack.h> /* task_stack_*(), ... */
99
#include <linux/kdebug.h> /* oops_begin/end, ... */
10-
#include <linux/extable.h> /* search_exception_tables */
1110
#include <linux/memblock.h> /* max_low_pfn */
1211
#include <linux/kfence.h> /* kfence_handle_page_fault */
1312
#include <linux/kprobes.h> /* NOKPROBE_SYMBOL, ... */

arch/x86/mm/tlb.c

Lines changed: 44 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -607,18 +607,15 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next,
607607
cond_mitigation(tsk);
608608

609609
/*
610-
* Stop remote flushes for the previous mm.
611-
* Skip kernel threads; we never send init_mm TLB flushing IPIs,
612-
* but the bitmap manipulation can cause cache line contention.
610+
* Leave this CPU in prev's mm_cpumask. Atomic writes to
611+
* mm_cpumask can be expensive under contention. The CPU
612+
* will be removed lazily at TLB flush time.
613613
*/
614-
if (prev != &init_mm) {
615-
VM_WARN_ON_ONCE(!cpumask_test_cpu(cpu,
616-
mm_cpumask(prev)));
617-
cpumask_clear_cpu(cpu, mm_cpumask(prev));
618-
}
614+
VM_WARN_ON_ONCE(prev != &init_mm && !cpumask_test_cpu(cpu,
615+
mm_cpumask(prev)));
619616

620617
/* Start receiving IPIs and then read tlb_gen (and LAM below) */
621-
if (next != &init_mm)
618+
if (next != &init_mm && !cpumask_test_cpu(cpu, mm_cpumask(next)))
622619
cpumask_set_cpu(cpu, mm_cpumask(next));
623620
next_tlb_gen = atomic64_read(&next->context.tlb_gen);
624621

@@ -760,10 +757,13 @@ static void flush_tlb_func(void *info)
760757
if (!local) {
761758
inc_irq_stat(irq_tlb_count);
762759
count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
760+
}
763761

764-
/* Can only happen on remote CPUs */
765-
if (f->mm && f->mm != loaded_mm)
766-
return;
762+
/* The CPU was left in the mm_cpumask of the target mm. Clear it. */
763+
if (f->mm && f->mm != loaded_mm) {
764+
cpumask_clear_cpu(raw_smp_processor_id(), mm_cpumask(f->mm));
765+
trace_tlb_flush(TLB_REMOTE_WRONG_CPU, 0);
766+
return;
767767
}
768768

769769
if (unlikely(loaded_mm == &init_mm))
@@ -893,9 +893,36 @@ static void flush_tlb_func(void *info)
893893
nr_invalidate);
894894
}
895895

896-
static bool tlb_is_not_lazy(int cpu, void *data)
896+
static bool should_flush_tlb(int cpu, void *data)
897+
{
898+
struct flush_tlb_info *info = data;
899+
900+
/* Lazy TLB will get flushed at the next context switch. */
901+
if (per_cpu(cpu_tlbstate_shared.is_lazy, cpu))
902+
return false;
903+
904+
/* No mm means kernel memory flush. */
905+
if (!info->mm)
906+
return true;
907+
908+
/* The target mm is loaded, and the CPU is not lazy. */
909+
if (per_cpu(cpu_tlbstate.loaded_mm, cpu) == info->mm)
910+
return true;
911+
912+
/* In cpumask, but not the loaded mm? Periodically remove by flushing. */
913+
if (info->trim_cpumask)
914+
return true;
915+
916+
return false;
917+
}
918+
919+
static bool should_trim_cpumask(struct mm_struct *mm)
897920
{
898-
return !per_cpu(cpu_tlbstate_shared.is_lazy, cpu);
921+
if (time_after(jiffies, READ_ONCE(mm->context.next_trim_cpumask))) {
922+
WRITE_ONCE(mm->context.next_trim_cpumask, jiffies + HZ);
923+
return true;
924+
}
925+
return false;
899926
}
900927

901928
DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state_shared, cpu_tlbstate_shared);
@@ -929,7 +956,7 @@ STATIC_NOPV void native_flush_tlb_multi(const struct cpumask *cpumask,
929956
if (info->freed_tables)
930957
on_each_cpu_mask(cpumask, flush_tlb_func, (void *)info, true);
931958
else
932-
on_each_cpu_cond_mask(tlb_is_not_lazy, flush_tlb_func,
959+
on_each_cpu_cond_mask(should_flush_tlb, flush_tlb_func,
933960
(void *)info, 1, cpumask);
934961
}
935962

@@ -980,6 +1007,7 @@ static struct flush_tlb_info *get_flush_tlb_info(struct mm_struct *mm,
9801007
info->freed_tables = freed_tables;
9811008
info->new_tlb_gen = new_tlb_gen;
9821009
info->initiating_cpu = smp_processor_id();
1010+
info->trim_cpumask = 0;
9831011

9841012
return info;
9851013
}
@@ -1022,6 +1050,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
10221050
* flush_tlb_func_local() directly in this case.
10231051
*/
10241052
if (cpumask_any_but(mm_cpumask(mm), cpu) < nr_cpu_ids) {
1053+
info->trim_cpumask = should_trim_cpumask(mm);
10251054
flush_tlb_multi(mm_cpumask(mm), info);
10261055
} else if (mm == this_cpu_read(cpu_tlbstate.loaded_mm)) {
10271056
lockdep_assert_irqs_enabled();

include/linux/mm_types.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1406,6 +1406,7 @@ enum tlb_flush_reason {
14061406
TLB_LOCAL_SHOOTDOWN,
14071407
TLB_LOCAL_MM_SHOOTDOWN,
14081408
TLB_REMOTE_SEND_IPI,
1409+
TLB_REMOTE_WRONG_CPU,
14091410
NR_TLB_FLUSH_REASONS,
14101411
};
14111412

tools/testing/selftests/x86/lam.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,7 @@ static uint64_t set_metadata(uint64_t src, unsigned long lam)
237237
* both pointers should point to the same address.
238238
*
239239
* @return:
240-
* 0: value on the pointer with metadate and value on original are same
240+
* 0: value on the pointer with metadata and value on original are same
241241
* 1: not same.
242242
*/
243243
static int handle_lam_test(void *src, unsigned int lam)

0 commit comments

Comments
 (0)