Skip to content

Commit 2360899

Browse files
torvaldstorvalds
committed
Merge tag 'locking-core-2025-03-22' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull locking updates from Ingo Molnar: "Locking primitives: - Micro-optimize percpu_{,try_}cmpxchg{64,128}_op() and {,try_}cmpxchg{64,128} on x86 (Uros Bizjak) - mutexes: extend debug checks in mutex_lock() (Yunhui Cui) - Misc cleanups (Uros Bizjak) Lockdep: - Fix might_fault() lockdep check of current->mm->mmap_lock (Peter Zijlstra) - Don't disable interrupts on RT in disable_irq_nosync_lockdep.*() (Sebastian Andrzej Siewior) - Disable KASAN instrumentation of lockdep.c (Waiman Long) - Add kasan_check_byte() check in lock_acquire() (Waiman Long) - Misc cleanups (Sebastian Andrzej Siewior) Rust runtime integration: - Use Pin for all LockClassKey usages (Mitchell Levy) - sync: Add accessor for the lock behind a given guard (Alice Ryhl) - sync: condvar: Add wait_interruptible_freezable() (Alice Ryhl) - sync: lock: Add an example for Guard:: Lock_ref() (Boqun Feng) Split-lock detection feature (x86): - Fix warning mode with disabled mitigation mode (Maksim Davydov) Locking events: - Add locking events for rtmutex slow paths (Waiman Long) - Add locking events for lockdep (Waiman Long)" * tag 'locking-core-2025-03-22' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: lockdep: Remove disable_irq_lockdep() lockdep: Don't disable interrupts on RT in disable_irq_nosync_lockdep.*() rust: lockdep: Use Pin for all LockClassKey usages rust: sync: condvar: Add wait_interruptible_freezable() rust: sync: lock: Add an example for Guard:: Lock_ref() rust: sync: Add accessor for the lock behind a given guard locking/lockdep: Add kasan_check_byte() check in lock_acquire() locking/lockdep: Disable KASAN instrumentation of lockdep.c locking/lock_events: Add locking events for lockdep locking/lock_events: Add locking events for rtmutex slow paths x86/split_lock: Fix the delayed detection logic lockdep/mm: Fix might_fault() lockdep check of current->mm->mmap_lock x86/locking: Remove semicolon from "lock" prefix locking/mutex: Add MUTEX_WARN_ON() into fast path x86/locking: Use asm_inline for {,try_}cmpxchg{64,128} emulations x86/locking: Use ALT_OUTPUT_SP() for percpu_{,try_}cmpxchg{64,128}_op()
2 parents 3ba7dfb + 35e6b53 commit 2360899

File tree

24 files changed

+294
-109
lines changed

24 files changed

+294
-109
lines changed

‎arch/x86/include/asm/alternative.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@
4848
".popsection\n" \
4949
"671:"
5050

51-
#define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock; "
51+
#define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock "
5252

5353
#else /* ! CONFIG_SMP */
5454
#define LOCK_PREFIX_HERE ""

‎arch/x86/include/asm/barrier.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,11 @@
1212
*/
1313

1414
#ifdef CONFIG_X86_32
15-
#define mb() asm volatile(ALTERNATIVE("lock; addl $0,-4(%%esp)", "mfence", \
15+
#define mb() asm volatile(ALTERNATIVE("lock addl $0,-4(%%esp)", "mfence", \
1616
X86_FEATURE_XMM2) ::: "memory", "cc")
17-
#define rmb() asm volatile(ALTERNATIVE("lock; addl $0,-4(%%esp)", "lfence", \
17+
#define rmb() asm volatile(ALTERNATIVE("lock addl $0,-4(%%esp)", "lfence", \
1818
X86_FEATURE_XMM2) ::: "memory", "cc")
19-
#define wmb() asm volatile(ALTERNATIVE("lock; addl $0,-4(%%esp)", "sfence", \
19+
#define wmb() asm volatile(ALTERNATIVE("lock addl $0,-4(%%esp)", "sfence", \
2020
X86_FEATURE_XMM2) ::: "memory", "cc")
2121
#else
2222
#define __mb() asm volatile("mfence":::"memory")
@@ -50,7 +50,7 @@
5050
#define __dma_rmb() barrier()
5151
#define __dma_wmb() barrier()
5252

53-
#define __smp_mb() asm volatile("lock; addl $0,-4(%%" _ASM_SP ")" ::: "memory", "cc")
53+
#define __smp_mb() asm volatile("lock addl $0,-4(%%" _ASM_SP ")" ::: "memory", "cc")
5454

5555
#define __smp_rmb() dma_rmb()
5656
#define __smp_wmb() barrier()

‎arch/x86/include/asm/cmpxchg.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ extern void __add_wrong_size(void)
134134
__raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)
135135

136136
#define __sync_cmpxchg(ptr, old, new, size) \
137-
__raw_cmpxchg((ptr), (old), (new), (size), "lock; ")
137+
__raw_cmpxchg((ptr), (old), (new), (size), "lock ")
138138

139139
#define __cmpxchg_local(ptr, old, new, size) \
140140
__raw_cmpxchg((ptr), (old), (new), (size), "")
@@ -222,7 +222,7 @@ extern void __add_wrong_size(void)
222222
__raw_try_cmpxchg((ptr), (pold), (new), (size), LOCK_PREFIX)
223223

224224
#define __sync_try_cmpxchg(ptr, pold, new, size) \
225-
__raw_try_cmpxchg((ptr), (pold), (new), (size), "lock; ")
225+
__raw_try_cmpxchg((ptr), (pold), (new), (size), "lock ")
226226

227227
#define __try_cmpxchg_local(ptr, pold, new, size) \
228228
__raw_try_cmpxchg((ptr), (pold), (new), (size), "")

‎arch/x86/include/asm/cmpxchg_32.h

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -91,19 +91,21 @@ static __always_inline bool __try_cmpxchg64_local(volatile u64 *ptr, u64 *oldp,
9191
union __u64_halves o = { .full = (_old), }, \
9292
n = { .full = (_new), }; \
9393
\
94-
asm volatile(ALTERNATIVE(_lock_loc \
95-
"call cmpxchg8b_emu", \
96-
_lock "cmpxchg8b %a[ptr]", X86_FEATURE_CX8) \
97-
: ALT_OUTPUT_SP("+a" (o.low), "+d" (o.high)) \
98-
: "b" (n.low), "c" (n.high), [ptr] "S" (_ptr) \
99-
: "memory"); \
94+
asm_inline volatile( \
95+
ALTERNATIVE(_lock_loc \
96+
"call cmpxchg8b_emu", \
97+
_lock "cmpxchg8b %a[ptr]", X86_FEATURE_CX8) \
98+
: ALT_OUTPUT_SP("+a" (o.low), "+d" (o.high)) \
99+
: "b" (n.low), "c" (n.high), \
100+
[ptr] "S" (_ptr) \
101+
: "memory"); \
100102
\
101103
o.full; \
102104
})
103105

104106
static __always_inline u64 arch_cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
105107
{
106-
return __arch_cmpxchg64_emu(ptr, old, new, LOCK_PREFIX_HERE, "lock; ");
108+
return __arch_cmpxchg64_emu(ptr, old, new, LOCK_PREFIX_HERE, "lock ");
107109
}
108110
#define arch_cmpxchg64 arch_cmpxchg64
109111

@@ -119,14 +121,16 @@ static __always_inline u64 arch_cmpxchg64_local(volatile u64 *ptr, u64 old, u64
119121
n = { .full = (_new), }; \
120122
bool ret; \
121123
\
122-
asm volatile(ALTERNATIVE(_lock_loc \
123-
"call cmpxchg8b_emu", \
124-
_lock "cmpxchg8b %a[ptr]", X86_FEATURE_CX8) \
125-
CC_SET(e) \
126-
: ALT_OUTPUT_SP(CC_OUT(e) (ret), \
127-
"+a" (o.low), "+d" (o.high)) \
128-
: "b" (n.low), "c" (n.high), [ptr] "S" (_ptr) \
129-
: "memory"); \
124+
asm_inline volatile( \
125+
ALTERNATIVE(_lock_loc \
126+
"call cmpxchg8b_emu", \
127+
_lock "cmpxchg8b %a[ptr]", X86_FEATURE_CX8) \
128+
CC_SET(e) \
129+
: ALT_OUTPUT_SP(CC_OUT(e) (ret), \
130+
"+a" (o.low), "+d" (o.high)) \
131+
: "b" (n.low), "c" (n.high), \
132+
[ptr] "S" (_ptr) \
133+
: "memory"); \
130134
\
131135
if (unlikely(!ret)) \
132136
*(_oldp) = o.full; \
@@ -136,7 +140,7 @@ static __always_inline u64 arch_cmpxchg64_local(volatile u64 *ptr, u64 old, u64
136140

137141
static __always_inline bool arch_try_cmpxchg64(volatile u64 *ptr, u64 *oldp, u64 new)
138142
{
139-
return __arch_try_cmpxchg64_emu(ptr, oldp, new, LOCK_PREFIX_HERE, "lock; ");
143+
return __arch_try_cmpxchg64_emu(ptr, oldp, new, LOCK_PREFIX_HERE, "lock ");
140144
}
141145
#define arch_try_cmpxchg64 arch_try_cmpxchg64
142146

‎arch/x86/include/asm/edac.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ static inline void edac_atomic_scrub(void *va, u32 size)
1313
* are interrupt, DMA and SMP safe.
1414
*/
1515
for (i = 0; i < size / 4; i++, virt_addr++)
16-
asm volatile("lock; addl $0, %0"::"m" (*virt_addr));
16+
asm volatile("lock addl $0, %0"::"m" (*virt_addr));
1717
}
1818

1919
#endif /* _ASM_X86_EDAC_H */

‎arch/x86/include/asm/percpu.h

Lines changed: 37 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -348,15 +348,14 @@ do { \
348348
old__.var = _oval; \
349349
new__.var = _nval; \
350350
\
351-
asm qual (ALTERNATIVE("call this_cpu_cmpxchg8b_emu", \
352-
"cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \
353-
: [var] "+m" (__my_cpu_var(_var)), \
354-
"+a" (old__.low), \
355-
"+d" (old__.high) \
356-
: "b" (new__.low), \
357-
"c" (new__.high), \
358-
"S" (&(_var)) \
359-
: "memory"); \
351+
asm_inline qual ( \
352+
ALTERNATIVE("call this_cpu_cmpxchg8b_emu", \
353+
"cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \
354+
: ALT_OUTPUT_SP([var] "+m" (__my_cpu_var(_var)), \
355+
"+a" (old__.low), "+d" (old__.high)) \
356+
: "b" (new__.low), "c" (new__.high), \
357+
"S" (&(_var)) \
358+
: "memory"); \
360359
\
361360
old__.var; \
362361
})
@@ -378,17 +377,16 @@ do { \
378377
old__.var = *_oval; \
379378
new__.var = _nval; \
380379
\
381-
asm qual (ALTERNATIVE("call this_cpu_cmpxchg8b_emu", \
382-
"cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \
383-
CC_SET(z) \
384-
: CC_OUT(z) (success), \
385-
[var] "+m" (__my_cpu_var(_var)), \
386-
"+a" (old__.low), \
387-
"+d" (old__.high) \
388-
: "b" (new__.low), \
389-
"c" (new__.high), \
390-
"S" (&(_var)) \
391-
: "memory"); \
380+
asm_inline qual ( \
381+
ALTERNATIVE("call this_cpu_cmpxchg8b_emu", \
382+
"cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \
383+
CC_SET(z) \
384+
: ALT_OUTPUT_SP(CC_OUT(z) (success), \
385+
[var] "+m" (__my_cpu_var(_var)), \
386+
"+a" (old__.low), "+d" (old__.high)) \
387+
: "b" (new__.low), "c" (new__.high), \
388+
"S" (&(_var)) \
389+
: "memory"); \
392390
if (unlikely(!success)) \
393391
*_oval = old__.var; \
394392
\
@@ -419,15 +417,14 @@ do { \
419417
old__.var = _oval; \
420418
new__.var = _nval; \
421419
\
422-
asm qual (ALTERNATIVE("call this_cpu_cmpxchg16b_emu", \
423-
"cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \
424-
: [var] "+m" (__my_cpu_var(_var)), \
425-
"+a" (old__.low), \
426-
"+d" (old__.high) \
427-
: "b" (new__.low), \
428-
"c" (new__.high), \
429-
"S" (&(_var)) \
430-
: "memory"); \
420+
asm_inline qual ( \
421+
ALTERNATIVE("call this_cpu_cmpxchg16b_emu", \
422+
"cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \
423+
: ALT_OUTPUT_SP([var] "+m" (__my_cpu_var(_var)), \
424+
"+a" (old__.low), "+d" (old__.high)) \
425+
: "b" (new__.low), "c" (new__.high), \
426+
"S" (&(_var)) \
427+
: "memory"); \
431428
\
432429
old__.var; \
433430
})
@@ -449,19 +446,19 @@ do { \
449446
old__.var = *_oval; \
450447
new__.var = _nval; \
451448
\
452-
asm qual (ALTERNATIVE("call this_cpu_cmpxchg16b_emu", \
453-
"cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \
454-
CC_SET(z) \
455-
: CC_OUT(z) (success), \
456-
[var] "+m" (__my_cpu_var(_var)), \
457-
"+a" (old__.low), \
458-
"+d" (old__.high) \
459-
: "b" (new__.low), \
460-
"c" (new__.high), \
461-
"S" (&(_var)) \
462-
: "memory"); \
449+
asm_inline qual ( \
450+
ALTERNATIVE("call this_cpu_cmpxchg16b_emu", \
451+
"cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \
452+
CC_SET(z) \
453+
: ALT_OUTPUT_SP(CC_OUT(z) (success), \
454+
[var] "+m" (__my_cpu_var(_var)), \
455+
"+a" (old__.low), "+d" (old__.high)) \
456+
: "b" (new__.low), "c" (new__.high), \
457+
"S" (&(_var)) \
458+
: "memory"); \
463459
if (unlikely(!success)) \
464460
*_oval = old__.var; \
461+
\
465462
likely(success); \
466463
})
467464

‎arch/x86/include/asm/sync_bitops.h

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
*/
3232
static inline void sync_set_bit(long nr, volatile unsigned long *addr)
3333
{
34-
asm volatile("lock; " __ASM_SIZE(bts) " %1,%0"
34+
asm volatile("lock " __ASM_SIZE(bts) " %1,%0"
3535
: "+m" (ADDR)
3636
: "Ir" (nr)
3737
: "memory");
@@ -49,7 +49,7 @@ static inline void sync_set_bit(long nr, volatile unsigned long *addr)
4949
*/
5050
static inline void sync_clear_bit(long nr, volatile unsigned long *addr)
5151
{
52-
asm volatile("lock; " __ASM_SIZE(btr) " %1,%0"
52+
asm volatile("lock " __ASM_SIZE(btr) " %1,%0"
5353
: "+m" (ADDR)
5454
: "Ir" (nr)
5555
: "memory");
@@ -66,7 +66,7 @@ static inline void sync_clear_bit(long nr, volatile unsigned long *addr)
6666
*/
6767
static inline void sync_change_bit(long nr, volatile unsigned long *addr)
6868
{
69-
asm volatile("lock; " __ASM_SIZE(btc) " %1,%0"
69+
asm volatile("lock " __ASM_SIZE(btc) " %1,%0"
7070
: "+m" (ADDR)
7171
: "Ir" (nr)
7272
: "memory");
@@ -82,7 +82,7 @@ static inline void sync_change_bit(long nr, volatile unsigned long *addr)
8282
*/
8383
static inline bool sync_test_and_set_bit(long nr, volatile unsigned long *addr)
8484
{
85-
return GEN_BINARY_RMWcc("lock; " __ASM_SIZE(bts), *addr, c, "Ir", nr);
85+
return GEN_BINARY_RMWcc("lock " __ASM_SIZE(bts), *addr, c, "Ir", nr);
8686
}
8787

8888
/**
@@ -95,7 +95,7 @@ static inline bool sync_test_and_set_bit(long nr, volatile unsigned long *addr)
9595
*/
9696
static inline int sync_test_and_clear_bit(long nr, volatile unsigned long *addr)
9797
{
98-
return GEN_BINARY_RMWcc("lock; " __ASM_SIZE(btr), *addr, c, "Ir", nr);
98+
return GEN_BINARY_RMWcc("lock " __ASM_SIZE(btr), *addr, c, "Ir", nr);
9999
}
100100

101101
/**
@@ -108,7 +108,7 @@ static inline int sync_test_and_clear_bit(long nr, volatile unsigned long *addr)
108108
*/
109109
static inline int sync_test_and_change_bit(long nr, volatile unsigned long *addr)
110110
{
111-
return GEN_BINARY_RMWcc("lock; " __ASM_SIZE(btc), *addr, c, "Ir", nr);
111+
return GEN_BINARY_RMWcc("lock " __ASM_SIZE(btc), *addr, c, "Ir", nr);
112112
}
113113

114114
#define sync_test_bit(nr, addr) test_bit(nr, addr)

‎arch/x86/kernel/cpu/bus_lock.c

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,13 @@ static void __split_lock_reenable(struct work_struct *work)
192192
{
193193
sld_update_msr(true);
194194
}
195-
static DECLARE_DELAYED_WORK(sl_reenable, __split_lock_reenable);
195+
/*
196+
* In order for each CPU to schedule its delayed work independently of the
197+
* others, delayed work struct must be per-CPU. This is not required when
198+
* sysctl_sld_mitigate is enabled because of the semaphore that limits
199+
* the number of simultaneously scheduled delayed works to 1.
200+
*/
201+
static DEFINE_PER_CPU(struct delayed_work, sl_reenable);
196202

197203
/*
198204
* If a CPU goes offline with pending delayed work to re-enable split lock
@@ -213,7 +219,7 @@ static int splitlock_cpu_offline(unsigned int cpu)
213219

214220
static void split_lock_warn(unsigned long ip)
215221
{
216-
struct delayed_work *work;
222+
struct delayed_work *work = NULL;
217223
int cpu;
218224

219225
if (!current->reported_split_lock)
@@ -235,11 +241,17 @@ static void split_lock_warn(unsigned long ip)
235241
if (down_interruptible(&buslock_sem) == -EINTR)
236242
return;
237243
work = &sl_reenable_unlock;
238-
} else {
239-
work = &sl_reenable;
240244
}
241245

242246
cpu = get_cpu();
247+
248+
if (!work) {
249+
work = this_cpu_ptr(&sl_reenable);
250+
/* Deferred initialization of per-CPU struct */
251+
if (!work->work.func)
252+
INIT_DELAYED_WORK(work, __split_lock_reenable);
253+
}
254+
243255
schedule_delayed_work_on(cpu, work, 2);
244256

245257
/* Disable split lock detection on this CPU to make progress */

‎include/linux/interrupt.h

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -448,38 +448,30 @@ irq_calc_affinity_vectors(unsigned int minvec, unsigned int maxvec,
448448
static inline void disable_irq_nosync_lockdep(unsigned int irq)
449449
{
450450
disable_irq_nosync(irq);
451-
#ifdef CONFIG_LOCKDEP
451+
#if defined(CONFIG_LOCKDEP) && !defined(CONFIG_PREEMPT_RT)
452452
local_irq_disable();
453453
#endif
454454
}
455455

456456
static inline void disable_irq_nosync_lockdep_irqsave(unsigned int irq, unsigned long *flags)
457457
{
458458
disable_irq_nosync(irq);
459-
#ifdef CONFIG_LOCKDEP
459+
#if defined(CONFIG_LOCKDEP) && !defined(CONFIG_PREEMPT_RT)
460460
local_irq_save(*flags);
461461
#endif
462462
}
463463

464-
static inline void disable_irq_lockdep(unsigned int irq)
465-
{
466-
disable_irq(irq);
467-
#ifdef CONFIG_LOCKDEP
468-
local_irq_disable();
469-
#endif
470-
}
471-
472464
static inline void enable_irq_lockdep(unsigned int irq)
473465
{
474-
#ifdef CONFIG_LOCKDEP
466+
#if defined(CONFIG_LOCKDEP) && !defined(CONFIG_PREEMPT_RT)
475467
local_irq_enable();
476468
#endif
477469
enable_irq(irq);
478470
}
479471

480472
static inline void enable_irq_lockdep_irqrestore(unsigned int irq, unsigned long *flags)
481473
{
482-
#ifdef CONFIG_LOCKDEP
474+
#if defined(CONFIG_LOCKDEP) && !defined(CONFIG_PREEMPT_RT)
483475
local_irq_restore(*flags);
484476
#endif
485477
enable_irq(irq);

‎kernel/locking/Makefile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@ KCOV_INSTRUMENT := n
55

66
obj-y += mutex.o semaphore.o rwsem.o percpu-rwsem.o
77

8-
# Avoid recursion lockdep -> sanitizer -> ... -> lockdep.
8+
# Avoid recursion lockdep -> sanitizer -> ... -> lockdep & improve performance.
9+
KASAN_SANITIZE_lockdep.o := n
910
KCSAN_SANITIZE_lockdep.o := n
1011

1112
ifdef CONFIG_FUNCTION_TRACER

0 commit comments

Comments
 (0)