Skip to content

Commit abfdccd

Browse files
johnstultz-workPeter Zijlstra
authored andcommitted
sched/wake_q: Add helper to call wake_up_q after unlock with preemption disabled
A common pattern seen when wake_qs are used to defer a wakeup until after a lock is released is something like: preempt_disable(); raw_spin_unlock(lock); wake_up_q(wake_q); preempt_enable(); So create some raw_spin_unlock*_wake() helper functions to clean this up. Applies on top of the fix I submitted here: https://lore.kernel.org/lkml/20241212222138.2400498-1-jstultz@google.com/ NOTE: I recognise the unlock()/unlock_irq()/unlock_irqrestore() variants creates its own duplication, which we could use a macro to generate the similar functions, but I often dislike how those generation macros making finding the actual implementation harder, so I left the three functions as is. If folks would prefer otherwise, let me know and I'll switch it. Suggested-by: Peter Zijlstra <peterz@infradead.org> Signed-off-by: John Stultz <jstultz@google.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Link: https://lkml.kernel.org/r/20241217040803.243420-1-jstultz@google.com
1 parent c2db11a commit abfdccd

File tree

4 files changed

+44
-43
lines changed

4 files changed

+44
-43
lines changed

include/linux/sched/wake_q.h

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,4 +63,38 @@ extern void wake_q_add(struct wake_q_head *head, struct task_struct *task);
6363
extern void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task);
6464
extern void wake_up_q(struct wake_q_head *head);
6565

66+
/* Spin unlock helpers to unlock and call wake_up_q with preempt disabled */
67+
static inline
68+
void raw_spin_unlock_wake(raw_spinlock_t *lock, struct wake_q_head *wake_q)
69+
{
70+
guard(preempt)();
71+
raw_spin_unlock(lock);
72+
if (wake_q) {
73+
wake_up_q(wake_q);
74+
wake_q_init(wake_q);
75+
}
76+
}
77+
78+
static inline
79+
void raw_spin_unlock_irq_wake(raw_spinlock_t *lock, struct wake_q_head *wake_q)
80+
{
81+
guard(preempt)();
82+
raw_spin_unlock_irq(lock);
83+
if (wake_q) {
84+
wake_up_q(wake_q);
85+
wake_q_init(wake_q);
86+
}
87+
}
88+
89+
static inline
90+
void raw_spin_unlock_irqrestore_wake(raw_spinlock_t *lock, unsigned long flags,
91+
struct wake_q_head *wake_q)
92+
{
93+
guard(preempt)();
94+
raw_spin_unlock_irqrestore(lock, flags);
95+
if (wake_q) {
96+
wake_up_q(wake_q);
97+
wake_q_init(wake_q);
98+
}
99+
}
66100
#endif /* _LINUX_SCHED_WAKE_Q_H */

kernel/futex/pi.c

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1020,10 +1020,7 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl
10201020
* it sees the futex_q::pi_state.
10211021
*/
10221022
ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current, &wake_q);
1023-
preempt_disable();
1024-
raw_spin_unlock_irq(&q.pi_state->pi_mutex.wait_lock);
1025-
wake_up_q(&wake_q);
1026-
preempt_enable();
1023+
raw_spin_unlock_irq_wake(&q.pi_state->pi_mutex.wait_lock, &wake_q);
10271024

10281025
if (ret) {
10291026
if (ret == 1)

kernel/locking/mutex.c

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -657,10 +657,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
657657
goto err;
658658
}
659659

660-
raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
661-
/* Make sure we do wakeups before calling schedule */
662-
wake_up_q(&wake_q);
663-
wake_q_init(&wake_q);
660+
raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);
664661

665662
schedule_preempt_disabled();
666663

@@ -710,8 +707,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
710707
if (ww_ctx)
711708
ww_mutex_lock_acquired(ww, ww_ctx);
712709

713-
raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
714-
wake_up_q(&wake_q);
710+
raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);
715711
preempt_enable();
716712
return 0;
717713

@@ -720,10 +716,9 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
720716
__mutex_remove_waiter(lock, &waiter);
721717
err_early_kill:
722718
trace_contention_end(lock, ret);
723-
raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
719+
raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);
724720
debug_mutex_free_waiter(&waiter);
725721
mutex_release(&lock->dep_map, ip);
726-
wake_up_q(&wake_q);
727722
preempt_enable();
728723
return ret;
729724
}
@@ -935,10 +930,7 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
935930
if (owner & MUTEX_FLAG_HANDOFF)
936931
__mutex_handoff(lock, next);
937932

938-
preempt_disable();
939-
raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
940-
wake_up_q(&wake_q);
941-
preempt_enable();
933+
raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);
942934
}
943935

944936
#ifndef CONFIG_DEBUG_LOCK_ALLOC

kernel/locking/rtmutex.c

Lines changed: 5 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1292,13 +1292,7 @@ static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock,
12921292
*/
12931293
get_task_struct(owner);
12941294

1295-
preempt_disable();
1296-
raw_spin_unlock_irq(&lock->wait_lock);
1297-
/* wake up any tasks on the wake_q before calling rt_mutex_adjust_prio_chain */
1298-
wake_up_q(wake_q);
1299-
wake_q_init(wake_q);
1300-
preempt_enable();
1301-
1295+
raw_spin_unlock_irq_wake(&lock->wait_lock, wake_q);
13021296

13031297
res = rt_mutex_adjust_prio_chain(owner, chwalk, lock,
13041298
next_lock, waiter, task);
@@ -1642,13 +1636,7 @@ static int __sched rt_mutex_slowlock_block(struct rt_mutex_base *lock,
16421636
owner = rt_mutex_owner(lock);
16431637
else
16441638
owner = NULL;
1645-
preempt_disable();
1646-
raw_spin_unlock_irq(&lock->wait_lock);
1647-
if (wake_q) {
1648-
wake_up_q(wake_q);
1649-
wake_q_init(wake_q);
1650-
}
1651-
preempt_enable();
1639+
raw_spin_unlock_irq_wake(&lock->wait_lock, wake_q);
16521640

16531641
if (!owner || !rtmutex_spin_on_owner(lock, waiter, owner))
16541642
rt_mutex_schedule();
@@ -1799,10 +1787,7 @@ static int __sched rt_mutex_slowlock(struct rt_mutex_base *lock,
17991787
*/
18001788
raw_spin_lock_irqsave(&lock->wait_lock, flags);
18011789
ret = __rt_mutex_slowlock_locked(lock, ww_ctx, state, &wake_q);
1802-
preempt_disable();
1803-
raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
1804-
wake_up_q(&wake_q);
1805-
preempt_enable();
1790+
raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);
18061791
rt_mutex_post_schedule();
18071792

18081793
return ret;
@@ -1860,11 +1845,7 @@ static void __sched rtlock_slowlock_locked(struct rt_mutex_base *lock,
18601845
owner = rt_mutex_owner(lock);
18611846
else
18621847
owner = NULL;
1863-
preempt_disable();
1864-
raw_spin_unlock_irq(&lock->wait_lock);
1865-
wake_up_q(wake_q);
1866-
wake_q_init(wake_q);
1867-
preempt_enable();
1848+
raw_spin_unlock_irq_wake(&lock->wait_lock, wake_q);
18681849

18691850
if (!owner || !rtmutex_spin_on_owner(lock, &waiter, owner))
18701851
schedule_rtlock();
@@ -1893,10 +1874,7 @@ static __always_inline void __sched rtlock_slowlock(struct rt_mutex_base *lock)
18931874

18941875
raw_spin_lock_irqsave(&lock->wait_lock, flags);
18951876
rtlock_slowlock_locked(lock, &wake_q);
1896-
preempt_disable();
1897-
raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
1898-
wake_up_q(&wake_q);
1899-
preempt_enable();
1877+
raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);
19001878
}
19011879

19021880
#endif /* RT_MUTEX_BUILD_SPINLOCKS */

0 commit comments

Comments
 (0)