Skip to content

Commit 676e8cf

Browse files
author
Peter Zijlstra
committed
sched,livepatch: Untangle cond_resched() and live-patching
With the goal of deprecating / removing VOLUNTARY preempt, live-patch needs to stop relying on cond_resched() to make forward progress. Instead, rely on schedule() with TASK_FREEZABLE set. Just like live-patching, the freezer needs to be able to stop tasks in a safe / known state. [bigeasy: use likely() in __klp_sched_try_switch() and update comments] Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Reviewed-by: Thomas Gleixner <tglx@linutronix.de> Reviewed-by: Petr Mladek <pmladek@suse.com> Tested-by: Petr Mladek <pmladek@suse.com> Tested-by: Miroslav Benes <mbenes@suse.cz> Acked-by: Miroslav Benes <mbenes@suse.cz> Acked-by: Josh Poimboeuf <jpoimboe@kernel.org> Link: https://lore.kernel.org/r/20250509113659.wkP_HJ5z@linutronix.de
1 parent b7ca574 commit 676e8cf

File tree

4 files changed

+27
-92
lines changed

4 files changed

+27
-92
lines changed

‎include/linux/livepatch_sched.h

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,27 +3,23 @@
33
#define _LINUX_LIVEPATCH_SCHED_H_
44

55
#include <linux/jump_label.h>
6-
#include <linux/static_call_types.h>
6+
#include <linux/sched.h>
77

88
#ifdef CONFIG_LIVEPATCH
99

1010
void __klp_sched_try_switch(void);
1111

12-
#if !defined(CONFIG_PREEMPT_DYNAMIC) || !defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
13-
1412
DECLARE_STATIC_KEY_FALSE(klp_sched_try_switch_key);
1513

16-
static __always_inline void klp_sched_try_switch(void)
14+
static __always_inline void klp_sched_try_switch(struct task_struct *curr)
1715
{
18-
if (static_branch_unlikely(&klp_sched_try_switch_key))
16+
if (static_branch_unlikely(&klp_sched_try_switch_key) &&
17+
READ_ONCE(curr->__state) & TASK_FREEZABLE)
1918
__klp_sched_try_switch();
2019
}
2120

22-
#endif /* !CONFIG_PREEMPT_DYNAMIC || !CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */
23-
2421
#else /* !CONFIG_LIVEPATCH */
25-
static inline void klp_sched_try_switch(void) {}
26-
static inline void __klp_sched_try_switch(void) {}
22+
static inline void klp_sched_try_switch(struct task_struct *curr) {}
2723
#endif /* CONFIG_LIVEPATCH */
2824

2925
#endif /* _LINUX_LIVEPATCH_SCHED_H_ */

‎include/linux/sched.h

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@
4444
#include <linux/seqlock_types.h>
4545
#include <linux/kcsan.h>
4646
#include <linux/rv.h>
47-
#include <linux/livepatch_sched.h>
4847
#include <linux/uidgid_types.h>
4948
#include <linux/tracepoint-defs.h>
5049
#include <asm/kmap_size.h>
@@ -2089,9 +2088,6 @@ extern int __cond_resched(void);
20892088

20902089
#if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
20912090

2092-
void sched_dynamic_klp_enable(void);
2093-
void sched_dynamic_klp_disable(void);
2094-
20952091
DECLARE_STATIC_CALL(cond_resched, __cond_resched);
20962092

20972093
static __always_inline int _cond_resched(void)
@@ -2112,7 +2108,6 @@ static __always_inline int _cond_resched(void)
21122108

21132109
static inline int _cond_resched(void)
21142110
{
2115-
klp_sched_try_switch();
21162111
return __cond_resched();
21172112
}
21182113

@@ -2122,7 +2117,6 @@ static inline int _cond_resched(void)
21222117

21232118
static inline int _cond_resched(void)
21242119
{
2125-
klp_sched_try_switch();
21262120
return 0;
21272121
}
21282122

‎kernel/livepatch/transition.c

Lines changed: 14 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -29,22 +29,13 @@ static unsigned int klp_signals_cnt;
2929

3030
/*
3131
* When a livepatch is in progress, enable klp stack checking in
32-
* cond_resched(). This helps CPU-bound kthreads get patched.
32+
* schedule(). This helps CPU-bound kthreads get patched.
3333
*/
34-
#if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
35-
36-
#define klp_cond_resched_enable() sched_dynamic_klp_enable()
37-
#define klp_cond_resched_disable() sched_dynamic_klp_disable()
38-
39-
#else /* !CONFIG_PREEMPT_DYNAMIC || !CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */
4034

4135
DEFINE_STATIC_KEY_FALSE(klp_sched_try_switch_key);
42-
EXPORT_SYMBOL(klp_sched_try_switch_key);
4336

44-
#define klp_cond_resched_enable() static_branch_enable(&klp_sched_try_switch_key)
45-
#define klp_cond_resched_disable() static_branch_disable(&klp_sched_try_switch_key)
46-
47-
#endif /* CONFIG_PREEMPT_DYNAMIC && CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */
37+
#define klp_resched_enable() static_branch_enable(&klp_sched_try_switch_key)
38+
#define klp_resched_disable() static_branch_disable(&klp_sched_try_switch_key)
4839

4940
/*
5041
* This work can be performed periodically to finish patching or unpatching any
@@ -365,26 +356,18 @@ static bool klp_try_switch_task(struct task_struct *task)
365356

366357
void __klp_sched_try_switch(void)
367358
{
368-
if (likely(!klp_patch_pending(current)))
369-
return;
370-
371359
/*
372-
* This function is called from cond_resched() which is called in many
373-
* places throughout the kernel. Using the klp_mutex here might
374-
* deadlock.
375-
*
376-
* Instead, disable preemption to prevent racing with other callers of
377-
* klp_try_switch_task(). Thanks to task_call_func() they won't be
378-
* able to switch this task while it's running.
360+
* This function is called from __schedule() while a context switch is
361+
* about to happen. Preemption is already disabled and klp_mutex
362+
* can't be acquired.
363+
* Disabled preemption is used to prevent racing with other callers of
364+
* klp_try_switch_task(). Thanks to task_call_func() they won't be
365+
* able to switch to this task while it's running.
379366
*/
380-
preempt_disable();
367+
lockdep_assert_preemption_disabled();
381368

382-
/*
383-
* Make sure current didn't get patched between the above check and
384-
* preempt_disable().
385-
*/
386-
if (unlikely(!klp_patch_pending(current)))
387-
goto out;
369+
if (likely(!klp_patch_pending(current)))
370+
return;
388371

389372
/*
390373
* Enforce the order of the TIF_PATCH_PENDING read above and the
@@ -395,11 +378,7 @@ void __klp_sched_try_switch(void)
395378
smp_rmb();
396379

397380
klp_try_switch_task(current);
398-
399-
out:
400-
preempt_enable();
401381
}
402-
EXPORT_SYMBOL(__klp_sched_try_switch);
403382

404383
/*
405384
* Sends a fake signal to all non-kthread tasks with TIF_PATCH_PENDING set.
@@ -508,7 +487,7 @@ void klp_try_complete_transition(void)
508487
}
509488

510489
/* Done! Now cleanup the data structures. */
511-
klp_cond_resched_disable();
490+
klp_resched_disable();
512491
patch = klp_transition_patch;
513492
klp_complete_transition();
514493

@@ -560,7 +539,7 @@ void klp_start_transition(void)
560539
set_tsk_thread_flag(task, TIF_PATCH_PENDING);
561540
}
562541

563-
klp_cond_resched_enable();
542+
klp_resched_enable();
564543

565544
klp_signals_cnt = 0;
566545
}

‎kernel/sched/core.c

Lines changed: 8 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@
6666
#include <linux/vtime.h>
6767
#include <linux/wait_api.h>
6868
#include <linux/workqueue_api.h>
69+
#include <linux/livepatch_sched.h>
6970

7071
#ifdef CONFIG_PREEMPT_DYNAMIC
7172
# ifdef CONFIG_GENERIC_ENTRY
@@ -6676,6 +6677,8 @@ static void __sched notrace __schedule(int sched_mode)
66766677
if (sched_feat(HRTICK) || sched_feat(HRTICK_DL))
66776678
hrtick_clear(rq);
66786679

6680+
klp_sched_try_switch(prev);
6681+
66796682
local_irq_disable();
66806683
rcu_note_context_switch(preempt);
66816684

@@ -7336,7 +7339,6 @@ EXPORT_STATIC_CALL_TRAMP(might_resched);
73367339
static DEFINE_STATIC_KEY_FALSE(sk_dynamic_cond_resched);
73377340
int __sched dynamic_cond_resched(void)
73387341
{
7339-
klp_sched_try_switch();
73407342
if (!static_branch_unlikely(&sk_dynamic_cond_resched))
73417343
return 0;
73427344
return __cond_resched();
@@ -7508,16 +7510,14 @@ int sched_dynamic_mode(const char *str)
75087510
#endif
75097511

75107512
static DEFINE_MUTEX(sched_dynamic_mutex);
7511-
static bool klp_override;
75127513

75137514
static void __sched_dynamic_update(int mode)
75147515
{
75157516
/*
75167517
* Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in
75177518
* the ZERO state, which is invalid.
75187519
*/
7519-
if (!klp_override)
7520-
preempt_dynamic_enable(cond_resched);
7520+
preempt_dynamic_enable(cond_resched);
75217521
preempt_dynamic_enable(might_resched);
75227522
preempt_dynamic_enable(preempt_schedule);
75237523
preempt_dynamic_enable(preempt_schedule_notrace);
@@ -7526,8 +7526,7 @@ static void __sched_dynamic_update(int mode)
75267526

75277527
switch (mode) {
75287528
case preempt_dynamic_none:
7529-
if (!klp_override)
7530-
preempt_dynamic_enable(cond_resched);
7529+
preempt_dynamic_enable(cond_resched);
75317530
preempt_dynamic_disable(might_resched);
75327531
preempt_dynamic_disable(preempt_schedule);
75337532
preempt_dynamic_disable(preempt_schedule_notrace);
@@ -7538,8 +7537,7 @@ static void __sched_dynamic_update(int mode)
75387537
break;
75397538

75407539
case preempt_dynamic_voluntary:
7541-
if (!klp_override)
7542-
preempt_dynamic_enable(cond_resched);
7540+
preempt_dynamic_enable(cond_resched);
75437541
preempt_dynamic_enable(might_resched);
75447542
preempt_dynamic_disable(preempt_schedule);
75457543
preempt_dynamic_disable(preempt_schedule_notrace);
@@ -7550,8 +7548,7 @@ static void __sched_dynamic_update(int mode)
75507548
break;
75517549

75527550
case preempt_dynamic_full:
7553-
if (!klp_override)
7554-
preempt_dynamic_disable(cond_resched);
7551+
preempt_dynamic_disable(cond_resched);
75557552
preempt_dynamic_disable(might_resched);
75567553
preempt_dynamic_enable(preempt_schedule);
75577554
preempt_dynamic_enable(preempt_schedule_notrace);
@@ -7562,8 +7559,7 @@ static void __sched_dynamic_update(int mode)
75627559
break;
75637560

75647561
case preempt_dynamic_lazy:
7565-
if (!klp_override)
7566-
preempt_dynamic_disable(cond_resched);
7562+
preempt_dynamic_disable(cond_resched);
75677563
preempt_dynamic_disable(might_resched);
75687564
preempt_dynamic_enable(preempt_schedule);
75697565
preempt_dynamic_enable(preempt_schedule_notrace);
@@ -7584,36 +7580,6 @@ void sched_dynamic_update(int mode)
75847580
mutex_unlock(&sched_dynamic_mutex);
75857581
}
75867582

7587-
#ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL
7588-
7589-
static int klp_cond_resched(void)
7590-
{
7591-
__klp_sched_try_switch();
7592-
return __cond_resched();
7593-
}
7594-
7595-
void sched_dynamic_klp_enable(void)
7596-
{
7597-
mutex_lock(&sched_dynamic_mutex);
7598-
7599-
klp_override = true;
7600-
static_call_update(cond_resched, klp_cond_resched);
7601-
7602-
mutex_unlock(&sched_dynamic_mutex);
7603-
}
7604-
7605-
void sched_dynamic_klp_disable(void)
7606-
{
7607-
mutex_lock(&sched_dynamic_mutex);
7608-
7609-
klp_override = false;
7610-
__sched_dynamic_update(preempt_dynamic_mode);
7611-
7612-
mutex_unlock(&sched_dynamic_mutex);
7613-
}
7614-
7615-
#endif /* CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */
7616-
76177583
static int __init setup_preempt_mode(char *str)
76187584
{
76197585
int mode = sched_dynamic_mode(str);

0 commit comments

Comments
 (0)