@@ -131,13 +131,16 @@ bool static_key_fast_inc_not_disabled(struct static_key *key)
131
131
STATIC_KEY_CHECK_USE (key );
132
132
/*
133
133
* Negative key->enabled has a special meaning: it sends
134
- * static_key_slow_inc() down the slow path, and it is non-zero
135
- * so it counts as "enabled" in jump_label_update(). Note that
136
- * atomic_inc_unless_negative() checks >= 0, so roll our own.
134
+ * static_key_slow_inc/dec() down the slow path, and it is non-zero
135
+ * so it counts as "enabled" in jump_label_update().
136
+ *
137
+ * The INT_MAX overflow condition is either used by the networking
138
+ * code to reset or detected in the slow path of
139
+ * static_key_slow_inc_cpuslocked().
137
140
*/
138
141
v = atomic_read (& key -> enabled );
139
142
do {
140
- if (v <= 0 || ( v + 1 ) < 0 )
143
+ if (v <= 0 || v == INT_MAX )
141
144
return false;
142
145
} while (!likely (atomic_try_cmpxchg (& key -> enabled , & v , v + 1 )));
143
146
@@ -150,7 +153,7 @@ bool static_key_slow_inc_cpuslocked(struct static_key *key)
150
153
lockdep_assert_cpus_held ();
151
154
152
155
/*
153
- * Careful if we get concurrent static_key_slow_inc() calls;
156
+ * Careful if we get concurrent static_key_slow_inc/dec () calls;
154
157
* later calls must wait for the first one to _finish_ the
155
158
* jump_label_update() process. At the same time, however,
156
159
* the jump_label_update() call below wants to see
@@ -159,22 +162,24 @@ bool static_key_slow_inc_cpuslocked(struct static_key *key)
159
162
if (static_key_fast_inc_not_disabled (key ))
160
163
return true;
161
164
162
- jump_label_lock ( );
163
- if ( atomic_read ( & key -> enabled ) == 0 ) {
164
- atomic_set ( & key -> enabled , -1 );
165
+ guard ( mutex )( & jump_label_mutex );
166
+ /* Try to mark it as 'enabling in progress. */
167
+ if (! atomic_cmpxchg ( & key -> enabled , 0 , -1 )) {
165
168
jump_label_update (key );
166
169
/*
167
- * Ensure that if the above cmpxchg loop observes our positive
168
- * value, it must also observe all the text changes.
170
+ * Ensure that when static_key_fast_inc_not_disabled() or
171
+ * static_key_slow_try_dec() observe the positive value,
172
+ * they must also observe all the text changes.
169
173
*/
170
174
atomic_set_release (& key -> enabled , 1 );
171
175
} else {
172
- if (WARN_ON_ONCE (!static_key_fast_inc_not_disabled (key ))) {
173
- jump_label_unlock ();
176
+ /*
177
+ * While holding the mutex this should never observe
178
+ * anything else than a value >= 1 and succeed
179
+ */
180
+ if (WARN_ON_ONCE (!static_key_fast_inc_not_disabled (key )))
174
181
return false;
175
- }
176
182
}
177
- jump_label_unlock ();
178
183
return true;
179
184
}
180
185
@@ -247,20 +252,32 @@ EXPORT_SYMBOL_GPL(static_key_disable);
247
252
248
253
static bool static_key_slow_try_dec (struct static_key * key )
249
254
{
250
- int val ;
251
-
252
- val = atomic_fetch_add_unless (& key -> enabled , -1 , 1 );
253
- if (val == 1 )
254
- return false;
255
+ int v ;
255
256
256
257
/*
257
- * The negative count check is valid even when a negative
258
- * key->enabled is in use by static_key_slow_inc(); a
259
- * __static_key_slow_dec() before the first static_key_slow_inc()
260
- * returns is unbalanced, because all other static_key_slow_inc()
261
- * instances block while the update is in progress.
258
+ * Go into the slow path if key::enabled is less than or equal than
259
+ * one. One is valid to shut down the key, anything less than one
260
+ * is an imbalance, which is handled at the call site.
261
+ *
262
+ * That includes the special case of '-1' which is set in
263
+ * static_key_slow_inc_cpuslocked(), but that's harmless as it is
264
+ * fully serialized in the slow path below. By the time this task
265
+ * acquires the jump label lock the value is back to one and the
266
+ * retry under the lock must succeed.
262
267
*/
263
- WARN (val < 0 , "jump label: negative count!\n" );
268
+ v = atomic_read (& key -> enabled );
269
+ do {
270
+ /*
271
+ * Warn about the '-1' case though; since that means a
272
+ * decrement is concurrent with a first (0->1) increment. IOW
273
+ * people are trying to disable something that wasn't yet fully
274
+ * enabled. This suggests an ordering problem on the user side.
275
+ */
276
+ WARN_ON_ONCE (v < 0 );
277
+ if (v <= 1 )
278
+ return false;
279
+ } while (!likely (atomic_try_cmpxchg (& key -> enabled , & v , v - 1 )));
280
+
264
281
return true;
265
282
}
266
283
@@ -271,10 +288,11 @@ static void __static_key_slow_dec_cpuslocked(struct static_key *key)
271
288
if (static_key_slow_try_dec (key ))
272
289
return ;
273
290
274
- jump_label_lock ( );
275
- if (atomic_dec_and_test (& key -> enabled ))
291
+ guard ( mutex )( & jump_label_mutex );
292
+ if (atomic_cmpxchg (& key -> enabled , 1 , 0 ))
276
293
jump_label_update (key );
277
- jump_label_unlock ();
294
+ else
295
+ WARN_ON_ONCE (!static_key_slow_try_dec (key ));
278
296
}
279
297
280
298
static void __static_key_slow_dec (struct static_key * key )
0 commit comments