Skip to content

Commit 0aa7be0

Browse files
ubizjakPeter Zijlstra
ubizjak
authored and
Peter Zijlstra
committed
locking/atomic: Add generic try_cmpxchg64 support
Add generic support for try_cmpxchg64{,_acquire,_release,_relaxed} and their falbacks involving cmpxchg64. Signed-off-by: Uros Bizjak <ubizjak@gmail.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Link: https://lkml.kernel.org/r/20220515184205.103089-2-ubizjak@gmail.com
1 parent 6829061 commit 0aa7be0

File tree

4 files changed

+129
-16
lines changed

4 files changed

+129
-16
lines changed

‎include/linux/atomic/atomic-arch-fallback.h

Lines changed: 71 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,76 @@
147147

148148
#endif /* arch_try_cmpxchg_relaxed */
149149

150+
#ifndef arch_try_cmpxchg64_relaxed
151+
#ifdef arch_try_cmpxchg64
152+
#define arch_try_cmpxchg64_acquire arch_try_cmpxchg64
153+
#define arch_try_cmpxchg64_release arch_try_cmpxchg64
154+
#define arch_try_cmpxchg64_relaxed arch_try_cmpxchg64
155+
#endif /* arch_try_cmpxchg64 */
156+
157+
#ifndef arch_try_cmpxchg64
158+
#define arch_try_cmpxchg64(_ptr, _oldp, _new) \
159+
({ \
160+
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
161+
___r = arch_cmpxchg64((_ptr), ___o, (_new)); \
162+
if (unlikely(___r != ___o)) \
163+
*___op = ___r; \
164+
likely(___r == ___o); \
165+
})
166+
#endif /* arch_try_cmpxchg64 */
167+
168+
#ifndef arch_try_cmpxchg64_acquire
169+
#define arch_try_cmpxchg64_acquire(_ptr, _oldp, _new) \
170+
({ \
171+
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
172+
___r = arch_cmpxchg64_acquire((_ptr), ___o, (_new)); \
173+
if (unlikely(___r != ___o)) \
174+
*___op = ___r; \
175+
likely(___r == ___o); \
176+
})
177+
#endif /* arch_try_cmpxchg64_acquire */
178+
179+
#ifndef arch_try_cmpxchg64_release
180+
#define arch_try_cmpxchg64_release(_ptr, _oldp, _new) \
181+
({ \
182+
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
183+
___r = arch_cmpxchg64_release((_ptr), ___o, (_new)); \
184+
if (unlikely(___r != ___o)) \
185+
*___op = ___r; \
186+
likely(___r == ___o); \
187+
})
188+
#endif /* arch_try_cmpxchg64_release */
189+
190+
#ifndef arch_try_cmpxchg64_relaxed
191+
#define arch_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \
192+
({ \
193+
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
194+
___r = arch_cmpxchg64_relaxed((_ptr), ___o, (_new)); \
195+
if (unlikely(___r != ___o)) \
196+
*___op = ___r; \
197+
likely(___r == ___o); \
198+
})
199+
#endif /* arch_try_cmpxchg64_relaxed */
200+
201+
#else /* arch_try_cmpxchg64_relaxed */
202+
203+
#ifndef arch_try_cmpxchg64_acquire
204+
#define arch_try_cmpxchg64_acquire(...) \
205+
__atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__)
206+
#endif
207+
208+
#ifndef arch_try_cmpxchg64_release
209+
#define arch_try_cmpxchg64_release(...) \
210+
__atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__)
211+
#endif
212+
213+
#ifndef arch_try_cmpxchg64
214+
#define arch_try_cmpxchg64(...) \
215+
__atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__)
216+
#endif
217+
218+
#endif /* arch_try_cmpxchg64_relaxed */
219+
150220
#ifndef arch_atomic_read_acquire
151221
static __always_inline int
152222
arch_atomic_read_acquire(const atomic_t *v)
@@ -2386,4 +2456,4 @@ arch_atomic64_dec_if_positive(atomic64_t *v)
23862456
#endif
23872457

23882458
#endif /* _LINUX_ATOMIC_FALLBACK_H */
2389-
// 8e2cc06bc0d2c0967d2f8424762bd48555ee40ae
2459+
// b5e87bdd5ede61470c29f7a7e4de781af3770f09

‎include/linux/atomic/atomic-instrumented.h

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2006,6 +2006,44 @@ atomic_long_dec_if_positive(atomic_long_t *v)
20062006
arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
20072007
})
20082008

2009+
#define try_cmpxchg64(ptr, oldp, ...) \
2010+
({ \
2011+
typeof(ptr) __ai_ptr = (ptr); \
2012+
typeof(oldp) __ai_oldp = (oldp); \
2013+
kcsan_mb(); \
2014+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2015+
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2016+
arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2017+
})
2018+
2019+
#define try_cmpxchg64_acquire(ptr, oldp, ...) \
2020+
({ \
2021+
typeof(ptr) __ai_ptr = (ptr); \
2022+
typeof(oldp) __ai_oldp = (oldp); \
2023+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2024+
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2025+
arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2026+
})
2027+
2028+
#define try_cmpxchg64_release(ptr, oldp, ...) \
2029+
({ \
2030+
typeof(ptr) __ai_ptr = (ptr); \
2031+
typeof(oldp) __ai_oldp = (oldp); \
2032+
kcsan_release(); \
2033+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2034+
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2035+
arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2036+
})
2037+
2038+
#define try_cmpxchg64_relaxed(ptr, oldp, ...) \
2039+
({ \
2040+
typeof(ptr) __ai_ptr = (ptr); \
2041+
typeof(oldp) __ai_oldp = (oldp); \
2042+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2043+
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2044+
arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2045+
})
2046+
20092047
#define cmpxchg_local(ptr, ...) \
20102048
({ \
20112049
typeof(ptr) __ai_ptr = (ptr); \
@@ -2045,4 +2083,4 @@ atomic_long_dec_if_positive(atomic_long_t *v)
20452083
})
20462084

20472085
#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
2048-
// 87c974b93032afd42143613434d1a7788fa598f9
2086+
// 764f741eb77a7ad565dc8d99ce2837d5542e8aee

‎scripts/atomic/gen-atomic-fallback.sh

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -164,41 +164,44 @@ gen_xchg_fallbacks()
164164

165165
gen_try_cmpxchg_fallback()
166166
{
167+
local cmpxchg="$1"; shift;
167168
local order="$1"; shift;
168169

169170
cat <<EOF
170-
#ifndef arch_try_cmpxchg${order}
171-
#define arch_try_cmpxchg${order}(_ptr, _oldp, _new) \\
171+
#ifndef arch_try_${cmpxchg}${order}
172+
#define arch_try_${cmpxchg}${order}(_ptr, _oldp, _new) \\
172173
({ \\
173174
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \\
174-
___r = arch_cmpxchg${order}((_ptr), ___o, (_new)); \\
175+
___r = arch_${cmpxchg}${order}((_ptr), ___o, (_new)); \\
175176
if (unlikely(___r != ___o)) \\
176177
*___op = ___r; \\
177178
likely(___r == ___o); \\
178179
})
179-
#endif /* arch_try_cmpxchg${order} */
180+
#endif /* arch_try_${cmpxchg}${order} */
180181
181182
EOF
182183
}
183184

184185
gen_try_cmpxchg_fallbacks()
185186
{
186-
printf "#ifndef arch_try_cmpxchg_relaxed\n"
187-
printf "#ifdef arch_try_cmpxchg\n"
187+
local cmpxchg="$1"; shift;
188188

189-
gen_basic_fallbacks "arch_try_cmpxchg"
189+
printf "#ifndef arch_try_${cmpxchg}_relaxed\n"
190+
printf "#ifdef arch_try_${cmpxchg}\n"
190191

191-
printf "#endif /* arch_try_cmpxchg */\n\n"
192+
gen_basic_fallbacks "arch_try_${cmpxchg}"
193+
194+
printf "#endif /* arch_try_${cmpxchg} */\n\n"
192195

193196
for order in "" "_acquire" "_release" "_relaxed"; do
194-
gen_try_cmpxchg_fallback "${order}"
197+
gen_try_cmpxchg_fallback "${cmpxchg}" "${order}"
195198
done
196199

197-
printf "#else /* arch_try_cmpxchg_relaxed */\n"
200+
printf "#else /* arch_try_${cmpxchg}_relaxed */\n"
198201

199-
gen_order_fallbacks "arch_try_cmpxchg"
202+
gen_order_fallbacks "arch_try_${cmpxchg}"
200203

201-
printf "#endif /* arch_try_cmpxchg_relaxed */\n\n"
204+
printf "#endif /* arch_try_${cmpxchg}_relaxed */\n\n"
202205
}
203206

204207
cat << EOF
@@ -218,7 +221,9 @@ for xchg in "arch_xchg" "arch_cmpxchg" "arch_cmpxchg64"; do
218221
gen_xchg_fallbacks "${xchg}"
219222
done
220223

221-
gen_try_cmpxchg_fallbacks
224+
for cmpxchg in "cmpxchg" "cmpxchg64"; do
225+
gen_try_cmpxchg_fallbacks "${cmpxchg}"
226+
done
222227

223228
grep '^[a-z]' "$1" | while read name meta args; do
224229
gen_proto "${meta}" "${name}" "atomic" "int" ${args}

‎scripts/atomic/gen-atomic-instrumented.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ grep '^[a-z]' "$1" | while read name meta args; do
166166
done
167167

168168

169-
for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg"; do
169+
for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg" "try_cmpxchg64"; do
170170
for order in "" "_acquire" "_release" "_relaxed"; do
171171
gen_xchg "${xchg}" "${order}" ""
172172
printf "\n"

0 commit comments

Comments
 (0)