~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm64/include/asm/atomic_ll_sc.h

Version: ~ [ linux-5.13-rc5 ] ~ [ linux-5.12.9 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.42 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.124 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.193 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.235 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.271 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.271 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* SPDX-License-Identifier: GPL-2.0-only */
  2 /*
  3  * Based on arch/arm/include/asm/atomic.h
  4  *
  5  * Copyright (C) 1996 Russell King.
  6  * Copyright (C) 2002 Deep Blue Solutions Ltd.
  7  * Copyright (C) 2012 ARM Ltd.
  8  */
  9 
 10 #ifndef __ASM_ATOMIC_LL_SC_H
 11 #define __ASM_ATOMIC_LL_SC_H
 12 
 13 #ifndef __ARM64_IN_ATOMIC_IMPL
 14 #error "please don't include this file directly"
 15 #endif
 16 
 17 /*
 18  * AArch64 UP and SMP safe atomic ops.  We use load exclusive and
 19  * store exclusive to ensure that these are atomic.  We may loop
 20  * to ensure that the update happens.
 21  *
 22  * NOTE: these functions do *not* follow the PCS and must explicitly
 23  * save any clobbered registers other than x0 (regardless of return
 24  * value).  This is achieved through -fcall-saved-* compiler flags for
 25  * this file, which unfortunately don't work on a per-function basis
 26  * (the optimize attribute silently ignores these options).
 27  */
 28 
 29 #define ATOMIC_OP(op, asm_op, constraint)                               \
 30 __LL_SC_INLINE void                                                     \
 31 __LL_SC_PREFIX(arch_atomic_##op(int i, atomic_t *v))                    \
 32 {                                                                       \
 33         unsigned long tmp;                                              \
 34         int result;                                                     \
 35                                                                         \
 36         asm volatile("// atomic_" #op "\n"                              \
 37 "       prfm    pstl1strm, %2\n"                                        \
 38 "1:     ldxr    %w0, %2\n"                                              \
 39 "       " #asm_op "     %w0, %w0, %w3\n"                                \
 40 "       stxr    %w1, %w0, %2\n"                                         \
 41 "       cbnz    %w1, 1b"                                                \
 42         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
 43         : #constraint "r" (i));                                         \
 44 }                                                                       \
 45 __LL_SC_EXPORT(arch_atomic_##op);
 46 
 47 #define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\
 48 __LL_SC_INLINE int                                                      \
 49 __LL_SC_PREFIX(arch_atomic_##op##_return##name(int i, atomic_t *v))     \
 50 {                                                                       \
 51         unsigned long tmp;                                              \
 52         int result;                                                     \
 53                                                                         \
 54         asm volatile("// atomic_" #op "_return" #name "\n"              \
 55 "       prfm    pstl1strm, %2\n"                                        \
 56 "1:     ld" #acq "xr    %w0, %2\n"                                      \
 57 "       " #asm_op "     %w0, %w0, %w3\n"                                \
 58 "       st" #rel "xr    %w1, %w0, %2\n"                                 \
 59 "       cbnz    %w1, 1b\n"                                              \
 60 "       " #mb                                                           \
 61         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
 62         : #constraint "r" (i)                                           \
 63         : cl);                                                          \
 64                                                                         \
 65         return result;                                                  \
 66 }                                                                       \
 67 __LL_SC_EXPORT(arch_atomic_##op##_return##name);
 68 
 69 #define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint) \
 70 __LL_SC_INLINE int                                                      \
 71 __LL_SC_PREFIX(arch_atomic_fetch_##op##name(int i, atomic_t *v))        \
 72 {                                                                       \
 73         unsigned long tmp;                                              \
 74         int val, result;                                                \
 75                                                                         \
 76         asm volatile("// atomic_fetch_" #op #name "\n"                  \
 77 "       prfm    pstl1strm, %3\n"                                        \
 78 "1:     ld" #acq "xr    %w0, %3\n"                                      \
 79 "       " #asm_op "     %w1, %w0, %w4\n"                                \
 80 "       st" #rel "xr    %w2, %w1, %3\n"                                 \
 81 "       cbnz    %w2, 1b\n"                                              \
 82 "       " #mb                                                           \
 83         : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter)   \
 84         : #constraint "r" (i)                                           \
 85         : cl);                                                          \
 86                                                                         \
 87         return result;                                                  \
 88 }                                                                       \
 89 __LL_SC_EXPORT(arch_atomic_fetch_##op##name);
 90 
 91 #define ATOMIC_OPS(...)                                                 \
 92         ATOMIC_OP(__VA_ARGS__)                                          \
 93         ATOMIC_OP_RETURN(        , dmb ish,  , l, "memory", __VA_ARGS__)\
 94         ATOMIC_OP_RETURN(_relaxed,        ,  ,  ,         , __VA_ARGS__)\
 95         ATOMIC_OP_RETURN(_acquire,        , a,  , "memory", __VA_ARGS__)\
 96         ATOMIC_OP_RETURN(_release,        ,  , l, "memory", __VA_ARGS__)\
 97         ATOMIC_FETCH_OP (        , dmb ish,  , l, "memory", __VA_ARGS__)\
 98         ATOMIC_FETCH_OP (_relaxed,        ,  ,  ,         , __VA_ARGS__)\
 99         ATOMIC_FETCH_OP (_acquire,        , a,  , "memory", __VA_ARGS__)\
100         ATOMIC_FETCH_OP (_release,        ,  , l, "memory", __VA_ARGS__)
101 
102 ATOMIC_OPS(add, add, I)
103 ATOMIC_OPS(sub, sub, J)
104 
105 #undef ATOMIC_OPS
106 #define ATOMIC_OPS(...)                                                 \
107         ATOMIC_OP(__VA_ARGS__)                                          \
108         ATOMIC_FETCH_OP (        , dmb ish,  , l, "memory", __VA_ARGS__)\
109         ATOMIC_FETCH_OP (_relaxed,        ,  ,  ,         , __VA_ARGS__)\
110         ATOMIC_FETCH_OP (_acquire,        , a,  , "memory", __VA_ARGS__)\
111         ATOMIC_FETCH_OP (_release,        ,  , l, "memory", __VA_ARGS__)
112 
113 ATOMIC_OPS(and, and, )
114 ATOMIC_OPS(andnot, bic, )
115 ATOMIC_OPS(or, orr, )
116 ATOMIC_OPS(xor, eor, )
117 
118 #undef ATOMIC_OPS
119 #undef ATOMIC_FETCH_OP
120 #undef ATOMIC_OP_RETURN
121 #undef ATOMIC_OP
122 
123 #define ATOMIC64_OP(op, asm_op, constraint)                             \
124 __LL_SC_INLINE void                                                     \
125 __LL_SC_PREFIX(arch_atomic64_##op(s64 i, atomic64_t *v))                \
126 {                                                                       \
127         s64 result;                                                     \
128         unsigned long tmp;                                              \
129                                                                         \
130         asm volatile("// atomic64_" #op "\n"                            \
131 "       prfm    pstl1strm, %2\n"                                        \
132 "1:     ldxr    %0, %2\n"                                               \
133 "       " #asm_op "     %0, %0, %3\n"                                   \
134 "       stxr    %w1, %0, %2\n"                                          \
135 "       cbnz    %w1, 1b"                                                \
136         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
137         : #constraint "r" (i));                                         \
138 }                                                                       \
139 __LL_SC_EXPORT(arch_atomic64_##op);
140 
141 #define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\
142 __LL_SC_INLINE s64                                                      \
143 __LL_SC_PREFIX(arch_atomic64_##op##_return##name(s64 i, atomic64_t *v))\
144 {                                                                       \
145         s64 result;                                                     \
146         unsigned long tmp;                                              \
147                                                                         \
148         asm volatile("// atomic64_" #op "_return" #name "\n"            \
149 "       prfm    pstl1strm, %2\n"                                        \
150 "1:     ld" #acq "xr    %0, %2\n"                                       \
151 "       " #asm_op "     %0, %0, %3\n"                                   \
152 "       st" #rel "xr    %w1, %0, %2\n"                                  \
153 "       cbnz    %w1, 1b\n"                                              \
154 "       " #mb                                                           \
155         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
156         : #constraint "r" (i)                                           \
157         : cl);                                                          \
158                                                                         \
159         return result;                                                  \
160 }                                                                       \
161 __LL_SC_EXPORT(arch_atomic64_##op##_return##name);
162 
163 #define ATOMIC64_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint)\
164 __LL_SC_INLINE s64                                                      \
165 __LL_SC_PREFIX(arch_atomic64_fetch_##op##name(s64 i, atomic64_t *v))    \
166 {                                                                       \
167         s64 result, val;                                                \
168         unsigned long tmp;                                              \
169                                                                         \
170         asm volatile("// atomic64_fetch_" #op #name "\n"                \
171 "       prfm    pstl1strm, %3\n"                                        \
172 "1:     ld" #acq "xr    %0, %3\n"                                       \
173 "       " #asm_op "     %1, %0, %4\n"                                   \
174 "       st" #rel "xr    %w2, %1, %3\n"                                  \
175 "       cbnz    %w2, 1b\n"                                              \
176 "       " #mb                                                           \
177         : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter)   \
178         : #constraint "r" (i)                                           \
179         : cl);                                                          \
180                                                                         \
181         return result;                                                  \
182 }                                                                       \
183 __LL_SC_EXPORT(arch_atomic64_fetch_##op##name);
184 
185 #define ATOMIC64_OPS(...)                                               \
186         ATOMIC64_OP(__VA_ARGS__)                                        \
187         ATOMIC64_OP_RETURN(, dmb ish,  , l, "memory", __VA_ARGS__)      \
188         ATOMIC64_OP_RETURN(_relaxed,,  ,  ,         , __VA_ARGS__)      \
189         ATOMIC64_OP_RETURN(_acquire,, a,  , "memory", __VA_ARGS__)      \
190         ATOMIC64_OP_RETURN(_release,,  , l, "memory", __VA_ARGS__)      \
191         ATOMIC64_FETCH_OP (, dmb ish,  , l, "memory", __VA_ARGS__)      \
192         ATOMIC64_FETCH_OP (_relaxed,,  ,  ,         , __VA_ARGS__)      \
193         ATOMIC64_FETCH_OP (_acquire,, a,  , "memory", __VA_ARGS__)      \
194         ATOMIC64_FETCH_OP (_release,,  , l, "memory", __VA_ARGS__)
195 
196 ATOMIC64_OPS(add, add, I)
197 ATOMIC64_OPS(sub, sub, J)
198 
199 #undef ATOMIC64_OPS
200 #define ATOMIC64_OPS(...)                                               \
201         ATOMIC64_OP(__VA_ARGS__)                                        \
202         ATOMIC64_FETCH_OP (, dmb ish,  , l, "memory", __VA_ARGS__)      \
203         ATOMIC64_FETCH_OP (_relaxed,,  ,  ,         , __VA_ARGS__)      \
204         ATOMIC64_FETCH_OP (_acquire,, a,  , "memory", __VA_ARGS__)      \
205         ATOMIC64_FETCH_OP (_release,,  , l, "memory", __VA_ARGS__)
206 
207 ATOMIC64_OPS(and, and, L)
208 ATOMIC64_OPS(andnot, bic, )
209 ATOMIC64_OPS(or, orr, L)
210 ATOMIC64_OPS(xor, eor, L)
211 
212 #undef ATOMIC64_OPS
213 #undef ATOMIC64_FETCH_OP
214 #undef ATOMIC64_OP_RETURN
215 #undef ATOMIC64_OP
216 
217 __LL_SC_INLINE s64
218 __LL_SC_PREFIX(arch_atomic64_dec_if_positive(atomic64_t *v))
219 {
220         s64 result;
221         unsigned long tmp;
222 
223         asm volatile("// atomic64_dec_if_positive\n"
224 "       prfm    pstl1strm, %2\n"
225 "1:     ldxr    %0, %2\n"
226 "       subs    %0, %0, #1\n"
227 "       b.lt    2f\n"
228 "       stlxr   %w1, %0, %2\n"
229 "       cbnz    %w1, 1b\n"
230 "       dmb     ish\n"
231 "2:"
232         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
233         :
234         : "cc", "memory");
235 
236         return result;
237 }
238 __LL_SC_EXPORT(arch_atomic64_dec_if_positive);
239 
240 #define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl, constraint)  \
241 __LL_SC_INLINE u##sz                                                    \
242 __LL_SC_PREFIX(__cmpxchg_case_##name##sz(volatile void *ptr,            \
243                                          unsigned long old,             \
244                                          u##sz new))                    \
245 {                                                                       \
246         unsigned long tmp;                                              \
247         u##sz oldval;                                                   \
248                                                                         \
249         /*                                                              \
250          * Sub-word sizes require explicit casting so that the compare  \
251          * part of the cmpxchg doesn't end up interpreting non-zero     \
252          * upper bits of the register containing "old".                 \
253          */                                                             \
254         if (sz < 32)                                                    \
255                 old = (u##sz)old;                                       \
256                                                                         \
257         asm volatile(                                                   \
258         "       prfm    pstl1strm, %[v]\n"                              \
259         "1:     ld" #acq "xr" #sfx "\t%" #w "[oldval], %[v]\n"          \
260         "       eor     %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n"  \
261         "       cbnz    %" #w "[tmp], 2f\n"                             \
262         "       st" #rel "xr" #sfx "\t%w[tmp], %" #w "[new], %[v]\n"    \
263         "       cbnz    %w[tmp], 1b\n"                                  \
264         "       " #mb "\n"                                              \
265         "2:"                                                            \
266         : [tmp] "=&r" (tmp), [oldval] "=&r" (oldval),                   \
267           [v] "+Q" (*(u##sz *)ptr)                                      \
268         : [old] #constraint "r" (old), [new] "r" (new)                  \
269         : cl);                                                          \
270                                                                         \
271         return oldval;                                                  \
272 }                                                                       \
273 __LL_SC_EXPORT(__cmpxchg_case_##name##sz);
274 
275 /*
276  * Earlier versions of GCC (no later than 8.1.0) appear to incorrectly
277  * handle the 'K' constraint for the value 4294967295 - thus we use no
278  * constraint for 32 bit operations.
279  */
280 __CMPXCHG_CASE(w, b,     ,  8,        ,  ,  ,         , )
281 __CMPXCHG_CASE(w, h,     , 16,        ,  ,  ,         , )
282 __CMPXCHG_CASE(w,  ,     , 32,        ,  ,  ,         , )
283 __CMPXCHG_CASE( ,  ,     , 64,        ,  ,  ,         , L)
284 __CMPXCHG_CASE(w, b, acq_,  8,        , a,  , "memory", )
285 __CMPXCHG_CASE(w, h, acq_, 16,        , a,  , "memory", )
286 __CMPXCHG_CASE(w,  , acq_, 32,        , a,  , "memory", )
287 __CMPXCHG_CASE( ,  , acq_, 64,        , a,  , "memory", L)
288 __CMPXCHG_CASE(w, b, rel_,  8,        ,  , l, "memory", )
289 __CMPXCHG_CASE(w, h, rel_, 16,        ,  , l, "memory", )
290 __CMPXCHG_CASE(w,  , rel_, 32,        ,  , l, "memory", )
291 __CMPXCHG_CASE( ,  , rel_, 64,        ,  , l, "memory", L)
292 __CMPXCHG_CASE(w, b,  mb_,  8, dmb ish,  , l, "memory", )
293 __CMPXCHG_CASE(w, h,  mb_, 16, dmb ish,  , l, "memory", )
294 __CMPXCHG_CASE(w,  ,  mb_, 32, dmb ish,  , l, "memory", )
295 __CMPXCHG_CASE( ,  ,  mb_, 64, dmb ish,  , l, "memory", L)
296 
297 #undef __CMPXCHG_CASE
298 
299 #define __CMPXCHG_DBL(name, mb, rel, cl)                                \
300 __LL_SC_INLINE long                                                     \
301 __LL_SC_PREFIX(__cmpxchg_double##name(unsigned long old1,               \
302                                       unsigned long old2,               \
303                                       unsigned long new1,               \
304                                       unsigned long new2,               \
305                                       volatile void *ptr))              \
306 {                                                                       \
307         unsigned long tmp, ret;                                         \
308                                                                         \
309         asm volatile("// __cmpxchg_double" #name "\n"                   \
310         "       prfm    pstl1strm, %2\n"                                \
311         "1:     ldxp    %0, %1, %2\n"                                   \
312         "       eor     %0, %0, %3\n"                                   \
313         "       eor     %1, %1, %4\n"                                   \
314         "       orr     %1, %0, %1\n"                                   \
315         "       cbnz    %1, 2f\n"                                       \
316         "       st" #rel "xp    %w0, %5, %6, %2\n"                      \
317         "       cbnz    %w0, 1b\n"                                      \
318         "       " #mb "\n"                                              \
319         "2:"                                                            \
320         : "=&r" (tmp), "=&r" (ret), "+Q" (*(unsigned long *)ptr)        \
321         : "r" (old1), "r" (old2), "r" (new1), "r" (new2)                \
322         : cl);                                                          \
323                                                                         \
324         return ret;                                                     \
325 }                                                                       \
326 __LL_SC_EXPORT(__cmpxchg_double##name);
327 
328 __CMPXCHG_DBL(   ,        ,  ,         )
329 __CMPXCHG_DBL(_mb, dmb ish, l, "memory")
330 
331 #undef __CMPXCHG_DBL
332 
333 #endif  /* __ASM_ATOMIC_LL_SC_H */
334 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp