~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm/include/asm/atomic.h

Version: ~ [ linux-5.2-rc1 ] ~ [ linux-5.1.2 ] ~ [ linux-5.0.16 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.43 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.119 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.176 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.179 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.139 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.67 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *  arch/arm/include/asm/atomic.h
  3  *
  4  *  Copyright (C) 1996 Russell King.
  5  *  Copyright (C) 2002 Deep Blue Solutions Ltd.
  6  *
  7  * This program is free software; you can redistribute it and/or modify
  8  * it under the terms of the GNU General Public License version 2 as
  9  * published by the Free Software Foundation.
 10  */
 11 #ifndef __ASM_ARM_ATOMIC_H
 12 #define __ASM_ARM_ATOMIC_H
 13 
 14 #include <linux/compiler.h>
 15 #include <linux/types.h>
 16 #include <asm/system.h>
 17 
 18 #define ATOMIC_INIT(i)  { (i) }
 19 
 20 #ifdef __KERNEL__
 21 
 22 /*
 23  * On ARM, ordinary assignment (str instruction) doesn't clear the local
 24  * strex/ldrex monitor on some implementations. The reason we can use it for
 25  * atomic_set() is the clrex or dummy strex done on every exception return.
 26  */
 27 #define atomic_read(v)  (*(volatile int *)&(v)->counter)
 28 #define atomic_set(v,i) (((v)->counter) = (i))
 29 
 30 #if __LINUX_ARM_ARCH__ >= 6
 31 
 32 /*
 33  * ARMv6 UP and SMP safe atomic ops.  We use load exclusive and
 34  * store exclusive to ensure that these are atomic.  We may loop
 35  * to ensure that the update happens.
 36  */
 37 static inline void atomic_add(int i, atomic_t *v)
 38 {
 39         unsigned long tmp;
 40         int result;
 41 
 42         __asm__ __volatile__("@ atomic_add\n"
 43 "1:     ldrex   %0, [%3]\n"
 44 "       add     %0, %0, %4\n"
 45 "       strex   %1, %0, [%3]\n"
 46 "       teq     %1, #0\n"
 47 "       bne     1b"
 48         : "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)
 49         : "r" (&v->counter), "Ir" (i)
 50         : "cc");
 51 }
 52 
 53 static inline int atomic_add_return(int i, atomic_t *v)
 54 {
 55         unsigned long tmp;
 56         int result;
 57 
 58         smp_mb();
 59 
 60         __asm__ __volatile__("@ atomic_add_return\n"
 61 "1:     ldrex   %0, [%3]\n"
 62 "       add     %0, %0, %4\n"
 63 "       strex   %1, %0, [%3]\n"
 64 "       teq     %1, #0\n"
 65 "       bne     1b"
 66         : "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)
 67         : "r" (&v->counter), "Ir" (i)
 68         : "cc");
 69 
 70         smp_mb();
 71 
 72         return result;
 73 }
 74 
 75 static inline void atomic_sub(int i, atomic_t *v)
 76 {
 77         unsigned long tmp;
 78         int result;
 79 
 80         __asm__ __volatile__("@ atomic_sub\n"
 81 "1:     ldrex   %0, [%3]\n"
 82 "       sub     %0, %0, %4\n"
 83 "       strex   %1, %0, [%3]\n"
 84 "       teq     %1, #0\n"
 85 "       bne     1b"
 86         : "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)
 87         : "r" (&v->counter), "Ir" (i)
 88         : "cc");
 89 }
 90 
 91 static inline int atomic_sub_return(int i, atomic_t *v)
 92 {
 93         unsigned long tmp;
 94         int result;
 95 
 96         smp_mb();
 97 
 98         __asm__ __volatile__("@ atomic_sub_return\n"
 99 "1:     ldrex   %0, [%3]\n"
100 "       sub     %0, %0, %4\n"
101 "       strex   %1, %0, [%3]\n"
102 "       teq     %1, #0\n"
103 "       bne     1b"
104         : "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)
105         : "r" (&v->counter), "Ir" (i)
106         : "cc");
107 
108         smp_mb();
109 
110         return result;
111 }
112 
113 static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new)
114 {
115         unsigned long oldval, res;
116 
117         smp_mb();
118 
119         do {
120                 __asm__ __volatile__("@ atomic_cmpxchg\n"
121                 "ldrex  %1, [%3]\n"
122                 "mov    %0, #0\n"
123                 "teq    %1, %4\n"
124                 "strexeq %0, %5, [%3]\n"
125                     : "=&r" (res), "=&r" (oldval), "+Qo" (ptr->counter)
126                     : "r" (&ptr->counter), "Ir" (old), "r" (new)
127                     : "cc");
128         } while (res);
129 
130         smp_mb();
131 
132         return oldval;
133 }
134 
135 static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr)
136 {
137         unsigned long tmp, tmp2;
138 
139         __asm__ __volatile__("@ atomic_clear_mask\n"
140 "1:     ldrex   %0, [%3]\n"
141 "       bic     %0, %0, %4\n"
142 "       strex   %1, %0, [%3]\n"
143 "       teq     %1, #0\n"
144 "       bne     1b"
145         : "=&r" (tmp), "=&r" (tmp2), "+Qo" (*addr)
146         : "r" (addr), "Ir" (mask)
147         : "cc");
148 }
149 
150 #else /* ARM_ARCH_6 */
151 
152 #ifdef CONFIG_SMP
153 #error SMP not supported on pre-ARMv6 CPUs
154 #endif
155 
156 static inline int atomic_add_return(int i, atomic_t *v)
157 {
158         unsigned long flags;
159         int val;
160 
161         raw_local_irq_save(flags);
162         val = v->counter;
163         v->counter = val += i;
164         raw_local_irq_restore(flags);
165 
166         return val;
167 }
168 #define atomic_add(i, v)        (void) atomic_add_return(i, v)
169 
170 static inline int atomic_sub_return(int i, atomic_t *v)
171 {
172         unsigned long flags;
173         int val;
174 
175         raw_local_irq_save(flags);
176         val = v->counter;
177         v->counter = val -= i;
178         raw_local_irq_restore(flags);
179 
180         return val;
181 }
182 #define atomic_sub(i, v)        (void) atomic_sub_return(i, v)
183 
184 static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
185 {
186         int ret;
187         unsigned long flags;
188 
189         raw_local_irq_save(flags);
190         ret = v->counter;
191         if (likely(ret == old))
192                 v->counter = new;
193         raw_local_irq_restore(flags);
194 
195         return ret;
196 }
197 
198 static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr)
199 {
200         unsigned long flags;
201 
202         raw_local_irq_save(flags);
203         *addr &= ~mask;
204         raw_local_irq_restore(flags);
205 }
206 
207 #endif /* __LINUX_ARM_ARCH__ */
208 
209 #define atomic_xchg(v, new) (xchg(&((v)->counter), new))
210 
211 static inline int atomic_add_unless(atomic_t *v, int a, int u)
212 {
213         int c, old;
214 
215         c = atomic_read(v);
216         while (c != u && (old = atomic_cmpxchg((v), c, c + a)) != c)
217                 c = old;
218         return c != u;
219 }
220 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
221 
222 #define atomic_inc(v)           atomic_add(1, v)
223 #define atomic_dec(v)           atomic_sub(1, v)
224 
225 #define atomic_inc_and_test(v)  (atomic_add_return(1, v) == 0)
226 #define atomic_dec_and_test(v)  (atomic_sub_return(1, v) == 0)
227 #define atomic_inc_return(v)    (atomic_add_return(1, v))
228 #define atomic_dec_return(v)    (atomic_sub_return(1, v))
229 #define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0)
230 
231 #define atomic_add_negative(i,v) (atomic_add_return(i, v) < 0)
232 
233 #define smp_mb__before_atomic_dec()     smp_mb()
234 #define smp_mb__after_atomic_dec()      smp_mb()
235 #define smp_mb__before_atomic_inc()     smp_mb()
236 #define smp_mb__after_atomic_inc()      smp_mb()
237 
238 #ifndef CONFIG_GENERIC_ATOMIC64
239 typedef struct {
240         u64 __aligned(8) counter;
241 } atomic64_t;
242 
243 #define ATOMIC64_INIT(i) { (i) }
244 
245 static inline u64 atomic64_read(atomic64_t *v)
246 {
247         u64 result;
248 
249         __asm__ __volatile__("@ atomic64_read\n"
250 "       ldrexd  %0, %H0, [%1]"
251         : "=&r" (result)
252         : "r" (&v->counter), "Qo" (v->counter)
253         );
254 
255         return result;
256 }
257 
258 static inline void atomic64_set(atomic64_t *v, u64 i)
259 {
260         u64 tmp;
261 
262         __asm__ __volatile__("@ atomic64_set\n"
263 "1:     ldrexd  %0, %H0, [%2]\n"
264 "       strexd  %0, %3, %H3, [%2]\n"
265 "       teq     %0, #0\n"
266 "       bne     1b"
267         : "=&r" (tmp), "=Qo" (v->counter)
268         : "r" (&v->counter), "r" (i)
269         : "cc");
270 }
271 
272 static inline void atomic64_add(u64 i, atomic64_t *v)
273 {
274         u64 result;
275         unsigned long tmp;
276 
277         __asm__ __volatile__("@ atomic64_add\n"
278 "1:     ldrexd  %0, %H0, [%3]\n"
279 "       adds    %0, %0, %4\n"
280 "       adc     %H0, %H0, %H4\n"
281 "       strexd  %1, %0, %H0, [%3]\n"
282 "       teq     %1, #0\n"
283 "       bne     1b"
284         : "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)
285         : "r" (&v->counter), "r" (i)
286         : "cc");
287 }
288 
289 static inline u64 atomic64_add_return(u64 i, atomic64_t *v)
290 {
291         u64 result;
292         unsigned long tmp;
293 
294         smp_mb();
295 
296         __asm__ __volatile__("@ atomic64_add_return\n"
297 "1:     ldrexd  %0, %H0, [%3]\n"
298 "       adds    %0, %0, %4\n"
299 "       adc     %H0, %H0, %H4\n"
300 "       strexd  %1, %0, %H0, [%3]\n"
301 "       teq     %1, #0\n"
302 "       bne     1b"
303         : "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)
304         : "r" (&v->counter), "r" (i)
305         : "cc");
306 
307         smp_mb();
308 
309         return result;
310 }
311 
312 static inline void atomic64_sub(u64 i, atomic64_t *v)
313 {
314         u64 result;
315         unsigned long tmp;
316 
317         __asm__ __volatile__("@ atomic64_sub\n"
318 "1:     ldrexd  %0, %H0, [%3]\n"
319 "       subs    %0, %0, %4\n"
320 "       sbc     %H0, %H0, %H4\n"
321 "       strexd  %1, %0, %H0, [%3]\n"
322 "       teq     %1, #0\n"
323 "       bne     1b"
324         : "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)
325         : "r" (&v->counter), "r" (i)
326         : "cc");
327 }
328 
329 static inline u64 atomic64_sub_return(u64 i, atomic64_t *v)
330 {
331         u64 result;
332         unsigned long tmp;
333 
334         smp_mb();
335 
336         __asm__ __volatile__("@ atomic64_sub_return\n"
337 "1:     ldrexd  %0, %H0, [%3]\n"
338 "       subs    %0, %0, %4\n"
339 "       sbc     %H0, %H0, %H4\n"
340 "       strexd  %1, %0, %H0, [%3]\n"
341 "       teq     %1, #0\n"
342 "       bne     1b"
343         : "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)
344         : "r" (&v->counter), "r" (i)
345         : "cc");
346 
347         smp_mb();
348 
349         return result;
350 }
351 
352 static inline u64 atomic64_cmpxchg(atomic64_t *ptr, u64 old, u64 new)
353 {
354         u64 oldval;
355         unsigned long res;
356 
357         smp_mb();
358 
359         do {
360                 __asm__ __volatile__("@ atomic64_cmpxchg\n"
361                 "ldrexd         %1, %H1, [%3]\n"
362                 "mov            %0, #0\n"
363                 "teq            %1, %4\n"
364                 "teqeq          %H1, %H4\n"
365                 "strexdeq       %0, %5, %H5, [%3]"
366                 : "=&r" (res), "=&r" (oldval), "+Qo" (ptr->counter)
367                 : "r" (&ptr->counter), "r" (old), "r" (new)
368                 : "cc");
369         } while (res);
370 
371         smp_mb();
372 
373         return oldval;
374 }
375 
376 static inline u64 atomic64_xchg(atomic64_t *ptr, u64 new)
377 {
378         u64 result;
379         unsigned long tmp;
380 
381         smp_mb();
382 
383         __asm__ __volatile__("@ atomic64_xchg\n"
384 "1:     ldrexd  %0, %H0, [%3]\n"
385 "       strexd  %1, %4, %H4, [%3]\n"
386 "       teq     %1, #0\n"
387 "       bne     1b"
388         : "=&r" (result), "=&r" (tmp), "+Qo" (ptr->counter)
389         : "r" (&ptr->counter), "r" (new)
390         : "cc");
391 
392         smp_mb();
393 
394         return result;
395 }
396 
397 static inline u64 atomic64_dec_if_positive(atomic64_t *v)
398 {
399         u64 result;
400         unsigned long tmp;
401 
402         smp_mb();
403 
404         __asm__ __volatile__("@ atomic64_dec_if_positive\n"
405 "1:     ldrexd  %0, %H0, [%3]\n"
406 "       subs    %0, %0, #1\n"
407 "       sbc     %H0, %H0, #0\n"
408 "       teq     %H0, #0\n"
409 "       bmi     2f\n"
410 "       strexd  %1, %0, %H0, [%3]\n"
411 "       teq     %1, #0\n"
412 "       bne     1b\n"
413 "2:"
414         : "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)
415         : "r" (&v->counter)
416         : "cc");
417 
418         smp_mb();
419 
420         return result;
421 }
422 
423 static inline int atomic64_add_unless(atomic64_t *v, u64 a, u64 u)
424 {
425         u64 val;
426         unsigned long tmp;
427         int ret = 1;
428 
429         smp_mb();
430 
431         __asm__ __volatile__("@ atomic64_add_unless\n"
432 "1:     ldrexd  %0, %H0, [%4]\n"
433 "       teq     %0, %5\n"
434 "       teqeq   %H0, %H5\n"
435 "       moveq   %1, #0\n"
436 "       beq     2f\n"
437 "       adds    %0, %0, %6\n"
438 "       adc     %H0, %H0, %H6\n"
439 "       strexd  %2, %0, %H0, [%4]\n"
440 "       teq     %2, #0\n"
441 "       bne     1b\n"
442 "2:"
443         : "=&r" (val), "+r" (ret), "=&r" (tmp), "+Qo" (v->counter)
444         : "r" (&v->counter), "r" (u), "r" (a)
445         : "cc");
446 
447         if (ret)
448                 smp_mb();
449 
450         return ret;
451 }
452 
453 #define atomic64_add_negative(a, v)     (atomic64_add_return((a), (v)) < 0)
454 #define atomic64_inc(v)                 atomic64_add(1LL, (v))
455 #define atomic64_inc_return(v)          atomic64_add_return(1LL, (v))
456 #define atomic64_inc_and_test(v)        (atomic64_inc_return(v) == 0)
457 #define atomic64_sub_and_test(a, v)     (atomic64_sub_return((a), (v)) == 0)
458 #define atomic64_dec(v)                 atomic64_sub(1LL, (v))
459 #define atomic64_dec_return(v)          atomic64_sub_return(1LL, (v))
460 #define atomic64_dec_and_test(v)        (atomic64_dec_return((v)) == 0)
461 #define atomic64_inc_not_zero(v)        atomic64_add_unless((v), 1LL, 0LL)
462 
463 #else /* !CONFIG_GENERIC_ATOMIC64 */
464 #include <asm-generic/atomic64.h>
465 #endif
466 #include <asm-generic/atomic-long.h>
467 #endif
468 #endif
469 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp