~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arc/include/asm/cmpxchg.h

Version: ~ [ linux-5.15-rc1 ] ~ [ linux-5.14.5 ] ~ [ linux-5.13.18 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.66 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.147 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.206 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.246 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.282 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.283 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
  3  *
  4  * This program is free software; you can redistribute it and/or modify
  5  * it under the terms of the GNU General Public License version 2 as
  6  * published by the Free Software Foundation.
  7  */
  8 
  9 #ifndef __ASM_ARC_CMPXCHG_H
 10 #define __ASM_ARC_CMPXCHG_H
 11 
 12 #include <linux/types.h>
 13 
 14 #include <asm/barrier.h>
 15 #include <asm/smp.h>
 16 
 17 #ifdef CONFIG_ARC_HAS_LLSC
 18 
 19 static inline unsigned long
 20 __cmpxchg(volatile void *ptr, unsigned long expected, unsigned long new)
 21 {
 22         unsigned long prev;
 23 
 24         /*
 25          * Explicit full memory barrier needed before/after as
 26          * LLOCK/SCOND thmeselves don't provide any such semantics
 27          */
 28         smp_mb();
 29 
 30         __asm__ __volatile__(
 31         "1:     llock   %0, [%1]        \n"
 32         "       brne    %0, %2, 2f      \n"
 33         "       scond   %3, [%1]        \n"
 34         "       bnz     1b              \n"
 35         "2:                             \n"
 36         : "=&r"(prev)   /* Early clobber, to prevent reg reuse */
 37         : "r"(ptr),     /* Not "m": llock only supports reg direct addr mode */
 38           "ir"(expected),
 39           "r"(new)      /* can't be "ir". scond can't take LIMM for "b" */
 40         : "cc", "memory"); /* so that gcc knows memory is being written here */
 41 
 42         smp_mb();
 43 
 44         return prev;
 45 }
 46 
 47 #elif !defined(CONFIG_ARC_PLAT_EZNPS)
 48 
 49 static inline unsigned long
 50 __cmpxchg(volatile void *ptr, unsigned long expected, unsigned long new)
 51 {
 52         unsigned long flags;
 53         int prev;
 54         volatile unsigned long *p = ptr;
 55 
 56         /*
 57          * spin lock/unlock provide the needed smp_mb() before/after
 58          */
 59         atomic_ops_lock(flags);
 60         prev = *p;
 61         if (prev == expected)
 62                 *p = new;
 63         atomic_ops_unlock(flags);
 64         return prev;
 65 }
 66 
 67 #else /* CONFIG_ARC_PLAT_EZNPS */
 68 
 69 static inline unsigned long
 70 __cmpxchg(volatile void *ptr, unsigned long expected, unsigned long new)
 71 {
 72         /*
 73          * Explicit full memory barrier needed before/after
 74          */
 75         smp_mb();
 76 
 77         write_aux_reg(CTOP_AUX_GPA1, expected);
 78 
 79         __asm__ __volatile__(
 80         "       mov r2, %0\n"
 81         "       mov r3, %1\n"
 82         "       .word %2\n"
 83         "       mov %0, r2"
 84         : "+r"(new)
 85         : "r"(ptr), "i"(CTOP_INST_EXC_DI_R2_R2_R3)
 86         : "r2", "r3", "memory");
 87 
 88         smp_mb();
 89 
 90         return new;
 91 }
 92 
 93 #endif /* CONFIG_ARC_HAS_LLSC */
 94 
 95 #define cmpxchg(ptr, o, n) ((typeof(*(ptr)))__cmpxchg((ptr), \
 96                                 (unsigned long)(o), (unsigned long)(n)))
 97 
 98 /*
 99  * atomic_cmpxchg is same as cmpxchg
100  *   LLSC: only different in data-type, semantics are exactly same
101  *  !LLSC: cmpxchg() has to use an external lock atomic_ops_lock to guarantee
102  *         semantics, and this lock also happens to be used by atomic_*()
103  */
104 #define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
105 
106 
107 #ifndef CONFIG_ARC_PLAT_EZNPS
108 
109 /*
110  * xchg (reg with memory) based on "Native atomic" EX insn
111  */
112 static inline unsigned long __xchg(unsigned long val, volatile void *ptr,
113                                    int size)
114 {
115         extern unsigned long __xchg_bad_pointer(void);
116 
117         switch (size) {
118         case 4:
119                 smp_mb();
120 
121                 __asm__ __volatile__(
122                 "       ex  %0, [%1]    \n"
123                 : "+r"(val)
124                 : "r"(ptr)
125                 : "memory");
126 
127                 smp_mb();
128 
129                 return val;
130         }
131         return __xchg_bad_pointer();
132 }
133 
134 #define _xchg(ptr, with) ((typeof(*(ptr)))__xchg((unsigned long)(with), (ptr), \
135                                                  sizeof(*(ptr))))
136 
137 /*
138  * xchg() maps directly to ARC EX instruction which guarantees atomicity.
139  * However in !LLSC config, it also needs to be use @atomic_ops_lock spinlock
140  * due to a subtle reason:
141  *  - For !LLSC, cmpxchg() needs to use that lock (see above) and there is lot
142  *    of  kernel code which calls xchg()/cmpxchg() on same data (see llist.h)
143  *    Hence xchg() needs to follow same locking rules.
144  *
145  * Technically the lock is also needed for UP (boils down to irq save/restore)
146  * but we can cheat a bit since cmpxchg() atomic_ops_lock() would cause irqs to
147  * be disabled thus can't possibly be interrpted/preempted/clobbered by xchg()
148  * Other way around, xchg is one instruction anyways, so can't be interrupted
149  * as such
150  */
151 
152 #if !defined(CONFIG_ARC_HAS_LLSC) && defined(CONFIG_SMP)
153 
154 #define xchg(ptr, with)                 \
155 ({                                      \
156         unsigned long flags;            \
157         typeof(*(ptr)) old_val;         \
158                                         \
159         atomic_ops_lock(flags);         \
160         old_val = _xchg(ptr, with);     \
161         atomic_ops_unlock(flags);       \
162         old_val;                        \
163 })
164 
165 #else
166 
167 #define xchg(ptr, with)  _xchg(ptr, with)
168 
169 #endif
170 
171 #else /* CONFIG_ARC_PLAT_EZNPS */
172 
173 static inline unsigned long __xchg(unsigned long val, volatile void *ptr,
174                                    int size)
175 {
176         extern unsigned long __xchg_bad_pointer(void);
177 
178         switch (size) {
179         case 4:
180                 /*
181                  * Explicit full memory barrier needed before/after
182                  */
183                 smp_mb();
184 
185                 __asm__ __volatile__(
186                 "       mov r2, %0\n"
187                 "       mov r3, %1\n"
188                 "       .word %2\n"
189                 "       mov %0, r2\n"
190                 : "+r"(val)
191                 : "r"(ptr), "i"(CTOP_INST_XEX_DI_R2_R2_R3)
192                 : "r2", "r3", "memory");
193 
194                 smp_mb();
195 
196                 return val;
197         }
198         return __xchg_bad_pointer();
199 }
200 
201 #define xchg(ptr, with) ((typeof(*(ptr)))__xchg((unsigned long)(with), (ptr), \
202                                                  sizeof(*(ptr))))
203 
204 #endif /* CONFIG_ARC_PLAT_EZNPS */
205 
206 /*
207  * "atomic" variant of xchg()
208  * REQ: It needs to follow the same serialization rules as other atomic_xxx()
209  * Since xchg() doesn't always do that, it would seem that following defintion
210  * is incorrect. But here's the rationale:
211  *   SMP : Even xchg() takes the atomic_ops_lock, so OK.
212  *   LLSC: atomic_ops_lock are not relevant at all (even if SMP, since LLSC
213  *         is natively "SMP safe", no serialization required).
214  *   UP  : other atomics disable IRQ, so no way a difft ctxt atomic_xchg()
215  *         could clobber them. atomic_xchg() itself would be 1 insn, so it
216  *         can't be clobbered by others. Thus no serialization required when
217  *         atomic_xchg is involved.
218  */
219 #define atomic_xchg(v, new) (xchg(&((v)->counter), new))
220 
221 #endif
222 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp