~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/include/asm-generic/barrier.h

Version: ~ [ linux-5.16 ] ~ [ linux-5.15.13 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.90 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.170 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.224 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.261 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.296 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.298 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Generic barrier definitions.
  3  *
  4  * It should be possible to use these on really simple architectures,
  5  * but it serves more as a starting point for new ports.
  6  *
  7  * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
  8  * Written by David Howells (dhowells@redhat.com)
  9  *
 10  * This program is free software; you can redistribute it and/or
 11  * modify it under the terms of the GNU General Public Licence
 12  * as published by the Free Software Foundation; either version
 13  * 2 of the Licence, or (at your option) any later version.
 14  */
 15 #ifndef __ASM_GENERIC_BARRIER_H
 16 #define __ASM_GENERIC_BARRIER_H
 17 
 18 #ifndef __ASSEMBLY__
 19 
 20 #include <linux/compiler.h>
 21 
 22 #ifndef nop
 23 #define nop()   asm volatile ("nop")
 24 #endif
 25 
 26 /*
 27  * Force strict CPU ordering. And yes, this is required on UP too when we're
 28  * talking to devices.
 29  *
 30  * Fall back to compiler barriers if nothing better is provided.
 31  */
 32 
 33 #ifndef mb
 34 #define mb()    barrier()
 35 #endif
 36 
 37 #ifndef rmb
 38 #define rmb()   mb()
 39 #endif
 40 
 41 #ifndef wmb
 42 #define wmb()   mb()
 43 #endif
 44 
 45 #ifndef dma_rmb
 46 #define dma_rmb()       rmb()
 47 #endif
 48 
 49 #ifndef dma_wmb
 50 #define dma_wmb()       wmb()
 51 #endif
 52 
 53 #ifndef read_barrier_depends
 54 #define read_barrier_depends()          do { } while (0)
 55 #endif
 56 
 57 #ifndef __smp_mb
 58 #define __smp_mb()      mb()
 59 #endif
 60 
 61 #ifndef __smp_rmb
 62 #define __smp_rmb()     rmb()
 63 #endif
 64 
 65 #ifndef __smp_wmb
 66 #define __smp_wmb()     wmb()
 67 #endif
 68 
 69 #ifndef __smp_read_barrier_depends
 70 #define __smp_read_barrier_depends()    read_barrier_depends()
 71 #endif
 72 
 73 #ifdef CONFIG_SMP
 74 
 75 #ifndef smp_mb
 76 #define smp_mb()        __smp_mb()
 77 #endif
 78 
 79 #ifndef smp_rmb
 80 #define smp_rmb()       __smp_rmb()
 81 #endif
 82 
 83 #ifndef smp_wmb
 84 #define smp_wmb()       __smp_wmb()
 85 #endif
 86 
 87 #ifndef smp_read_barrier_depends
 88 #define smp_read_barrier_depends()      __smp_read_barrier_depends()
 89 #endif
 90 
 91 #else   /* !CONFIG_SMP */
 92 
 93 #ifndef smp_mb
 94 #define smp_mb()        barrier()
 95 #endif
 96 
 97 #ifndef smp_rmb
 98 #define smp_rmb()       barrier()
 99 #endif
100 
101 #ifndef smp_wmb
102 #define smp_wmb()       barrier()
103 #endif
104 
105 #ifndef smp_read_barrier_depends
106 #define smp_read_barrier_depends()      do { } while (0)
107 #endif
108 
109 #endif  /* CONFIG_SMP */
110 
111 #ifndef __smp_store_mb
112 #define __smp_store_mb(var, value)  do { WRITE_ONCE(var, value); __smp_mb(); } while (0)
113 #endif
114 
115 #ifndef __smp_mb__before_atomic
116 #define __smp_mb__before_atomic()       __smp_mb()
117 #endif
118 
119 #ifndef __smp_mb__after_atomic
120 #define __smp_mb__after_atomic()        __smp_mb()
121 #endif
122 
123 #ifndef __smp_store_release
124 #define __smp_store_release(p, v)                                       \
125 do {                                                                    \
126         compiletime_assert_atomic_type(*p);                             \
127         __smp_mb();                                                     \
128         WRITE_ONCE(*p, v);                                              \
129 } while (0)
130 #endif
131 
132 #ifndef __smp_load_acquire
133 #define __smp_load_acquire(p)                                           \
134 ({                                                                      \
135         typeof(*p) ___p1 = READ_ONCE(*p);                               \
136         compiletime_assert_atomic_type(*p);                             \
137         __smp_mb();                                                     \
138         ___p1;                                                          \
139 })
140 #endif
141 
142 #ifdef CONFIG_SMP
143 
144 #ifndef smp_store_mb
145 #define smp_store_mb(var, value)  __smp_store_mb(var, value)
146 #endif
147 
148 #ifndef smp_mb__before_atomic
149 #define smp_mb__before_atomic() __smp_mb__before_atomic()
150 #endif
151 
152 #ifndef smp_mb__after_atomic
153 #define smp_mb__after_atomic()  __smp_mb__after_atomic()
154 #endif
155 
156 #ifndef smp_store_release
157 #define smp_store_release(p, v) __smp_store_release(p, v)
158 #endif
159 
160 #ifndef smp_load_acquire
161 #define smp_load_acquire(p) __smp_load_acquire(p)
162 #endif
163 
164 #else   /* !CONFIG_SMP */
165 
166 #ifndef smp_store_mb
167 #define smp_store_mb(var, value)  do { WRITE_ONCE(var, value); barrier(); } while (0)
168 #endif
169 
170 #ifndef smp_mb__before_atomic
171 #define smp_mb__before_atomic() barrier()
172 #endif
173 
174 #ifndef smp_mb__after_atomic
175 #define smp_mb__after_atomic()  barrier()
176 #endif
177 
178 #ifndef smp_store_release
179 #define smp_store_release(p, v)                                         \
180 do {                                                                    \
181         compiletime_assert_atomic_type(*p);                             \
182         barrier();                                                      \
183         WRITE_ONCE(*p, v);                                              \
184 } while (0)
185 #endif
186 
187 #ifndef smp_load_acquire
188 #define smp_load_acquire(p)                                             \
189 ({                                                                      \
190         typeof(*p) ___p1 = READ_ONCE(*p);                               \
191         compiletime_assert_atomic_type(*p);                             \
192         barrier();                                                      \
193         ___p1;                                                          \
194 })
195 #endif
196 
197 #endif  /* CONFIG_SMP */
198 
199 /* Barriers for virtual machine guests when talking to an SMP host */
200 #define virt_mb() __smp_mb()
201 #define virt_rmb() __smp_rmb()
202 #define virt_wmb() __smp_wmb()
203 #define virt_read_barrier_depends() __smp_read_barrier_depends()
204 #define virt_store_mb(var, value) __smp_store_mb(var, value)
205 #define virt_mb__before_atomic() __smp_mb__before_atomic()
206 #define virt_mb__after_atomic() __smp_mb__after_atomic()
207 #define virt_store_release(p, v) __smp_store_release(p, v)
208 #define virt_load_acquire(p) __smp_load_acquire(p)
209 
210 /**
211  * smp_acquire__after_ctrl_dep() - Provide ACQUIRE ordering after a control dependency
212  *
213  * A control dependency provides a LOAD->STORE order, the additional RMB
214  * provides LOAD->LOAD order, together they provide LOAD->{LOAD,STORE} order,
215  * aka. (load)-ACQUIRE.
216  *
217  * Architectures that do not do load speculation can have this be barrier().
218  */
219 #ifndef smp_acquire__after_ctrl_dep
220 #define smp_acquire__after_ctrl_dep()           smp_rmb()
221 #endif
222 
223 /**
224  * smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees
225  * @ptr: pointer to the variable to wait on
226  * @cond: boolean expression to wait for
227  *
228  * Equivalent to using READ_ONCE() on the condition variable.
229  *
230  * Due to C lacking lambda expressions we load the value of *ptr into a
231  * pre-named variable @VAL to be used in @cond.
232  */
233 #ifndef smp_cond_load_relaxed
234 #define smp_cond_load_relaxed(ptr, cond_expr) ({                \
235         typeof(ptr) __PTR = (ptr);                              \
236         typeof(*ptr) VAL;                                       \
237         for (;;) {                                              \
238                 VAL = READ_ONCE(*__PTR);                        \
239                 if (cond_expr)                                  \
240                         break;                                  \
241                 cpu_relax();                                    \
242         }                                                       \
243         VAL;                                                    \
244 })
245 #endif
246 
247 /**
248  * smp_cond_load_acquire() - (Spin) wait for cond with ACQUIRE ordering
249  * @ptr: pointer to the variable to wait on
250  * @cond: boolean expression to wait for
251  *
252  * Equivalent to using smp_load_acquire() on the condition variable but employs
253  * the control dependency of the wait to reduce the barrier on many platforms.
254  */
255 #ifndef smp_cond_load_acquire
256 #define smp_cond_load_acquire(ptr, cond_expr) ({                \
257         typeof(*ptr) _val;                                      \
258         _val = smp_cond_load_relaxed(ptr, cond_expr);           \
259         smp_acquire__after_ctrl_dep();                          \
260         _val;                                                   \
261 })
262 #endif
263 
264 #endif /* !__ASSEMBLY__ */
265 #endif /* __ASM_GENERIC_BARRIER_H */
266 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp