~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm64/include/asm/percpu.h

Version: ~ [ linux-5.4.2 ] ~ [ linux-5.3.15 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.88 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.158 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.206 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.206 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.78 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Copyright (C) 2013 ARM Ltd.
  3  *
  4  * This program is free software; you can redistribute it and/or modify
  5  * it under the terms of the GNU General Public License version 2 as
  6  * published by the Free Software Foundation.
  7  *
  8  * This program is distributed in the hope that it will be useful,
  9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 11  * GNU General Public License for more details.
 12  *
 13  * You should have received a copy of the GNU General Public License
 14  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 15  */
 16 #ifndef __ASM_PERCPU_H
 17 #define __ASM_PERCPU_H
 18 
 19 static inline void set_my_cpu_offset(unsigned long off)
 20 {
 21         asm volatile("msr tpidr_el1, %0" :: "r" (off) : "memory");
 22 }
 23 
 24 static inline unsigned long __my_cpu_offset(void)
 25 {
 26         unsigned long off;
 27 
 28         /*
 29          * We want to allow caching the value, so avoid using volatile and
 30          * instead use a fake stack read to hazard against barrier().
 31          */
 32         asm("mrs %0, tpidr_el1" : "=r" (off) :
 33                 "Q" (*(const unsigned long *)current_stack_pointer));
 34 
 35         return off;
 36 }
 37 #define __my_cpu_offset __my_cpu_offset()
 38 
 39 #define PERCPU_OP(op, asm_op)                                           \
 40 static inline unsigned long __percpu_##op(void *ptr,                    \
 41                         unsigned long val, int size)                    \
 42 {                                                                       \
 43         unsigned long loop, ret;                                        \
 44                                                                         \
 45         switch (size) {                                                 \
 46         case 1:                                                         \
 47                 do {                                                    \
 48                         asm ("//__per_cpu_" #op "_1\n"                  \
 49                         "ldxrb    %w[ret], %[ptr]\n"                    \
 50                         #asm_op " %w[ret], %w[ret], %w[val]\n"          \
 51                         "stxrb    %w[loop], %w[ret], %[ptr]\n"          \
 52                         : [loop] "=&r" (loop), [ret] "=&r" (ret),       \
 53                           [ptr] "+Q"(*(u8 *)ptr)                        \
 54                         : [val] "Ir" (val));                            \
 55                 } while (loop);                                         \
 56                 break;                                                  \
 57         case 2:                                                         \
 58                 do {                                                    \
 59                         asm ("//__per_cpu_" #op "_2\n"                  \
 60                         "ldxrh    %w[ret], %[ptr]\n"                    \
 61                         #asm_op " %w[ret], %w[ret], %w[val]\n"          \
 62                         "stxrh    %w[loop], %w[ret], %[ptr]\n"          \
 63                         : [loop] "=&r" (loop), [ret] "=&r" (ret),       \
 64                           [ptr]  "+Q"(*(u16 *)ptr)                      \
 65                         : [val] "Ir" (val));                            \
 66                 } while (loop);                                         \
 67                 break;                                                  \
 68         case 4:                                                         \
 69                 do {                                                    \
 70                         asm ("//__per_cpu_" #op "_4\n"                  \
 71                         "ldxr     %w[ret], %[ptr]\n"                    \
 72                         #asm_op " %w[ret], %w[ret], %w[val]\n"          \
 73                         "stxr     %w[loop], %w[ret], %[ptr]\n"          \
 74                         : [loop] "=&r" (loop), [ret] "=&r" (ret),       \
 75                           [ptr] "+Q"(*(u32 *)ptr)                       \
 76                         : [val] "Ir" (val));                            \
 77                 } while (loop);                                         \
 78                 break;                                                  \
 79         case 8:                                                         \
 80                 do {                                                    \
 81                         asm ("//__per_cpu_" #op "_8\n"                  \
 82                         "ldxr     %[ret], %[ptr]\n"                     \
 83                         #asm_op " %[ret], %[ret], %[val]\n"             \
 84                         "stxr     %w[loop], %[ret], %[ptr]\n"           \
 85                         : [loop] "=&r" (loop), [ret] "=&r" (ret),       \
 86                           [ptr] "+Q"(*(u64 *)ptr)                       \
 87                         : [val] "Ir" (val));                            \
 88                 } while (loop);                                         \
 89                 break;                                                  \
 90         default:                                                        \
 91                 BUILD_BUG();                                            \
 92         }                                                               \
 93                                                                         \
 94         return ret;                                                     \
 95 }
 96 
 97 PERCPU_OP(add, add)
 98 PERCPU_OP(and, and)
 99 PERCPU_OP(or, orr)
100 #undef PERCPU_OP
101 
102 static inline unsigned long __percpu_read(void *ptr, int size)
103 {
104         unsigned long ret;
105 
106         switch (size) {
107         case 1:
108                 ret = ACCESS_ONCE(*(u8 *)ptr);
109                 break;
110         case 2:
111                 ret = ACCESS_ONCE(*(u16 *)ptr);
112                 break;
113         case 4:
114                 ret = ACCESS_ONCE(*(u32 *)ptr);
115                 break;
116         case 8:
117                 ret = ACCESS_ONCE(*(u64 *)ptr);
118                 break;
119         default:
120                 BUILD_BUG();
121         }
122 
123         return ret;
124 }
125 
126 static inline void __percpu_write(void *ptr, unsigned long val, int size)
127 {
128         switch (size) {
129         case 1:
130                 ACCESS_ONCE(*(u8 *)ptr) = (u8)val;
131                 break;
132         case 2:
133                 ACCESS_ONCE(*(u16 *)ptr) = (u16)val;
134                 break;
135         case 4:
136                 ACCESS_ONCE(*(u32 *)ptr) = (u32)val;
137                 break;
138         case 8:
139                 ACCESS_ONCE(*(u64 *)ptr) = (u64)val;
140                 break;
141         default:
142                 BUILD_BUG();
143         }
144 }
145 
146 static inline unsigned long __percpu_xchg(void *ptr, unsigned long val,
147                                                 int size)
148 {
149         unsigned long ret, loop;
150 
151         switch (size) {
152         case 1:
153                 do {
154                         asm ("//__percpu_xchg_1\n"
155                         "ldxrb %w[ret], %[ptr]\n"
156                         "stxrb %w[loop], %w[val], %[ptr]\n"
157                         : [loop] "=&r"(loop), [ret] "=&r"(ret),
158                           [ptr] "+Q"(*(u8 *)ptr)
159                         : [val] "r" (val));
160                 } while (loop);
161                 break;
162         case 2:
163                 do {
164                         asm ("//__percpu_xchg_2\n"
165                         "ldxrh %w[ret], %[ptr]\n"
166                         "stxrh %w[loop], %w[val], %[ptr]\n"
167                         : [loop] "=&r"(loop), [ret] "=&r"(ret),
168                           [ptr] "+Q"(*(u16 *)ptr)
169                         : [val] "r" (val));
170                 } while (loop);
171                 break;
172         case 4:
173                 do {
174                         asm ("//__percpu_xchg_4\n"
175                         "ldxr %w[ret], %[ptr]\n"
176                         "stxr %w[loop], %w[val], %[ptr]\n"
177                         : [loop] "=&r"(loop), [ret] "=&r"(ret),
178                           [ptr] "+Q"(*(u32 *)ptr)
179                         : [val] "r" (val));
180                 } while (loop);
181                 break;
182         case 8:
183                 do {
184                         asm ("//__percpu_xchg_8\n"
185                         "ldxr %[ret], %[ptr]\n"
186                         "stxr %w[loop], %[val], %[ptr]\n"
187                         : [loop] "=&r"(loop), [ret] "=&r"(ret),
188                           [ptr] "+Q"(*(u64 *)ptr)
189                         : [val] "r" (val));
190                 } while (loop);
191                 break;
192         default:
193                 BUILD_BUG();
194         }
195 
196         return ret;
197 }
198 
199 #define _percpu_read(pcp)                                               \
200 ({                                                                      \
201         typeof(pcp) __retval;                                           \
202         preempt_disable();                                              \
203         __retval = (typeof(pcp))__percpu_read(raw_cpu_ptr(&(pcp)),      \
204                                               sizeof(pcp));             \
205         preempt_enable();                                               \
206         __retval;                                                       \
207 })
208 
209 #define _percpu_write(pcp, val)                                         \
210 do {                                                                    \
211         preempt_disable();                                              \
212         __percpu_write(raw_cpu_ptr(&(pcp)), (unsigned long)(val),       \
213                                 sizeof(pcp));                           \
214         preempt_enable();                                               \
215 } while(0)                                                              \
216 
217 #define _pcp_protect(operation, pcp, val)                       \
218 ({                                                              \
219         typeof(pcp) __retval;                                   \
220         preempt_disable();                                      \
221         __retval = (typeof(pcp))operation(raw_cpu_ptr(&(pcp)),  \
222                                           (val), sizeof(pcp));  \
223         preempt_enable();                                       \
224         __retval;                                               \
225 })
226 
227 #define _percpu_add(pcp, val) \
228         _pcp_protect(__percpu_add, pcp, val)
229 
230 #define _percpu_add_return(pcp, val) _percpu_add(pcp, val)
231 
232 #define _percpu_and(pcp, val) \
233         _pcp_protect(__percpu_and, pcp, val)
234 
235 #define _percpu_or(pcp, val) \
236         _pcp_protect(__percpu_or, pcp, val)
237 
238 #define _percpu_xchg(pcp, val) (typeof(pcp)) \
239         _pcp_protect(__percpu_xchg, pcp, (unsigned long)(val))
240 
241 #define this_cpu_add_1(pcp, val) _percpu_add(pcp, val)
242 #define this_cpu_add_2(pcp, val) _percpu_add(pcp, val)
243 #define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
244 #define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
245 
246 #define this_cpu_add_return_1(pcp, val) _percpu_add_return(pcp, val)
247 #define this_cpu_add_return_2(pcp, val) _percpu_add_return(pcp, val)
248 #define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
249 #define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
250 
251 #define this_cpu_and_1(pcp, val) _percpu_and(pcp, val)
252 #define this_cpu_and_2(pcp, val) _percpu_and(pcp, val)
253 #define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
254 #define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
255 
256 #define this_cpu_or_1(pcp, val) _percpu_or(pcp, val)
257 #define this_cpu_or_2(pcp, val) _percpu_or(pcp, val)
258 #define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
259 #define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
260 
261 #define this_cpu_read_1(pcp) _percpu_read(pcp)
262 #define this_cpu_read_2(pcp) _percpu_read(pcp)
263 #define this_cpu_read_4(pcp) _percpu_read(pcp)
264 #define this_cpu_read_8(pcp) _percpu_read(pcp)
265 
266 #define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
267 #define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
268 #define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
269 #define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)
270 
271 #define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
272 #define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
273 #define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
274 #define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
275 
276 #include <asm-generic/percpu.h>
277 
278 #endif /* __ASM_PERCPU_H */
279 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp