~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/tools/testing/selftests/rseq/rseq-arm.h

Version: ~ [ linux-6.2-rc3 ] ~ [ linux-6.1.5 ] ~ [ linux-6.0.19 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.87 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.162 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.228 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.269 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.302 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.337 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
  2 /*
  3  * rseq-arm.h
  4  *
  5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
  6  */
  7 
  8 #define RSEQ_SIG        0x53053053
  9 
 10 #define rseq_smp_mb()   __asm__ __volatile__ ("dmb" ::: "memory", "cc")
 11 #define rseq_smp_rmb()  __asm__ __volatile__ ("dmb" ::: "memory", "cc")
 12 #define rseq_smp_wmb()  __asm__ __volatile__ ("dmb" ::: "memory", "cc")
 13 
 14 #define rseq_smp_load_acquire(p)                                        \
 15 __extension__ ({                                                        \
 16         __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
 17         rseq_smp_mb();                                                  \
 18         ____p1;                                                         \
 19 })
 20 
 21 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
 22 
 23 #define rseq_smp_store_release(p, v)                                    \
 24 do {                                                                    \
 25         rseq_smp_mb();                                                  \
 26         RSEQ_WRITE_ONCE(*p, v);                                         \
 27 } while (0)
 28 
 29 #ifdef RSEQ_SKIP_FASTPATH
 30 #include "rseq-skip.h"
 31 #else /* !RSEQ_SKIP_FASTPATH */
 32 
 33 #define __RSEQ_ASM_DEFINE_TABLE(version, flags, start_ip,               \
 34                                 post_commit_offset, abort_ip)           \
 35                 ".pushsection __rseq_table, \"aw\"\n\t"                 \
 36                 ".balign 32\n\t"                                        \
 37                 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
 38                 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
 39                 ".popsection\n\t"
 40 
 41 #define RSEQ_ASM_DEFINE_TABLE(start_ip, post_commit_ip, abort_ip)       \
 42         __RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip,                     \
 43                                 (post_commit_ip - start_ip), abort_ip)
 44 
 45 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
 46                 RSEQ_INJECT_ASM(1)                                      \
 47                 "adr r0, " __rseq_str(cs_label) "\n\t"                  \
 48                 "str r0, %[" __rseq_str(rseq_cs) "]\n\t"                \
 49                 __rseq_str(label) ":\n\t"
 50 
 51 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
 52                 RSEQ_INJECT_ASM(2)                                      \
 53                 "ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t" \
 54                 "cmp %[" __rseq_str(cpu_id) "], r0\n\t"         \
 55                 "bne " __rseq_str(label) "\n\t"
 56 
 57 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown,           \
 58                                 abort_label, version, flags,            \
 59                                 start_ip, post_commit_offset, abort_ip) \
 60                 ".balign 32\n\t"                                        \
 61                 __rseq_str(table_label) ":\n\t"                         \
 62                 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
 63                 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
 64                 ".word " __rseq_str(RSEQ_SIG) "\n\t"                    \
 65                 __rseq_str(label) ":\n\t"                               \
 66                 teardown                                                \
 67                 "b %l[" __rseq_str(abort_label) "]\n\t"
 68 
 69 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
 70                               start_ip, post_commit_ip, abort_ip)       \
 71         __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown,           \
 72                                 abort_label, 0x0, 0x0, start_ip,        \
 73                                 (post_commit_ip - start_ip), abort_ip)
 74 
 75 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
 76                 __rseq_str(label) ":\n\t"                               \
 77                 teardown                                                \
 78                 "b %l[" __rseq_str(cmpfail_label) "]\n\t"
 79 
 80 #define rseq_workaround_gcc_asm_size_guess()    __asm__ __volatile__("")
 81 
 82 static inline __attribute__((always_inline))
 83 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 84 {
 85         RSEQ_INJECT_C(9)
 86 
 87         rseq_workaround_gcc_asm_size_guess();
 88         __asm__ __volatile__ goto (
 89                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
 90                 /* Start rseq by storing table entry pointer into rseq_cs. */
 91                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 92                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 93                 RSEQ_INJECT_ASM(3)
 94                 "ldr r0, %[v]\n\t"
 95                 "cmp %[expect], r0\n\t"
 96                 "bne %l[cmpfail]\n\t"
 97                 RSEQ_INJECT_ASM(4)
 98 #ifdef RSEQ_COMPARE_TWICE
 99                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
100                 "ldr r0, %[v]\n\t"
101                 "cmp %[expect], r0\n\t"
102                 "bne %l[error2]\n\t"
103 #endif
104                 /* final store */
105                 "str %[newv], %[v]\n\t"
106                 "2:\n\t"
107                 RSEQ_INJECT_ASM(5)
108                 "b 5f\n\t"
109                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
110                 "5:\n\t"
111                 : /* gcc asm goto does not allow outputs */
112                 : [cpu_id]              "r" (cpu),
113                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
114                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
115                   [v]                   "m" (*v),
116                   [expect]              "r" (expect),
117                   [newv]                "r" (newv)
118                   RSEQ_INJECT_INPUT
119                 : "r0", "memory", "cc"
120                   RSEQ_INJECT_CLOBBER
121                 : abort, cmpfail
122 #ifdef RSEQ_COMPARE_TWICE
123                   , error1, error2
124 #endif
125         );
126         rseq_workaround_gcc_asm_size_guess();
127         return 0;
128 abort:
129         rseq_workaround_gcc_asm_size_guess();
130         RSEQ_INJECT_FAILED
131         return -1;
132 cmpfail:
133         rseq_workaround_gcc_asm_size_guess();
134         return 1;
135 #ifdef RSEQ_COMPARE_TWICE
136 error1:
137         rseq_bug("cpu_id comparison failed");
138 error2:
139         rseq_bug("expected value comparison failed");
140 #endif
141 }
142 
143 static inline __attribute__((always_inline))
144 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
145                                off_t voffp, intptr_t *load, int cpu)
146 {
147         RSEQ_INJECT_C(9)
148 
149         rseq_workaround_gcc_asm_size_guess();
150         __asm__ __volatile__ goto (
151                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
152                 /* Start rseq by storing table entry pointer into rseq_cs. */
153                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
154                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
155                 RSEQ_INJECT_ASM(3)
156                 "ldr r0, %[v]\n\t"
157                 "cmp %[expectnot], r0\n\t"
158                 "beq %l[cmpfail]\n\t"
159                 RSEQ_INJECT_ASM(4)
160 #ifdef RSEQ_COMPARE_TWICE
161                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
162                 "ldr r0, %[v]\n\t"
163                 "cmp %[expectnot], r0\n\t"
164                 "beq %l[error2]\n\t"
165 #endif
166                 "str r0, %[load]\n\t"
167                 "add r0, %[voffp]\n\t"
168                 "ldr r0, [r0]\n\t"
169                 /* final store */
170                 "str r0, %[v]\n\t"
171                 "2:\n\t"
172                 RSEQ_INJECT_ASM(5)
173                 "b 5f\n\t"
174                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
175                 "5:\n\t"
176                 : /* gcc asm goto does not allow outputs */
177                 : [cpu_id]              "r" (cpu),
178                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
179                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
180                   /* final store input */
181                   [v]                   "m" (*v),
182                   [expectnot]           "r" (expectnot),
183                   [voffp]               "Ir" (voffp),
184                   [load]                "m" (*load)
185                   RSEQ_INJECT_INPUT
186                 : "r0", "memory", "cc"
187                   RSEQ_INJECT_CLOBBER
188                 : abort, cmpfail
189 #ifdef RSEQ_COMPARE_TWICE
190                   , error1, error2
191 #endif
192         );
193         rseq_workaround_gcc_asm_size_guess();
194         return 0;
195 abort:
196         rseq_workaround_gcc_asm_size_guess();
197         RSEQ_INJECT_FAILED
198         return -1;
199 cmpfail:
200         rseq_workaround_gcc_asm_size_guess();
201         return 1;
202 #ifdef RSEQ_COMPARE_TWICE
203 error1:
204         rseq_bug("cpu_id comparison failed");
205 error2:
206         rseq_bug("expected value comparison failed");
207 #endif
208 }
209 
210 static inline __attribute__((always_inline))
211 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
212 {
213         RSEQ_INJECT_C(9)
214 
215         rseq_workaround_gcc_asm_size_guess();
216         __asm__ __volatile__ goto (
217                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
218                 /* Start rseq by storing table entry pointer into rseq_cs. */
219                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
220                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
221                 RSEQ_INJECT_ASM(3)
222 #ifdef RSEQ_COMPARE_TWICE
223                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
224 #endif
225                 "ldr r0, %[v]\n\t"
226                 "add r0, %[count]\n\t"
227                 /* final store */
228                 "str r0, %[v]\n\t"
229                 "2:\n\t"
230                 RSEQ_INJECT_ASM(4)
231                 "b 5f\n\t"
232                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
233                 "5:\n\t"
234                 : /* gcc asm goto does not allow outputs */
235                 : [cpu_id]              "r" (cpu),
236                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
237                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
238                   [v]                   "m" (*v),
239                   [count]               "Ir" (count)
240                   RSEQ_INJECT_INPUT
241                 : "r0", "memory", "cc"
242                   RSEQ_INJECT_CLOBBER
243                 : abort
244 #ifdef RSEQ_COMPARE_TWICE
245                   , error1
246 #endif
247         );
248         rseq_workaround_gcc_asm_size_guess();
249         return 0;
250 abort:
251         rseq_workaround_gcc_asm_size_guess();
252         RSEQ_INJECT_FAILED
253         return -1;
254 #ifdef RSEQ_COMPARE_TWICE
255 error1:
256         rseq_bug("cpu_id comparison failed");
257 #endif
258 }
259 
260 static inline __attribute__((always_inline))
261 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
262                                  intptr_t *v2, intptr_t newv2,
263                                  intptr_t newv, int cpu)
264 {
265         RSEQ_INJECT_C(9)
266 
267         rseq_workaround_gcc_asm_size_guess();
268         __asm__ __volatile__ goto (
269                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
270                 /* Start rseq by storing table entry pointer into rseq_cs. */
271                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
272                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
273                 RSEQ_INJECT_ASM(3)
274                 "ldr r0, %[v]\n\t"
275                 "cmp %[expect], r0\n\t"
276                 "bne %l[cmpfail]\n\t"
277                 RSEQ_INJECT_ASM(4)
278 #ifdef RSEQ_COMPARE_TWICE
279                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
280                 "ldr r0, %[v]\n\t"
281                 "cmp %[expect], r0\n\t"
282                 "bne %l[error2]\n\t"
283 #endif
284                 /* try store */
285                 "str %[newv2], %[v2]\n\t"
286                 RSEQ_INJECT_ASM(5)
287                 /* final store */
288                 "str %[newv], %[v]\n\t"
289                 "2:\n\t"
290                 RSEQ_INJECT_ASM(6)
291                 "b 5f\n\t"
292                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
293                 "5:\n\t"
294                 : /* gcc asm goto does not allow outputs */
295                 : [cpu_id]              "r" (cpu),
296                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
297                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
298                   /* try store input */
299                   [v2]                  "m" (*v2),
300                   [newv2]               "r" (newv2),
301                   /* final store input */
302                   [v]                   "m" (*v),
303                   [expect]              "r" (expect),
304                   [newv]                "r" (newv)
305                   RSEQ_INJECT_INPUT
306                 : "r0", "memory", "cc"
307                   RSEQ_INJECT_CLOBBER
308                 : abort, cmpfail
309 #ifdef RSEQ_COMPARE_TWICE
310                   , error1, error2
311 #endif
312         );
313         rseq_workaround_gcc_asm_size_guess();
314         return 0;
315 abort:
316         rseq_workaround_gcc_asm_size_guess();
317         RSEQ_INJECT_FAILED
318         return -1;
319 cmpfail:
320         rseq_workaround_gcc_asm_size_guess();
321         return 1;
322 #ifdef RSEQ_COMPARE_TWICE
323 error1:
324         rseq_bug("cpu_id comparison failed");
325 error2:
326         rseq_bug("expected value comparison failed");
327 #endif
328 }
329 
330 static inline __attribute__((always_inline))
331 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
332                                          intptr_t *v2, intptr_t newv2,
333                                          intptr_t newv, int cpu)
334 {
335         RSEQ_INJECT_C(9)
336 
337         rseq_workaround_gcc_asm_size_guess();
338         __asm__ __volatile__ goto (
339                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
340                 /* Start rseq by storing table entry pointer into rseq_cs. */
341                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
342                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
343                 RSEQ_INJECT_ASM(3)
344                 "ldr r0, %[v]\n\t"
345                 "cmp %[expect], r0\n\t"
346                 "bne %l[cmpfail]\n\t"
347                 RSEQ_INJECT_ASM(4)
348 #ifdef RSEQ_COMPARE_TWICE
349                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
350                 "ldr r0, %[v]\n\t"
351                 "cmp %[expect], r0\n\t"
352                 "bne %l[error2]\n\t"
353 #endif
354                 /* try store */
355                 "str %[newv2], %[v2]\n\t"
356                 RSEQ_INJECT_ASM(5)
357                 "dmb\n\t"       /* full mb provides store-release */
358                 /* final store */
359                 "str %[newv], %[v]\n\t"
360                 "2:\n\t"
361                 RSEQ_INJECT_ASM(6)
362                 "b 5f\n\t"
363                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
364                 "5:\n\t"
365                 : /* gcc asm goto does not allow outputs */
366                 : [cpu_id]              "r" (cpu),
367                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
368                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
369                   /* try store input */
370                   [v2]                  "m" (*v2),
371                   [newv2]               "r" (newv2),
372                   /* final store input */
373                   [v]                   "m" (*v),
374                   [expect]              "r" (expect),
375                   [newv]                "r" (newv)
376                   RSEQ_INJECT_INPUT
377                 : "r0", "memory", "cc"
378                   RSEQ_INJECT_CLOBBER
379                 : abort, cmpfail
380 #ifdef RSEQ_COMPARE_TWICE
381                   , error1, error2
382 #endif
383         );
384         rseq_workaround_gcc_asm_size_guess();
385         return 0;
386 abort:
387         rseq_workaround_gcc_asm_size_guess();
388         RSEQ_INJECT_FAILED
389         return -1;
390 cmpfail:
391         rseq_workaround_gcc_asm_size_guess();
392         return 1;
393 #ifdef RSEQ_COMPARE_TWICE
394 error1:
395         rseq_bug("cpu_id comparison failed");
396 error2:
397         rseq_bug("expected value comparison failed");
398 #endif
399 }
400 
401 static inline __attribute__((always_inline))
402 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
403                               intptr_t *v2, intptr_t expect2,
404                               intptr_t newv, int cpu)
405 {
406         RSEQ_INJECT_C(9)
407 
408         rseq_workaround_gcc_asm_size_guess();
409         __asm__ __volatile__ goto (
410                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
411                 /* Start rseq by storing table entry pointer into rseq_cs. */
412                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
413                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
414                 RSEQ_INJECT_ASM(3)
415                 "ldr r0, %[v]\n\t"
416                 "cmp %[expect], r0\n\t"
417                 "bne %l[cmpfail]\n\t"
418                 RSEQ_INJECT_ASM(4)
419                 "ldr r0, %[v2]\n\t"
420                 "cmp %[expect2], r0\n\t"
421                 "bne %l[cmpfail]\n\t"
422                 RSEQ_INJECT_ASM(5)
423 #ifdef RSEQ_COMPARE_TWICE
424                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
425                 "ldr r0, %[v]\n\t"
426                 "cmp %[expect], r0\n\t"
427                 "bne %l[error2]\n\t"
428                 "ldr r0, %[v2]\n\t"
429                 "cmp %[expect2], r0\n\t"
430                 "bne %l[error3]\n\t"
431 #endif
432                 /* final store */
433                 "str %[newv], %[v]\n\t"
434                 "2:\n\t"
435                 RSEQ_INJECT_ASM(6)
436                 "b 5f\n\t"
437                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
438                 "5:\n\t"
439                 : /* gcc asm goto does not allow outputs */
440                 : [cpu_id]              "r" (cpu),
441                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
442                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
443                   /* cmp2 input */
444                   [v2]                  "m" (*v2),
445                   [expect2]             "r" (expect2),
446                   /* final store input */
447                   [v]                   "m" (*v),
448                   [expect]              "r" (expect),
449                   [newv]                "r" (newv)
450                   RSEQ_INJECT_INPUT
451                 : "r0", "memory", "cc"
452                   RSEQ_INJECT_CLOBBER
453                 : abort, cmpfail
454 #ifdef RSEQ_COMPARE_TWICE
455                   , error1, error2, error3
456 #endif
457         );
458         rseq_workaround_gcc_asm_size_guess();
459         return 0;
460 abort:
461         rseq_workaround_gcc_asm_size_guess();
462         RSEQ_INJECT_FAILED
463         return -1;
464 cmpfail:
465         rseq_workaround_gcc_asm_size_guess();
466         return 1;
467 #ifdef RSEQ_COMPARE_TWICE
468 error1:
469         rseq_bug("cpu_id comparison failed");
470 error2:
471         rseq_bug("1st expected value comparison failed");
472 error3:
473         rseq_bug("2nd expected value comparison failed");
474 #endif
475 }
476 
477 static inline __attribute__((always_inline))
478 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
479                                  void *dst, void *src, size_t len,
480                                  intptr_t newv, int cpu)
481 {
482         uint32_t rseq_scratch[3];
483 
484         RSEQ_INJECT_C(9)
485 
486         rseq_workaround_gcc_asm_size_guess();
487         __asm__ __volatile__ goto (
488                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
489                 "str %[src], %[rseq_scratch0]\n\t"
490                 "str %[dst], %[rseq_scratch1]\n\t"
491                 "str %[len], %[rseq_scratch2]\n\t"
492                 /* Start rseq by storing table entry pointer into rseq_cs. */
493                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
494                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
495                 RSEQ_INJECT_ASM(3)
496                 "ldr r0, %[v]\n\t"
497                 "cmp %[expect], r0\n\t"
498                 "bne 5f\n\t"
499                 RSEQ_INJECT_ASM(4)
500 #ifdef RSEQ_COMPARE_TWICE
501                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
502                 "ldr r0, %[v]\n\t"
503                 "cmp %[expect], r0\n\t"
504                 "bne 7f\n\t"
505 #endif
506                 /* try memcpy */
507                 "cmp %[len], #0\n\t" \
508                 "beq 333f\n\t" \
509                 "222:\n\t" \
510                 "ldrb %%r0, [%[src]]\n\t" \
511                 "strb %%r0, [%[dst]]\n\t" \
512                 "adds %[src], #1\n\t" \
513                 "adds %[dst], #1\n\t" \
514                 "subs %[len], #1\n\t" \
515                 "bne 222b\n\t" \
516                 "333:\n\t" \
517                 RSEQ_INJECT_ASM(5)
518                 /* final store */
519                 "str %[newv], %[v]\n\t"
520                 "2:\n\t"
521                 RSEQ_INJECT_ASM(6)
522                 /* teardown */
523                 "ldr %[len], %[rseq_scratch2]\n\t"
524                 "ldr %[dst], %[rseq_scratch1]\n\t"
525                 "ldr %[src], %[rseq_scratch0]\n\t"
526                 "b 8f\n\t"
527                 RSEQ_ASM_DEFINE_ABORT(3, 4,
528                                       /* teardown */
529                                       "ldr %[len], %[rseq_scratch2]\n\t"
530                                       "ldr %[dst], %[rseq_scratch1]\n\t"
531                                       "ldr %[src], %[rseq_scratch0]\n\t",
532                                       abort, 1b, 2b, 4f)
533                 RSEQ_ASM_DEFINE_CMPFAIL(5,
534                                         /* teardown */
535                                         "ldr %[len], %[rseq_scratch2]\n\t"
536                                         "ldr %[dst], %[rseq_scratch1]\n\t"
537                                         "ldr %[src], %[rseq_scratch0]\n\t",
538                                         cmpfail)
539 #ifdef RSEQ_COMPARE_TWICE
540                 RSEQ_ASM_DEFINE_CMPFAIL(6,
541                                         /* teardown */
542                                         "ldr %[len], %[rseq_scratch2]\n\t"
543                                         "ldr %[dst], %[rseq_scratch1]\n\t"
544                                         "ldr %[src], %[rseq_scratch0]\n\t",
545                                         error1)
546                 RSEQ_ASM_DEFINE_CMPFAIL(7,
547                                         /* teardown */
548                                         "ldr %[len], %[rseq_scratch2]\n\t"
549                                         "ldr %[dst], %[rseq_scratch1]\n\t"
550                                         "ldr %[src], %[rseq_scratch0]\n\t",
551                                         error2)
552 #endif
553                 "8:\n\t"
554                 : /* gcc asm goto does not allow outputs */
555                 : [cpu_id]              "r" (cpu),
556                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
557                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
558                   /* final store input */
559                   [v]                   "m" (*v),
560                   [expect]              "r" (expect),
561                   [newv]                "r" (newv),
562                   /* try memcpy input */
563                   [dst]                 "r" (dst),
564                   [src]                 "r" (src),
565                   [len]                 "r" (len),
566                   [rseq_scratch0]       "m" (rseq_scratch[0]),
567                   [rseq_scratch1]       "m" (rseq_scratch[1]),
568                   [rseq_scratch2]       "m" (rseq_scratch[2])
569                   RSEQ_INJECT_INPUT
570                 : "r0", "memory", "cc"
571                   RSEQ_INJECT_CLOBBER
572                 : abort, cmpfail
573 #ifdef RSEQ_COMPARE_TWICE
574                   , error1, error2
575 #endif
576         );
577         rseq_workaround_gcc_asm_size_guess();
578         return 0;
579 abort:
580         rseq_workaround_gcc_asm_size_guess();
581         RSEQ_INJECT_FAILED
582         return -1;
583 cmpfail:
584         rseq_workaround_gcc_asm_size_guess();
585         return 1;
586 #ifdef RSEQ_COMPARE_TWICE
587 error1:
588         rseq_workaround_gcc_asm_size_guess();
589         rseq_bug("cpu_id comparison failed");
590 error2:
591         rseq_workaround_gcc_asm_size_guess();
592         rseq_bug("expected value comparison failed");
593 #endif
594 }
595 
596 static inline __attribute__((always_inline))
597 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
598                                          void *dst, void *src, size_t len,
599                                          intptr_t newv, int cpu)
600 {
601         uint32_t rseq_scratch[3];
602 
603         RSEQ_INJECT_C(9)
604 
605         rseq_workaround_gcc_asm_size_guess();
606         __asm__ __volatile__ goto (
607                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
608                 "str %[src], %[rseq_scratch0]\n\t"
609                 "str %[dst], %[rseq_scratch1]\n\t"
610                 "str %[len], %[rseq_scratch2]\n\t"
611                 /* Start rseq by storing table entry pointer into rseq_cs. */
612                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
613                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
614                 RSEQ_INJECT_ASM(3)
615                 "ldr r0, %[v]\n\t"
616                 "cmp %[expect], r0\n\t"
617                 "bne 5f\n\t"
618                 RSEQ_INJECT_ASM(4)
619 #ifdef RSEQ_COMPARE_TWICE
620                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
621                 "ldr r0, %[v]\n\t"
622                 "cmp %[expect], r0\n\t"
623                 "bne 7f\n\t"
624 #endif
625                 /* try memcpy */
626                 "cmp %[len], #0\n\t" \
627                 "beq 333f\n\t" \
628                 "222:\n\t" \
629                 "ldrb %%r0, [%[src]]\n\t" \
630                 "strb %%r0, [%[dst]]\n\t" \
631                 "adds %[src], #1\n\t" \
632                 "adds %[dst], #1\n\t" \
633                 "subs %[len], #1\n\t" \
634                 "bne 222b\n\t" \
635                 "333:\n\t" \
636                 RSEQ_INJECT_ASM(5)
637                 "dmb\n\t"       /* full mb provides store-release */
638                 /* final store */
639                 "str %[newv], %[v]\n\t"
640                 "2:\n\t"
641                 RSEQ_INJECT_ASM(6)
642                 /* teardown */
643                 "ldr %[len], %[rseq_scratch2]\n\t"
644                 "ldr %[dst], %[rseq_scratch1]\n\t"
645                 "ldr %[src], %[rseq_scratch0]\n\t"
646                 "b 8f\n\t"
647                 RSEQ_ASM_DEFINE_ABORT(3, 4,
648                                       /* teardown */
649                                       "ldr %[len], %[rseq_scratch2]\n\t"
650                                       "ldr %[dst], %[rseq_scratch1]\n\t"
651                                       "ldr %[src], %[rseq_scratch0]\n\t",
652                                       abort, 1b, 2b, 4f)
653                 RSEQ_ASM_DEFINE_CMPFAIL(5,
654                                         /* teardown */
655                                         "ldr %[len], %[rseq_scratch2]\n\t"
656                                         "ldr %[dst], %[rseq_scratch1]\n\t"
657                                         "ldr %[src], %[rseq_scratch0]\n\t",
658                                         cmpfail)
659 #ifdef RSEQ_COMPARE_TWICE
660                 RSEQ_ASM_DEFINE_CMPFAIL(6,
661                                         /* teardown */
662                                         "ldr %[len], %[rseq_scratch2]\n\t"
663                                         "ldr %[dst], %[rseq_scratch1]\n\t"
664                                         "ldr %[src], %[rseq_scratch0]\n\t",
665                                         error1)
666                 RSEQ_ASM_DEFINE_CMPFAIL(7,
667                                         /* teardown */
668                                         "ldr %[len], %[rseq_scratch2]\n\t"
669                                         "ldr %[dst], %[rseq_scratch1]\n\t"
670                                         "ldr %[src], %[rseq_scratch0]\n\t",
671                                         error2)
672 #endif
673                 "8:\n\t"
674                 : /* gcc asm goto does not allow outputs */
675                 : [cpu_id]              "r" (cpu),
676                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
677                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
678                   /* final store input */
679                   [v]                   "m" (*v),
680                   [expect]              "r" (expect),
681                   [newv]                "r" (newv),
682                   /* try memcpy input */
683                   [dst]                 "r" (dst),
684                   [src]                 "r" (src),
685                   [len]                 "r" (len),
686                   [rseq_scratch0]       "m" (rseq_scratch[0]),
687                   [rseq_scratch1]       "m" (rseq_scratch[1]),
688                   [rseq_scratch2]       "m" (rseq_scratch[2])
689                   RSEQ_INJECT_INPUT
690                 : "r0", "memory", "cc"
691                   RSEQ_INJECT_CLOBBER
692                 : abort, cmpfail
693 #ifdef RSEQ_COMPARE_TWICE
694                   , error1, error2
695 #endif
696         );
697         rseq_workaround_gcc_asm_size_guess();
698         return 0;
699 abort:
700         rseq_workaround_gcc_asm_size_guess();
701         RSEQ_INJECT_FAILED
702         return -1;
703 cmpfail:
704         rseq_workaround_gcc_asm_size_guess();
705         return 1;
706 #ifdef RSEQ_COMPARE_TWICE
707 error1:
708         rseq_workaround_gcc_asm_size_guess();
709         rseq_bug("cpu_id comparison failed");
710 error2:
711         rseq_workaround_gcc_asm_size_guess();
712         rseq_bug("expected value comparison failed");
713 #endif
714 }
715 
716 #endif /* !RSEQ_SKIP_FASTPATH */
717 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp