~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/tools/testing/selftests/rseq/rseq-arm.h

Version: ~ [ linux-6.2-rc3 ] ~ [ linux-6.1.5 ] ~ [ linux-6.0.19 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.87 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.162 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.228 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.269 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.302 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.337 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
  2 /*
  3  * rseq-arm.h
  4  *
  5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
  6  */
  7 
  8 /*
  9  * - ARM little endian
 10  *
 11  * RSEQ_SIG uses the udf A32 instruction with an uncommon immediate operand
 12  * value 0x5de3. This traps if user-space reaches this instruction by mistake,
 13  * and the uncommon operand ensures the kernel does not move the instruction
 14  * pointer to attacker-controlled code on rseq abort.
 15  *
 16  * The instruction pattern in the A32 instruction set is:
 17  *
 18  * e7f5def3    udf    #24035    ; 0x5de3
 19  *
 20  * This translates to the following instruction pattern in the T16 instruction
 21  * set:
 22  *
 23  * little endian:
 24  * def3        udf    #243      ; 0xf3
 25  * e7f5        b.n    <7f5>
 26  *
 27  * - ARMv6+ big endian (BE8):
 28  *
 29  * ARMv6+ -mbig-endian generates mixed endianness code vs data: little-endian
 30  * code and big-endian data. The data value of the signature needs to have its
 31  * byte order reversed to generate the trap instruction:
 32  *
 33  * Data: 0xf3def5e7
 34  *
 35  * Translates to this A32 instruction pattern:
 36  *
 37  * e7f5def3    udf    #24035    ; 0x5de3
 38  *
 39  * Translates to this T16 instruction pattern:
 40  *
 41  * def3        udf    #243      ; 0xf3
 42  * e7f5        b.n    <7f5>
 43  *
 44  * - Prior to ARMv6 big endian (BE32):
 45  *
 46  * Prior to ARMv6, -mbig-endian generates big-endian code and data
 47  * (which match), so the endianness of the data representation of the
 48  * signature should not be reversed. However, the choice between BE32
 49  * and BE8 is done by the linker, so we cannot know whether code and
 50  * data endianness will be mixed before the linker is invoked. So rather
 51  * than try to play tricks with the linker, the rseq signature is simply
 52  * data (not a trap instruction) prior to ARMv6 on big endian. This is
 53  * why the signature is expressed as data (.word) rather than as
 54  * instruction (.inst) in assembler.
 55  */
 56 
 57 #ifdef __ARMEB__
 58 #define RSEQ_SIG    0xf3def5e7      /* udf    #24035    ; 0x5de3 (ARMv6+) */
 59 #else
 60 #define RSEQ_SIG    0xe7f5def3      /* udf    #24035    ; 0x5de3 */
 61 #endif
 62 
 63 #define rseq_smp_mb()   __asm__ __volatile__ ("dmb" ::: "memory", "cc")
 64 #define rseq_smp_rmb()  __asm__ __volatile__ ("dmb" ::: "memory", "cc")
 65 #define rseq_smp_wmb()  __asm__ __volatile__ ("dmb" ::: "memory", "cc")
 66 
 67 #define rseq_smp_load_acquire(p)                                        \
 68 __extension__ ({                                                        \
 69         __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
 70         rseq_smp_mb();                                                  \
 71         ____p1;                                                         \
 72 })
 73 
 74 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
 75 
 76 #define rseq_smp_store_release(p, v)                                    \
 77 do {                                                                    \
 78         rseq_smp_mb();                                                  \
 79         RSEQ_WRITE_ONCE(*p, v);                                         \
 80 } while (0)
 81 
 82 #ifdef RSEQ_SKIP_FASTPATH
 83 #include "rseq-skip.h"
 84 #else /* !RSEQ_SKIP_FASTPATH */
 85 
 86 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip,        \
 87                                 post_commit_offset, abort_ip)           \
 88                 ".pushsection __rseq_cs, \"aw\"\n\t"                    \
 89                 ".balign 32\n\t"                                        \
 90                 __rseq_str(label) ":\n\t"                                       \
 91                 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
 92                 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
 93                 ".popsection\n\t"                                       \
 94                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
 95                 ".word " __rseq_str(label) "b, 0x0\n\t"                 \
 96                 ".popsection\n\t"
 97 
 98 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
 99         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
100                                 (post_commit_ip - start_ip), abort_ip)
101 
102 /*
103  * Exit points of a rseq critical section consist of all instructions outside
104  * of the critical section where a critical section can either branch to or
105  * reach through the normal course of its execution. The abort IP and the
106  * post-commit IP are already part of the __rseq_cs section and should not be
107  * explicitly defined as additional exit points. Knowing all exit points is
108  * useful to assist debuggers stepping over the critical section.
109  */
110 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
111                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
112                 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
113                 ".popsection\n\t"
114 
115 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
116                 RSEQ_INJECT_ASM(1)                                      \
117                 "adr r0, " __rseq_str(cs_label) "\n\t"                  \
118                 "str r0, %[" __rseq_str(rseq_cs) "]\n\t"                \
119                 __rseq_str(label) ":\n\t"
120 
121 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
122                 RSEQ_INJECT_ASM(2)                                      \
123                 "ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t" \
124                 "cmp %[" __rseq_str(cpu_id) "], r0\n\t"         \
125                 "bne " __rseq_str(label) "\n\t"
126 
127 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown,           \
128                                 abort_label, version, flags,            \
129                                 start_ip, post_commit_offset, abort_ip) \
130                 ".balign 32\n\t"                                        \
131                 __rseq_str(table_label) ":\n\t"                         \
132                 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
133                 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
134                 ".word " __rseq_str(RSEQ_SIG) "\n\t"                    \
135                 __rseq_str(label) ":\n\t"                               \
136                 teardown                                                \
137                 "b %l[" __rseq_str(abort_label) "]\n\t"
138 
139 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
140                               start_ip, post_commit_ip, abort_ip)       \
141         __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown,           \
142                                 abort_label, 0x0, 0x0, start_ip,        \
143                                 (post_commit_ip - start_ip), abort_ip)
144 
145 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
146                 __rseq_str(label) ":\n\t"                               \
147                 teardown                                                \
148                 "b %l[" __rseq_str(cmpfail_label) "]\n\t"
149 
150 #define rseq_workaround_gcc_asm_size_guess()    __asm__ __volatile__("")
151 
152 static inline __attribute__((always_inline))
153 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
154 {
155         RSEQ_INJECT_C(9)
156 
157         rseq_workaround_gcc_asm_size_guess();
158         __asm__ __volatile__ goto (
159                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
160                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
161 #ifdef RSEQ_COMPARE_TWICE
162                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
163                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
164 #endif
165                 /* Start rseq by storing table entry pointer into rseq_cs. */
166                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
167                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
168                 RSEQ_INJECT_ASM(3)
169                 "ldr r0, %[v]\n\t"
170                 "cmp %[expect], r0\n\t"
171                 "bne %l[cmpfail]\n\t"
172                 RSEQ_INJECT_ASM(4)
173 #ifdef RSEQ_COMPARE_TWICE
174                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
175                 "ldr r0, %[v]\n\t"
176                 "cmp %[expect], r0\n\t"
177                 "bne %l[error2]\n\t"
178 #endif
179                 /* final store */
180                 "str %[newv], %[v]\n\t"
181                 "2:\n\t"
182                 RSEQ_INJECT_ASM(5)
183                 "b 5f\n\t"
184                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
185                 "5:\n\t"
186                 : /* gcc asm goto does not allow outputs */
187                 : [cpu_id]              "r" (cpu),
188                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
189                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
190                   [v]                   "m" (*v),
191                   [expect]              "r" (expect),
192                   [newv]                "r" (newv)
193                   RSEQ_INJECT_INPUT
194                 : "r0", "memory", "cc"
195                   RSEQ_INJECT_CLOBBER
196                 : abort, cmpfail
197 #ifdef RSEQ_COMPARE_TWICE
198                   , error1, error2
199 #endif
200         );
201         rseq_workaround_gcc_asm_size_guess();
202         return 0;
203 abort:
204         rseq_workaround_gcc_asm_size_guess();
205         RSEQ_INJECT_FAILED
206         return -1;
207 cmpfail:
208         rseq_workaround_gcc_asm_size_guess();
209         return 1;
210 #ifdef RSEQ_COMPARE_TWICE
211 error1:
212         rseq_bug("cpu_id comparison failed");
213 error2:
214         rseq_bug("expected value comparison failed");
215 #endif
216 }
217 
218 static inline __attribute__((always_inline))
219 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
220                                off_t voffp, intptr_t *load, int cpu)
221 {
222         RSEQ_INJECT_C(9)
223 
224         rseq_workaround_gcc_asm_size_guess();
225         __asm__ __volatile__ goto (
226                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
227                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
228 #ifdef RSEQ_COMPARE_TWICE
229                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
230                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
231 #endif
232                 /* Start rseq by storing table entry pointer into rseq_cs. */
233                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
234                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
235                 RSEQ_INJECT_ASM(3)
236                 "ldr r0, %[v]\n\t"
237                 "cmp %[expectnot], r0\n\t"
238                 "beq %l[cmpfail]\n\t"
239                 RSEQ_INJECT_ASM(4)
240 #ifdef RSEQ_COMPARE_TWICE
241                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
242                 "ldr r0, %[v]\n\t"
243                 "cmp %[expectnot], r0\n\t"
244                 "beq %l[error2]\n\t"
245 #endif
246                 "str r0, %[load]\n\t"
247                 "add r0, %[voffp]\n\t"
248                 "ldr r0, [r0]\n\t"
249                 /* final store */
250                 "str r0, %[v]\n\t"
251                 "2:\n\t"
252                 RSEQ_INJECT_ASM(5)
253                 "b 5f\n\t"
254                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
255                 "5:\n\t"
256                 : /* gcc asm goto does not allow outputs */
257                 : [cpu_id]              "r" (cpu),
258                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
259                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
260                   /* final store input */
261                   [v]                   "m" (*v),
262                   [expectnot]           "r" (expectnot),
263                   [voffp]               "Ir" (voffp),
264                   [load]                "m" (*load)
265                   RSEQ_INJECT_INPUT
266                 : "r0", "memory", "cc"
267                   RSEQ_INJECT_CLOBBER
268                 : abort, cmpfail
269 #ifdef RSEQ_COMPARE_TWICE
270                   , error1, error2
271 #endif
272         );
273         rseq_workaround_gcc_asm_size_guess();
274         return 0;
275 abort:
276         rseq_workaround_gcc_asm_size_guess();
277         RSEQ_INJECT_FAILED
278         return -1;
279 cmpfail:
280         rseq_workaround_gcc_asm_size_guess();
281         return 1;
282 #ifdef RSEQ_COMPARE_TWICE
283 error1:
284         rseq_bug("cpu_id comparison failed");
285 error2:
286         rseq_bug("expected value comparison failed");
287 #endif
288 }
289 
290 static inline __attribute__((always_inline))
291 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
292 {
293         RSEQ_INJECT_C(9)
294 
295         rseq_workaround_gcc_asm_size_guess();
296         __asm__ __volatile__ goto (
297                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
298 #ifdef RSEQ_COMPARE_TWICE
299                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
300 #endif
301                 /* Start rseq by storing table entry pointer into rseq_cs. */
302                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
303                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
304                 RSEQ_INJECT_ASM(3)
305 #ifdef RSEQ_COMPARE_TWICE
306                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
307 #endif
308                 "ldr r0, %[v]\n\t"
309                 "add r0, %[count]\n\t"
310                 /* final store */
311                 "str r0, %[v]\n\t"
312                 "2:\n\t"
313                 RSEQ_INJECT_ASM(4)
314                 "b 5f\n\t"
315                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
316                 "5:\n\t"
317                 : /* gcc asm goto does not allow outputs */
318                 : [cpu_id]              "r" (cpu),
319                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
320                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
321                   [v]                   "m" (*v),
322                   [count]               "Ir" (count)
323                   RSEQ_INJECT_INPUT
324                 : "r0", "memory", "cc"
325                   RSEQ_INJECT_CLOBBER
326                 : abort
327 #ifdef RSEQ_COMPARE_TWICE
328                   , error1
329 #endif
330         );
331         rseq_workaround_gcc_asm_size_guess();
332         return 0;
333 abort:
334         rseq_workaround_gcc_asm_size_guess();
335         RSEQ_INJECT_FAILED
336         return -1;
337 #ifdef RSEQ_COMPARE_TWICE
338 error1:
339         rseq_bug("cpu_id comparison failed");
340 #endif
341 }
342 
343 static inline __attribute__((always_inline))
344 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
345                                  intptr_t *v2, intptr_t newv2,
346                                  intptr_t newv, int cpu)
347 {
348         RSEQ_INJECT_C(9)
349 
350         rseq_workaround_gcc_asm_size_guess();
351         __asm__ __volatile__ goto (
352                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
353                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
354 #ifdef RSEQ_COMPARE_TWICE
355                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
356                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
357 #endif
358                 /* Start rseq by storing table entry pointer into rseq_cs. */
359                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
360                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
361                 RSEQ_INJECT_ASM(3)
362                 "ldr r0, %[v]\n\t"
363                 "cmp %[expect], r0\n\t"
364                 "bne %l[cmpfail]\n\t"
365                 RSEQ_INJECT_ASM(4)
366 #ifdef RSEQ_COMPARE_TWICE
367                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
368                 "ldr r0, %[v]\n\t"
369                 "cmp %[expect], r0\n\t"
370                 "bne %l[error2]\n\t"
371 #endif
372                 /* try store */
373                 "str %[newv2], %[v2]\n\t"
374                 RSEQ_INJECT_ASM(5)
375                 /* final store */
376                 "str %[newv], %[v]\n\t"
377                 "2:\n\t"
378                 RSEQ_INJECT_ASM(6)
379                 "b 5f\n\t"
380                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
381                 "5:\n\t"
382                 : /* gcc asm goto does not allow outputs */
383                 : [cpu_id]              "r" (cpu),
384                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
385                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
386                   /* try store input */
387                   [v2]                  "m" (*v2),
388                   [newv2]               "r" (newv2),
389                   /* final store input */
390                   [v]                   "m" (*v),
391                   [expect]              "r" (expect),
392                   [newv]                "r" (newv)
393                   RSEQ_INJECT_INPUT
394                 : "r0", "memory", "cc"
395                   RSEQ_INJECT_CLOBBER
396                 : abort, cmpfail
397 #ifdef RSEQ_COMPARE_TWICE
398                   , error1, error2
399 #endif
400         );
401         rseq_workaround_gcc_asm_size_guess();
402         return 0;
403 abort:
404         rseq_workaround_gcc_asm_size_guess();
405         RSEQ_INJECT_FAILED
406         return -1;
407 cmpfail:
408         rseq_workaround_gcc_asm_size_guess();
409         return 1;
410 #ifdef RSEQ_COMPARE_TWICE
411 error1:
412         rseq_bug("cpu_id comparison failed");
413 error2:
414         rseq_bug("expected value comparison failed");
415 #endif
416 }
417 
418 static inline __attribute__((always_inline))
419 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
420                                          intptr_t *v2, intptr_t newv2,
421                                          intptr_t newv, int cpu)
422 {
423         RSEQ_INJECT_C(9)
424 
425         rseq_workaround_gcc_asm_size_guess();
426         __asm__ __volatile__ goto (
427                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
428                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
429 #ifdef RSEQ_COMPARE_TWICE
430                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
431                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
432 #endif
433                 /* Start rseq by storing table entry pointer into rseq_cs. */
434                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
435                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
436                 RSEQ_INJECT_ASM(3)
437                 "ldr r0, %[v]\n\t"
438                 "cmp %[expect], r0\n\t"
439                 "bne %l[cmpfail]\n\t"
440                 RSEQ_INJECT_ASM(4)
441 #ifdef RSEQ_COMPARE_TWICE
442                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
443                 "ldr r0, %[v]\n\t"
444                 "cmp %[expect], r0\n\t"
445                 "bne %l[error2]\n\t"
446 #endif
447                 /* try store */
448                 "str %[newv2], %[v2]\n\t"
449                 RSEQ_INJECT_ASM(5)
450                 "dmb\n\t"       /* full mb provides store-release */
451                 /* final store */
452                 "str %[newv], %[v]\n\t"
453                 "2:\n\t"
454                 RSEQ_INJECT_ASM(6)
455                 "b 5f\n\t"
456                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
457                 "5:\n\t"
458                 : /* gcc asm goto does not allow outputs */
459                 : [cpu_id]              "r" (cpu),
460                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
461                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
462                   /* try store input */
463                   [v2]                  "m" (*v2),
464                   [newv2]               "r" (newv2),
465                   /* final store input */
466                   [v]                   "m" (*v),
467                   [expect]              "r" (expect),
468                   [newv]                "r" (newv)
469                   RSEQ_INJECT_INPUT
470                 : "r0", "memory", "cc"
471                   RSEQ_INJECT_CLOBBER
472                 : abort, cmpfail
473 #ifdef RSEQ_COMPARE_TWICE
474                   , error1, error2
475 #endif
476         );
477         rseq_workaround_gcc_asm_size_guess();
478         return 0;
479 abort:
480         rseq_workaround_gcc_asm_size_guess();
481         RSEQ_INJECT_FAILED
482         return -1;
483 cmpfail:
484         rseq_workaround_gcc_asm_size_guess();
485         return 1;
486 #ifdef RSEQ_COMPARE_TWICE
487 error1:
488         rseq_bug("cpu_id comparison failed");
489 error2:
490         rseq_bug("expected value comparison failed");
491 #endif
492 }
493 
494 static inline __attribute__((always_inline))
495 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
496                               intptr_t *v2, intptr_t expect2,
497                               intptr_t newv, int cpu)
498 {
499         RSEQ_INJECT_C(9)
500 
501         rseq_workaround_gcc_asm_size_guess();
502         __asm__ __volatile__ goto (
503                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
504                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
505 #ifdef RSEQ_COMPARE_TWICE
506                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
507                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
508                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
509 #endif
510                 /* Start rseq by storing table entry pointer into rseq_cs. */
511                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
512                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
513                 RSEQ_INJECT_ASM(3)
514                 "ldr r0, %[v]\n\t"
515                 "cmp %[expect], r0\n\t"
516                 "bne %l[cmpfail]\n\t"
517                 RSEQ_INJECT_ASM(4)
518                 "ldr r0, %[v2]\n\t"
519                 "cmp %[expect2], r0\n\t"
520                 "bne %l[cmpfail]\n\t"
521                 RSEQ_INJECT_ASM(5)
522 #ifdef RSEQ_COMPARE_TWICE
523                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
524                 "ldr r0, %[v]\n\t"
525                 "cmp %[expect], r0\n\t"
526                 "bne %l[error2]\n\t"
527                 "ldr r0, %[v2]\n\t"
528                 "cmp %[expect2], r0\n\t"
529                 "bne %l[error3]\n\t"
530 #endif
531                 /* final store */
532                 "str %[newv], %[v]\n\t"
533                 "2:\n\t"
534                 RSEQ_INJECT_ASM(6)
535                 "b 5f\n\t"
536                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
537                 "5:\n\t"
538                 : /* gcc asm goto does not allow outputs */
539                 : [cpu_id]              "r" (cpu),
540                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
541                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
542                   /* cmp2 input */
543                   [v2]                  "m" (*v2),
544                   [expect2]             "r" (expect2),
545                   /* final store input */
546                   [v]                   "m" (*v),
547                   [expect]              "r" (expect),
548                   [newv]                "r" (newv)
549                   RSEQ_INJECT_INPUT
550                 : "r0", "memory", "cc"
551                   RSEQ_INJECT_CLOBBER
552                 : abort, cmpfail
553 #ifdef RSEQ_COMPARE_TWICE
554                   , error1, error2, error3
555 #endif
556         );
557         rseq_workaround_gcc_asm_size_guess();
558         return 0;
559 abort:
560         rseq_workaround_gcc_asm_size_guess();
561         RSEQ_INJECT_FAILED
562         return -1;
563 cmpfail:
564         rseq_workaround_gcc_asm_size_guess();
565         return 1;
566 #ifdef RSEQ_COMPARE_TWICE
567 error1:
568         rseq_bug("cpu_id comparison failed");
569 error2:
570         rseq_bug("1st expected value comparison failed");
571 error3:
572         rseq_bug("2nd expected value comparison failed");
573 #endif
574 }
575 
576 static inline __attribute__((always_inline))
577 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
578                                  void *dst, void *src, size_t len,
579                                  intptr_t newv, int cpu)
580 {
581         uint32_t rseq_scratch[3];
582 
583         RSEQ_INJECT_C(9)
584 
585         rseq_workaround_gcc_asm_size_guess();
586         __asm__ __volatile__ goto (
587                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
588                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
589 #ifdef RSEQ_COMPARE_TWICE
590                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
591                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
592 #endif
593                 "str %[src], %[rseq_scratch0]\n\t"
594                 "str %[dst], %[rseq_scratch1]\n\t"
595                 "str %[len], %[rseq_scratch2]\n\t"
596                 /* Start rseq by storing table entry pointer into rseq_cs. */
597                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
598                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
599                 RSEQ_INJECT_ASM(3)
600                 "ldr r0, %[v]\n\t"
601                 "cmp %[expect], r0\n\t"
602                 "bne 5f\n\t"
603                 RSEQ_INJECT_ASM(4)
604 #ifdef RSEQ_COMPARE_TWICE
605                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
606                 "ldr r0, %[v]\n\t"
607                 "cmp %[expect], r0\n\t"
608                 "bne 7f\n\t"
609 #endif
610                 /* try memcpy */
611                 "cmp %[len], #0\n\t" \
612                 "beq 333f\n\t" \
613                 "222:\n\t" \
614                 "ldrb %%r0, [%[src]]\n\t" \
615                 "strb %%r0, [%[dst]]\n\t" \
616                 "adds %[src], #1\n\t" \
617                 "adds %[dst], #1\n\t" \
618                 "subs %[len], #1\n\t" \
619                 "bne 222b\n\t" \
620                 "333:\n\t" \
621                 RSEQ_INJECT_ASM(5)
622                 /* final store */
623                 "str %[newv], %[v]\n\t"
624                 "2:\n\t"
625                 RSEQ_INJECT_ASM(6)
626                 /* teardown */
627                 "ldr %[len], %[rseq_scratch2]\n\t"
628                 "ldr %[dst], %[rseq_scratch1]\n\t"
629                 "ldr %[src], %[rseq_scratch0]\n\t"
630                 "b 8f\n\t"
631                 RSEQ_ASM_DEFINE_ABORT(3, 4,
632                                       /* teardown */
633                                       "ldr %[len], %[rseq_scratch2]\n\t"
634                                       "ldr %[dst], %[rseq_scratch1]\n\t"
635                                       "ldr %[src], %[rseq_scratch0]\n\t",
636                                       abort, 1b, 2b, 4f)
637                 RSEQ_ASM_DEFINE_CMPFAIL(5,
638                                         /* teardown */
639                                         "ldr %[len], %[rseq_scratch2]\n\t"
640                                         "ldr %[dst], %[rseq_scratch1]\n\t"
641                                         "ldr %[src], %[rseq_scratch0]\n\t",
642                                         cmpfail)
643 #ifdef RSEQ_COMPARE_TWICE
644                 RSEQ_ASM_DEFINE_CMPFAIL(6,
645                                         /* teardown */
646                                         "ldr %[len], %[rseq_scratch2]\n\t"
647                                         "ldr %[dst], %[rseq_scratch1]\n\t"
648                                         "ldr %[src], %[rseq_scratch0]\n\t",
649                                         error1)
650                 RSEQ_ASM_DEFINE_CMPFAIL(7,
651                                         /* teardown */
652                                         "ldr %[len], %[rseq_scratch2]\n\t"
653                                         "ldr %[dst], %[rseq_scratch1]\n\t"
654                                         "ldr %[src], %[rseq_scratch0]\n\t",
655                                         error2)
656 #endif
657                 "8:\n\t"
658                 : /* gcc asm goto does not allow outputs */
659                 : [cpu_id]              "r" (cpu),
660                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
661                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
662                   /* final store input */
663                   [v]                   "m" (*v),
664                   [expect]              "r" (expect),
665                   [newv]                "r" (newv),
666                   /* try memcpy input */
667                   [dst]                 "r" (dst),
668                   [src]                 "r" (src),
669                   [len]                 "r" (len),
670                   [rseq_scratch0]       "m" (rseq_scratch[0]),
671                   [rseq_scratch1]       "m" (rseq_scratch[1]),
672                   [rseq_scratch2]       "m" (rseq_scratch[2])
673                   RSEQ_INJECT_INPUT
674                 : "r0", "memory", "cc"
675                   RSEQ_INJECT_CLOBBER
676                 : abort, cmpfail
677 #ifdef RSEQ_COMPARE_TWICE
678                   , error1, error2
679 #endif
680         );
681         rseq_workaround_gcc_asm_size_guess();
682         return 0;
683 abort:
684         rseq_workaround_gcc_asm_size_guess();
685         RSEQ_INJECT_FAILED
686         return -1;
687 cmpfail:
688         rseq_workaround_gcc_asm_size_guess();
689         return 1;
690 #ifdef RSEQ_COMPARE_TWICE
691 error1:
692         rseq_workaround_gcc_asm_size_guess();
693         rseq_bug("cpu_id comparison failed");
694 error2:
695         rseq_workaround_gcc_asm_size_guess();
696         rseq_bug("expected value comparison failed");
697 #endif
698 }
699 
700 static inline __attribute__((always_inline))
701 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
702                                          void *dst, void *src, size_t len,
703                                          intptr_t newv, int cpu)
704 {
705         uint32_t rseq_scratch[3];
706 
707         RSEQ_INJECT_C(9)
708 
709         rseq_workaround_gcc_asm_size_guess();
710         __asm__ __volatile__ goto (
711                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
712                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
713 #ifdef RSEQ_COMPARE_TWICE
714                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
715                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
716 #endif
717                 "str %[src], %[rseq_scratch0]\n\t"
718                 "str %[dst], %[rseq_scratch1]\n\t"
719                 "str %[len], %[rseq_scratch2]\n\t"
720                 /* Start rseq by storing table entry pointer into rseq_cs. */
721                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
722                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
723                 RSEQ_INJECT_ASM(3)
724                 "ldr r0, %[v]\n\t"
725                 "cmp %[expect], r0\n\t"
726                 "bne 5f\n\t"
727                 RSEQ_INJECT_ASM(4)
728 #ifdef RSEQ_COMPARE_TWICE
729                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
730                 "ldr r0, %[v]\n\t"
731                 "cmp %[expect], r0\n\t"
732                 "bne 7f\n\t"
733 #endif
734                 /* try memcpy */
735                 "cmp %[len], #0\n\t" \
736                 "beq 333f\n\t" \
737                 "222:\n\t" \
738                 "ldrb %%r0, [%[src]]\n\t" \
739                 "strb %%r0, [%[dst]]\n\t" \
740                 "adds %[src], #1\n\t" \
741                 "adds %[dst], #1\n\t" \
742                 "subs %[len], #1\n\t" \
743                 "bne 222b\n\t" \
744                 "333:\n\t" \
745                 RSEQ_INJECT_ASM(5)
746                 "dmb\n\t"       /* full mb provides store-release */
747                 /* final store */
748                 "str %[newv], %[v]\n\t"
749                 "2:\n\t"
750                 RSEQ_INJECT_ASM(6)
751                 /* teardown */
752                 "ldr %[len], %[rseq_scratch2]\n\t"
753                 "ldr %[dst], %[rseq_scratch1]\n\t"
754                 "ldr %[src], %[rseq_scratch0]\n\t"
755                 "b 8f\n\t"
756                 RSEQ_ASM_DEFINE_ABORT(3, 4,
757                                       /* teardown */
758                                       "ldr %[len], %[rseq_scratch2]\n\t"
759                                       "ldr %[dst], %[rseq_scratch1]\n\t"
760                                       "ldr %[src], %[rseq_scratch0]\n\t",
761                                       abort, 1b, 2b, 4f)
762                 RSEQ_ASM_DEFINE_CMPFAIL(5,
763                                         /* teardown */
764                                         "ldr %[len], %[rseq_scratch2]\n\t"
765                                         "ldr %[dst], %[rseq_scratch1]\n\t"
766                                         "ldr %[src], %[rseq_scratch0]\n\t",
767                                         cmpfail)
768 #ifdef RSEQ_COMPARE_TWICE
769                 RSEQ_ASM_DEFINE_CMPFAIL(6,
770                                         /* teardown */
771                                         "ldr %[len], %[rseq_scratch2]\n\t"
772                                         "ldr %[dst], %[rseq_scratch1]\n\t"
773                                         "ldr %[src], %[rseq_scratch0]\n\t",
774                                         error1)
775                 RSEQ_ASM_DEFINE_CMPFAIL(7,
776                                         /* teardown */
777                                         "ldr %[len], %[rseq_scratch2]\n\t"
778                                         "ldr %[dst], %[rseq_scratch1]\n\t"
779                                         "ldr %[src], %[rseq_scratch0]\n\t",
780                                         error2)
781 #endif
782                 "8:\n\t"
783                 : /* gcc asm goto does not allow outputs */
784                 : [cpu_id]              "r" (cpu),
785                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
786                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
787                   /* final store input */
788                   [v]                   "m" (*v),
789                   [expect]              "r" (expect),
790                   [newv]                "r" (newv),
791                   /* try memcpy input */
792                   [dst]                 "r" (dst),
793                   [src]                 "r" (src),
794                   [len]                 "r" (len),
795                   [rseq_scratch0]       "m" (rseq_scratch[0]),
796                   [rseq_scratch1]       "m" (rseq_scratch[1]),
797                   [rseq_scratch2]       "m" (rseq_scratch[2])
798                   RSEQ_INJECT_INPUT
799                 : "r0", "memory", "cc"
800                   RSEQ_INJECT_CLOBBER
801                 : abort, cmpfail
802 #ifdef RSEQ_COMPARE_TWICE
803                   , error1, error2
804 #endif
805         );
806         rseq_workaround_gcc_asm_size_guess();
807         return 0;
808 abort:
809         rseq_workaround_gcc_asm_size_guess();
810         RSEQ_INJECT_FAILED
811         return -1;
812 cmpfail:
813         rseq_workaround_gcc_asm_size_guess();
814         return 1;
815 #ifdef RSEQ_COMPARE_TWICE
816 error1:
817         rseq_workaround_gcc_asm_size_guess();
818         rseq_bug("cpu_id comparison failed");
819 error2:
820         rseq_workaround_gcc_asm_size_guess();
821         rseq_bug("expected value comparison failed");
822 #endif
823 }
824 
825 #endif /* !RSEQ_SKIP_FASTPATH */
826 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp