~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/tools/testing/selftests/rseq/rseq-ppc.h

Version: ~ [ linux-6.2-rc3 ] ~ [ linux-6.1.5 ] ~ [ linux-6.0.19 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.87 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.162 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.228 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.269 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.302 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.337 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
  2 /*
  3  * rseq-ppc.h
  4  *
  5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
  6  * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
  7  */
  8 
  9 #define RSEQ_SIG        0x53053053
 10 
 11 #define rseq_smp_mb()           __asm__ __volatile__ ("sync"    ::: "memory", "cc")
 12 #define rseq_smp_lwsync()       __asm__ __volatile__ ("lwsync"  ::: "memory", "cc")
 13 #define rseq_smp_rmb()          rseq_smp_lwsync()
 14 #define rseq_smp_wmb()          rseq_smp_lwsync()
 15 
 16 #define rseq_smp_load_acquire(p)                                        \
 17 __extension__ ({                                                        \
 18         __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
 19         rseq_smp_lwsync();                                              \
 20         ____p1;                                                         \
 21 })
 22 
 23 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_lwsync()
 24 
 25 #define rseq_smp_store_release(p, v)                                    \
 26 do {                                                                    \
 27         rseq_smp_lwsync();                                              \
 28         RSEQ_WRITE_ONCE(*p, v);                                         \
 29 } while (0)
 30 
 31 #ifdef RSEQ_SKIP_FASTPATH
 32 #include "rseq-skip.h"
 33 #else /* !RSEQ_SKIP_FASTPATH */
 34 
 35 /*
 36  * The __rseq_table section can be used by debuggers to better handle
 37  * single-stepping through the restartable critical sections.
 38  */
 39 
 40 #ifdef __PPC64__
 41 
 42 #define STORE_WORD      "std "
 43 #define LOAD_WORD       "ld "
 44 #define LOADX_WORD      "ldx "
 45 #define CMP_WORD        "cmpd "
 46 
 47 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                          \
 48                         start_ip, post_commit_offset, abort_ip)                 \
 49                 ".pushsection __rseq_table, \"aw\"\n\t"                         \
 50                 ".balign 32\n\t"                                                \
 51                 __rseq_str(label) ":\n\t"                                       \
 52                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t"      \
 53                 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
 54                 ".popsection\n\t"
 55 
 56 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                        \
 57                 RSEQ_INJECT_ASM(1)                                              \
 58                 "lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t"             \
 59                 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t"       \
 60                 "rldicr %%r17, %%r17, 32, 31\n\t"                               \
 61                 "oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t"        \
 62                 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t"            \
 63                 "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t"                     \
 64                 __rseq_str(label) ":\n\t"
 65 
 66 #else /* #ifdef __PPC64__ */
 67 
 68 #define STORE_WORD      "stw "
 69 #define LOAD_WORD       "lwz "
 70 #define LOADX_WORD      "lwzx "
 71 #define CMP_WORD        "cmpw "
 72 
 73 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                          \
 74                         start_ip, post_commit_offset, abort_ip)                 \
 75                 ".pushsection __rseq_table, \"aw\"\n\t"                         \
 76                 ".balign 32\n\t"                                                \
 77                 __rseq_str(label) ":\n\t"                                       \
 78                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t"      \
 79                 /* 32-bit only supported on BE */                               \
 80                 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
 81                 ".popsection\n\t"
 82 
 83 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                        \
 84                 RSEQ_INJECT_ASM(1)                                              \
 85                 "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t"                  \
 86                 "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t"           \
 87                 "stw %%r17, %[" __rseq_str(rseq_cs) "]\n\t"                     \
 88                 __rseq_str(label) ":\n\t"
 89 
 90 #endif /* #ifdef __PPC64__ */
 91 
 92 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip)        \
 93                 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
 94                                         (post_commit_ip - start_ip), abort_ip)
 95 
 96 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)                      \
 97                 RSEQ_INJECT_ASM(2)                                              \
 98                 "lwz %%r17, %[" __rseq_str(current_cpu_id) "]\n\t"              \
 99                 "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t"                \
100                 "bne- cr7, " __rseq_str(label) "\n\t"
101 
102 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label)                               \
103                 ".pushsection __rseq_failure, \"ax\"\n\t"                       \
104                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                            \
105                 __rseq_str(label) ":\n\t"                                       \
106                 "b %l[" __rseq_str(abort_label) "]\n\t"                         \
107                 ".popsection\n\t"
108 
109 /*
110  * RSEQ_ASM_OPs: asm operations for rseq
111  *      RSEQ_ASM_OP_R_*: has hard-code registers in it
112  *      RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
113  */
114 #define RSEQ_ASM_OP_CMPEQ(var, expect, label)                                   \
115                 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"                   \
116                 CMP_WORD "cr7, %%r17, %[" __rseq_str(expect) "]\n\t"            \
117                 "bne- cr7, " __rseq_str(label) "\n\t"
118 
119 #define RSEQ_ASM_OP_CMPNE(var, expectnot, label)                                \
120                 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"                   \
121                 CMP_WORD "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t"         \
122                 "beq- cr7, " __rseq_str(label) "\n\t"
123 
124 #define RSEQ_ASM_OP_STORE(value, var)                                           \
125                 STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
126 
127 /* Load @var to r17 */
128 #define RSEQ_ASM_OP_R_LOAD(var)                                                 \
129                 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
130 
131 /* Store r17 to @var */
132 #define RSEQ_ASM_OP_R_STORE(var)                                                \
133                 STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
134 
135 /* Add @count to r17 */
136 #define RSEQ_ASM_OP_R_ADD(count)                                                \
137                 "add %%r17, %[" __rseq_str(count) "], %%r17\n\t"
138 
139 /* Load (r17 + voffp) to r17 */
140 #define RSEQ_ASM_OP_R_LOADX(voffp)                                              \
141                 LOADX_WORD "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
142 
143 /* TODO: implement a faster memcpy. */
144 #define RSEQ_ASM_OP_R_MEMCPY() \
145                 "cmpdi %%r19, 0\n\t" \
146                 "beq 333f\n\t" \
147                 "addi %%r20, %%r20, -1\n\t" \
148                 "addi %%r21, %%r21, -1\n\t" \
149                 "222:\n\t" \
150                 "lbzu %%r18, 1(%%r20)\n\t" \
151                 "stbu %%r18, 1(%%r21)\n\t" \
152                 "addi %%r19, %%r19, -1\n\t" \
153                 "cmpdi %%r19, 0\n\t" \
154                 "bne 222b\n\t" \
155                 "333:\n\t" \
156 
157 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)                       \
158                 STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"                  \
159                 __rseq_str(post_commit_label) ":\n\t"
160 
161 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)                  \
162                 STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
163                 __rseq_str(post_commit_label) ":\n\t"
164 
165 static inline __attribute__((always_inline))
166 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
167 {
168         RSEQ_INJECT_C(9)
169 
170         __asm__ __volatile__ goto (
171                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
172                 /* Start rseq by storing table entry pointer into rseq_cs. */
173                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
174                 /* cmp cpuid */
175                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
176                 RSEQ_INJECT_ASM(3)
177                 /* cmp @v equal to @expect */
178                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
179                 RSEQ_INJECT_ASM(4)
180 #ifdef RSEQ_COMPARE_TWICE
181                 /* cmp cpuid */
182                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
183                 /* cmp @v equal to @expect */
184                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
185 #endif
186                 /* final store */
187                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
188                 RSEQ_INJECT_ASM(5)
189                 RSEQ_ASM_DEFINE_ABORT(4, abort)
190                 : /* gcc asm goto does not allow outputs */
191                 : [cpu_id]              "r" (cpu),
192                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
193                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
194                   [v]                   "m" (*v),
195                   [expect]              "r" (expect),
196                   [newv]                "r" (newv)
197                   RSEQ_INJECT_INPUT
198                 : "memory", "cc", "r17"
199                   RSEQ_INJECT_CLOBBER
200                 : abort, cmpfail
201 #ifdef RSEQ_COMPARE_TWICE
202                   , error1, error2
203 #endif
204         );
205         return 0;
206 abort:
207         RSEQ_INJECT_FAILED
208         return -1;
209 cmpfail:
210         return 1;
211 #ifdef RSEQ_COMPARE_TWICE
212 error1:
213         rseq_bug("cpu_id comparison failed");
214 error2:
215         rseq_bug("expected value comparison failed");
216 #endif
217 }
218 
219 static inline __attribute__((always_inline))
220 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
221                                off_t voffp, intptr_t *load, int cpu)
222 {
223         RSEQ_INJECT_C(9)
224 
225         __asm__ __volatile__ goto (
226                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
227                 /* Start rseq by storing table entry pointer into rseq_cs. */
228                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
229                 /* cmp cpuid */
230                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
231                 RSEQ_INJECT_ASM(3)
232                 /* cmp @v not equal to @expectnot */
233                 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
234                 RSEQ_INJECT_ASM(4)
235 #ifdef RSEQ_COMPARE_TWICE
236                 /* cmp cpuid */
237                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
238                 /* cmp @v not equal to @expectnot */
239                 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
240 #endif
241                 /* load the value of @v */
242                 RSEQ_ASM_OP_R_LOAD(v)
243                 /* store it in @load */
244                 RSEQ_ASM_OP_R_STORE(load)
245                 /* dereference voffp(v) */
246                 RSEQ_ASM_OP_R_LOADX(voffp)
247                 /* final store the value at voffp(v) */
248                 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
249                 RSEQ_INJECT_ASM(5)
250                 RSEQ_ASM_DEFINE_ABORT(4, abort)
251                 : /* gcc asm goto does not allow outputs */
252                 : [cpu_id]              "r" (cpu),
253                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
254                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
255                   /* final store input */
256                   [v]                   "m" (*v),
257                   [expectnot]           "r" (expectnot),
258                   [voffp]               "b" (voffp),
259                   [load]                "m" (*load)
260                   RSEQ_INJECT_INPUT
261                 : "memory", "cc", "r17"
262                   RSEQ_INJECT_CLOBBER
263                 : abort, cmpfail
264 #ifdef RSEQ_COMPARE_TWICE
265                   , error1, error2
266 #endif
267         );
268         return 0;
269 abort:
270         RSEQ_INJECT_FAILED
271         return -1;
272 cmpfail:
273         return 1;
274 #ifdef RSEQ_COMPARE_TWICE
275 error1:
276         rseq_bug("cpu_id comparison failed");
277 error2:
278         rseq_bug("expected value comparison failed");
279 #endif
280 }
281 
282 static inline __attribute__((always_inline))
283 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
284 {
285         RSEQ_INJECT_C(9)
286 
287         __asm__ __volatile__ goto (
288                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
289                 /* Start rseq by storing table entry pointer into rseq_cs. */
290                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
291                 /* cmp cpuid */
292                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
293                 RSEQ_INJECT_ASM(3)
294 #ifdef RSEQ_COMPARE_TWICE
295                 /* cmp cpuid */
296                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
297 #endif
298                 /* load the value of @v */
299                 RSEQ_ASM_OP_R_LOAD(v)
300                 /* add @count to it */
301                 RSEQ_ASM_OP_R_ADD(count)
302                 /* final store */
303                 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
304                 RSEQ_INJECT_ASM(4)
305                 RSEQ_ASM_DEFINE_ABORT(4, abort)
306                 : /* gcc asm goto does not allow outputs */
307                 : [cpu_id]              "r" (cpu),
308                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
309                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
310                   /* final store input */
311                   [v]                   "m" (*v),
312                   [count]               "r" (count)
313                   RSEQ_INJECT_INPUT
314                 : "memory", "cc", "r17"
315                   RSEQ_INJECT_CLOBBER
316                 : abort
317 #ifdef RSEQ_COMPARE_TWICE
318                   , error1
319 #endif
320         );
321         return 0;
322 abort:
323         RSEQ_INJECT_FAILED
324         return -1;
325 #ifdef RSEQ_COMPARE_TWICE
326 error1:
327         rseq_bug("cpu_id comparison failed");
328 #endif
329 }
330 
331 static inline __attribute__((always_inline))
332 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
333                                  intptr_t *v2, intptr_t newv2,
334                                  intptr_t newv, int cpu)
335 {
336         RSEQ_INJECT_C(9)
337 
338         __asm__ __volatile__ goto (
339                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
340                 /* Start rseq by storing table entry pointer into rseq_cs. */
341                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
342                 /* cmp cpuid */
343                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
344                 RSEQ_INJECT_ASM(3)
345                 /* cmp @v equal to @expect */
346                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
347                 RSEQ_INJECT_ASM(4)
348 #ifdef RSEQ_COMPARE_TWICE
349                 /* cmp cpuid */
350                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
351                 /* cmp @v equal to @expect */
352                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
353 #endif
354                 /* try store */
355                 RSEQ_ASM_OP_STORE(newv2, v2)
356                 RSEQ_INJECT_ASM(5)
357                 /* final store */
358                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
359                 RSEQ_INJECT_ASM(6)
360                 RSEQ_ASM_DEFINE_ABORT(4, abort)
361                 : /* gcc asm goto does not allow outputs */
362                 : [cpu_id]              "r" (cpu),
363                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
364                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
365                   /* try store input */
366                   [v2]                  "m" (*v2),
367                   [newv2]               "r" (newv2),
368                   /* final store input */
369                   [v]                   "m" (*v),
370                   [expect]              "r" (expect),
371                   [newv]                "r" (newv)
372                   RSEQ_INJECT_INPUT
373                 : "memory", "cc", "r17"
374                   RSEQ_INJECT_CLOBBER
375                 : abort, cmpfail
376 #ifdef RSEQ_COMPARE_TWICE
377                   , error1, error2
378 #endif
379         );
380         return 0;
381 abort:
382         RSEQ_INJECT_FAILED
383         return -1;
384 cmpfail:
385         return 1;
386 #ifdef RSEQ_COMPARE_TWICE
387 error1:
388         rseq_bug("cpu_id comparison failed");
389 error2:
390         rseq_bug("expected value comparison failed");
391 #endif
392 }
393 
394 static inline __attribute__((always_inline))
395 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
396                                          intptr_t *v2, intptr_t newv2,
397                                          intptr_t newv, int cpu)
398 {
399         RSEQ_INJECT_C(9)
400 
401         __asm__ __volatile__ goto (
402                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
403                 /* Start rseq by storing table entry pointer into rseq_cs. */
404                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
405                 /* cmp cpuid */
406                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
407                 RSEQ_INJECT_ASM(3)
408                 /* cmp @v equal to @expect */
409                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
410                 RSEQ_INJECT_ASM(4)
411 #ifdef RSEQ_COMPARE_TWICE
412                 /* cmp cpuid */
413                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
414                 /* cmp @v equal to @expect */
415                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
416 #endif
417                 /* try store */
418                 RSEQ_ASM_OP_STORE(newv2, v2)
419                 RSEQ_INJECT_ASM(5)
420                 /* for 'release' */
421                 "lwsync\n\t"
422                 /* final store */
423                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
424                 RSEQ_INJECT_ASM(6)
425                 RSEQ_ASM_DEFINE_ABORT(4, abort)
426                 : /* gcc asm goto does not allow outputs */
427                 : [cpu_id]              "r" (cpu),
428                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
429                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
430                   /* try store input */
431                   [v2]                  "m" (*v2),
432                   [newv2]               "r" (newv2),
433                   /* final store input */
434                   [v]                   "m" (*v),
435                   [expect]              "r" (expect),
436                   [newv]                "r" (newv)
437                   RSEQ_INJECT_INPUT
438                 : "memory", "cc", "r17"
439                   RSEQ_INJECT_CLOBBER
440                 : abort, cmpfail
441 #ifdef RSEQ_COMPARE_TWICE
442                   , error1, error2
443 #endif
444         );
445         return 0;
446 abort:
447         RSEQ_INJECT_FAILED
448         return -1;
449 cmpfail:
450         return 1;
451 #ifdef RSEQ_COMPARE_TWICE
452 error1:
453         rseq_bug("cpu_id comparison failed");
454 error2:
455         rseq_bug("expected value comparison failed");
456 #endif
457 }
458 
459 static inline __attribute__((always_inline))
460 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
461                               intptr_t *v2, intptr_t expect2,
462                               intptr_t newv, int cpu)
463 {
464         RSEQ_INJECT_C(9)
465 
466         __asm__ __volatile__ goto (
467                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
468                 /* Start rseq by storing table entry pointer into rseq_cs. */
469                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
470                 /* cmp cpuid */
471                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
472                 RSEQ_INJECT_ASM(3)
473                 /* cmp @v equal to @expect */
474                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
475                 RSEQ_INJECT_ASM(4)
476                 /* cmp @v2 equal to @expct2 */
477                 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
478                 RSEQ_INJECT_ASM(5)
479 #ifdef RSEQ_COMPARE_TWICE
480                 /* cmp cpuid */
481                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
482                 /* cmp @v equal to @expect */
483                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
484                 /* cmp @v2 equal to @expct2 */
485                 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
486 #endif
487                 /* final store */
488                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
489                 RSEQ_INJECT_ASM(6)
490                 RSEQ_ASM_DEFINE_ABORT(4, abort)
491                 : /* gcc asm goto does not allow outputs */
492                 : [cpu_id]              "r" (cpu),
493                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
494                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
495                   /* cmp2 input */
496                   [v2]                  "m" (*v2),
497                   [expect2]             "r" (expect2),
498                   /* final store input */
499                   [v]                   "m" (*v),
500                   [expect]              "r" (expect),
501                   [newv]                "r" (newv)
502                   RSEQ_INJECT_INPUT
503                 : "memory", "cc", "r17"
504                   RSEQ_INJECT_CLOBBER
505                 : abort, cmpfail
506 #ifdef RSEQ_COMPARE_TWICE
507                   , error1, error2, error3
508 #endif
509         );
510         return 0;
511 abort:
512         RSEQ_INJECT_FAILED
513         return -1;
514 cmpfail:
515         return 1;
516 #ifdef RSEQ_COMPARE_TWICE
517 error1:
518         rseq_bug("cpu_id comparison failed");
519 error2:
520         rseq_bug("1st expected value comparison failed");
521 error3:
522         rseq_bug("2nd expected value comparison failed");
523 #endif
524 }
525 
526 static inline __attribute__((always_inline))
527 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
528                                  void *dst, void *src, size_t len,
529                                  intptr_t newv, int cpu)
530 {
531         RSEQ_INJECT_C(9)
532 
533         __asm__ __volatile__ goto (
534                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
535                 /* setup for mempcy */
536                 "mr %%r19, %[len]\n\t"
537                 "mr %%r20, %[src]\n\t"
538                 "mr %%r21, %[dst]\n\t"
539                 /* Start rseq by storing table entry pointer into rseq_cs. */
540                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
541                 /* cmp cpuid */
542                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
543                 RSEQ_INJECT_ASM(3)
544                 /* cmp @v equal to @expect */
545                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
546                 RSEQ_INJECT_ASM(4)
547 #ifdef RSEQ_COMPARE_TWICE
548                 /* cmp cpuid */
549                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
550                 /* cmp @v equal to @expect */
551                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
552 #endif
553                 /* try memcpy */
554                 RSEQ_ASM_OP_R_MEMCPY()
555                 RSEQ_INJECT_ASM(5)
556                 /* final store */
557                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
558                 RSEQ_INJECT_ASM(6)
559                 /* teardown */
560                 RSEQ_ASM_DEFINE_ABORT(4, abort)
561                 : /* gcc asm goto does not allow outputs */
562                 : [cpu_id]              "r" (cpu),
563                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
564                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
565                   /* final store input */
566                   [v]                   "m" (*v),
567                   [expect]              "r" (expect),
568                   [newv]                "r" (newv),
569                   /* try memcpy input */
570                   [dst]                 "r" (dst),
571                   [src]                 "r" (src),
572                   [len]                 "r" (len)
573                   RSEQ_INJECT_INPUT
574                 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
575                   RSEQ_INJECT_CLOBBER
576                 : abort, cmpfail
577 #ifdef RSEQ_COMPARE_TWICE
578                   , error1, error2
579 #endif
580         );
581         return 0;
582 abort:
583         RSEQ_INJECT_FAILED
584         return -1;
585 cmpfail:
586         return 1;
587 #ifdef RSEQ_COMPARE_TWICE
588 error1:
589         rseq_bug("cpu_id comparison failed");
590 error2:
591         rseq_bug("expected value comparison failed");
592 #endif
593 }
594 
595 static inline __attribute__((always_inline))
596 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
597                                          void *dst, void *src, size_t len,
598                                          intptr_t newv, int cpu)
599 {
600         RSEQ_INJECT_C(9)
601 
602         __asm__ __volatile__ goto (
603                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
604                 /* setup for mempcy */
605                 "mr %%r19, %[len]\n\t"
606                 "mr %%r20, %[src]\n\t"
607                 "mr %%r21, %[dst]\n\t"
608                 /* Start rseq by storing table entry pointer into rseq_cs. */
609                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
610                 /* cmp cpuid */
611                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
612                 RSEQ_INJECT_ASM(3)
613                 /* cmp @v equal to @expect */
614                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
615                 RSEQ_INJECT_ASM(4)
616 #ifdef RSEQ_COMPARE_TWICE
617                 /* cmp cpuid */
618                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
619                 /* cmp @v equal to @expect */
620                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
621 #endif
622                 /* try memcpy */
623                 RSEQ_ASM_OP_R_MEMCPY()
624                 RSEQ_INJECT_ASM(5)
625                 /* for 'release' */
626                 "lwsync\n\t"
627                 /* final store */
628                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
629                 RSEQ_INJECT_ASM(6)
630                 /* teardown */
631                 RSEQ_ASM_DEFINE_ABORT(4, abort)
632                 : /* gcc asm goto does not allow outputs */
633                 : [cpu_id]              "r" (cpu),
634                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
635                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
636                   /* final store input */
637                   [v]                   "m" (*v),
638                   [expect]              "r" (expect),
639                   [newv]                "r" (newv),
640                   /* try memcpy input */
641                   [dst]                 "r" (dst),
642                   [src]                 "r" (src),
643                   [len]                 "r" (len)
644                   RSEQ_INJECT_INPUT
645                 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
646                   RSEQ_INJECT_CLOBBER
647                 : abort, cmpfail
648 #ifdef RSEQ_COMPARE_TWICE
649                   , error1, error2
650 #endif
651         );
652         return 0;
653 abort:
654         RSEQ_INJECT_FAILED
655         return -1;
656 cmpfail:
657         return 1;
658 #ifdef RSEQ_COMPARE_TWICE
659 error1:
660         rseq_bug("cpu_id comparison failed");
661 error2:
662         rseq_bug("expected value comparison failed");
663 #endif
664 }
665 
666 #undef STORE_WORD
667 #undef LOAD_WORD
668 #undef LOADX_WORD
669 #undef CMP_WORD
670 
671 #endif /* !RSEQ_SKIP_FASTPATH */
672 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp