~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/tools/testing/selftests/rseq/rseq-x86.h

Version: ~ [ linux-6.4-rc3 ] ~ [ linux-6.3.4 ] ~ [ linux-6.2.16 ] ~ [ linux-6.1.30 ] ~ [ linux-6.0.19 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.113 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.180 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.243 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.283 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.315 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.337 ] ~ [ linux-4.4.302 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
  2 /*
  3  * rseq-x86.h
  4  *
  5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
  6  */
  7 
  8 #include <stdint.h>
  9 
 10 #define RSEQ_SIG        0x53053053
 11 
 12 #ifdef __x86_64__
 13 
 14 #define rseq_smp_mb()   \
 15         __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
 16 #define rseq_smp_rmb()  rseq_barrier()
 17 #define rseq_smp_wmb()  rseq_barrier()
 18 
 19 #define rseq_smp_load_acquire(p)                                        \
 20 __extension__ ({                                                        \
 21         __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
 22         rseq_barrier();                                                 \
 23         ____p1;                                                         \
 24 })
 25 
 26 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
 27 
 28 #define rseq_smp_store_release(p, v)                                    \
 29 do {                                                                    \
 30         rseq_barrier();                                                 \
 31         RSEQ_WRITE_ONCE(*p, v);                                         \
 32 } while (0)
 33 
 34 #ifdef RSEQ_SKIP_FASTPATH
 35 #include "rseq-skip.h"
 36 #else /* !RSEQ_SKIP_FASTPATH */
 37 
 38 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
 39                                 start_ip, post_commit_offset, abort_ip) \
 40                 ".pushsection __rseq_table, \"aw\"\n\t"                 \
 41                 ".balign 32\n\t"                                        \
 42                 __rseq_str(label) ":\n\t"                               \
 43                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
 44                 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
 45                 ".popsection\n\t"
 46 
 47 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
 48         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
 49                                 (post_commit_ip - start_ip), abort_ip)
 50 
 51 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
 52                 RSEQ_INJECT_ASM(1)                                      \
 53                 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t"       \
 54                 "movq %%rax, %[" __rseq_str(rseq_cs) "]\n\t"            \
 55                 __rseq_str(label) ":\n\t"
 56 
 57 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
 58                 RSEQ_INJECT_ASM(2)                                      \
 59                 "cmpl %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
 60                 "jnz " __rseq_str(label) "\n\t"
 61 
 62 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
 63                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 64                 /* Disassembler-friendly signature: nopl <sig>(%rip). */\
 65                 ".byte 0x0f, 0x1f, 0x05\n\t"                            \
 66                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
 67                 __rseq_str(label) ":\n\t"                               \
 68                 teardown                                                \
 69                 "jmp %l[" __rseq_str(abort_label) "]\n\t"               \
 70                 ".popsection\n\t"
 71 
 72 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
 73                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 74                 __rseq_str(label) ":\n\t"                               \
 75                 teardown                                                \
 76                 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t"             \
 77                 ".popsection\n\t"
 78 
 79 static inline __attribute__((always_inline))
 80 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 81 {
 82         RSEQ_INJECT_C(9)
 83 
 84         __asm__ __volatile__ goto (
 85                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 86                 /* Start rseq by storing table entry pointer into rseq_cs. */
 87                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 88                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 89                 RSEQ_INJECT_ASM(3)
 90                 "cmpq %[v], %[expect]\n\t"
 91                 "jnz %l[cmpfail]\n\t"
 92                 RSEQ_INJECT_ASM(4)
 93 #ifdef RSEQ_COMPARE_TWICE
 94                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 95                 "cmpq %[v], %[expect]\n\t"
 96                 "jnz %l[error2]\n\t"
 97 #endif
 98                 /* final store */
 99                 "movq %[newv], %[v]\n\t"
100                 "2:\n\t"
101                 RSEQ_INJECT_ASM(5)
102                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
103                 : /* gcc asm goto does not allow outputs */
104                 : [cpu_id]              "r" (cpu),
105                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
106                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
107                   [v]                   "m" (*v),
108                   [expect]              "r" (expect),
109                   [newv]                "r" (newv)
110                 : "memory", "cc", "rax"
111                   RSEQ_INJECT_CLOBBER
112                 : abort, cmpfail
113 #ifdef RSEQ_COMPARE_TWICE
114                   , error1, error2
115 #endif
116         );
117         return 0;
118 abort:
119         RSEQ_INJECT_FAILED
120         return -1;
121 cmpfail:
122         return 1;
123 #ifdef RSEQ_COMPARE_TWICE
124 error1:
125         rseq_bug("cpu_id comparison failed");
126 error2:
127         rseq_bug("expected value comparison failed");
128 #endif
129 }
130 
131 /*
132  * Compare @v against @expectnot. When it does _not_ match, load @v
133  * into @load, and store the content of *@v + voffp into @v.
134  */
135 static inline __attribute__((always_inline))
136 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
137                                off_t voffp, intptr_t *load, int cpu)
138 {
139         RSEQ_INJECT_C(9)
140 
141         __asm__ __volatile__ goto (
142                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
143                 /* Start rseq by storing table entry pointer into rseq_cs. */
144                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
145                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
146                 RSEQ_INJECT_ASM(3)
147                 "movq %[v], %%rbx\n\t"
148                 "cmpq %%rbx, %[expectnot]\n\t"
149                 "je %l[cmpfail]\n\t"
150                 RSEQ_INJECT_ASM(4)
151 #ifdef RSEQ_COMPARE_TWICE
152                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
153                 "movq %[v], %%rbx\n\t"
154                 "cmpq %%rbx, %[expectnot]\n\t"
155                 "je %l[error2]\n\t"
156 #endif
157                 "movq %%rbx, %[load]\n\t"
158                 "addq %[voffp], %%rbx\n\t"
159                 "movq (%%rbx), %%rbx\n\t"
160                 /* final store */
161                 "movq %%rbx, %[v]\n\t"
162                 "2:\n\t"
163                 RSEQ_INJECT_ASM(5)
164                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
165                 : /* gcc asm goto does not allow outputs */
166                 : [cpu_id]              "r" (cpu),
167                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
168                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
169                   /* final store input */
170                   [v]                   "m" (*v),
171                   [expectnot]           "r" (expectnot),
172                   [voffp]               "er" (voffp),
173                   [load]                "m" (*load)
174                 : "memory", "cc", "rax", "rbx"
175                   RSEQ_INJECT_CLOBBER
176                 : abort, cmpfail
177 #ifdef RSEQ_COMPARE_TWICE
178                   , error1, error2
179 #endif
180         );
181         return 0;
182 abort:
183         RSEQ_INJECT_FAILED
184         return -1;
185 cmpfail:
186         return 1;
187 #ifdef RSEQ_COMPARE_TWICE
188 error1:
189         rseq_bug("cpu_id comparison failed");
190 error2:
191         rseq_bug("expected value comparison failed");
192 #endif
193 }
194 
195 static inline __attribute__((always_inline))
196 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
197 {
198         RSEQ_INJECT_C(9)
199 
200         __asm__ __volatile__ goto (
201                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
202                 /* Start rseq by storing table entry pointer into rseq_cs. */
203                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
204                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
205                 RSEQ_INJECT_ASM(3)
206 #ifdef RSEQ_COMPARE_TWICE
207                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
208 #endif
209                 /* final store */
210                 "addq %[count], %[v]\n\t"
211                 "2:\n\t"
212                 RSEQ_INJECT_ASM(4)
213                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
214                 : /* gcc asm goto does not allow outputs */
215                 : [cpu_id]              "r" (cpu),
216                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
217                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
218                   /* final store input */
219                   [v]                   "m" (*v),
220                   [count]               "er" (count)
221                 : "memory", "cc", "rax"
222                   RSEQ_INJECT_CLOBBER
223                 : abort
224 #ifdef RSEQ_COMPARE_TWICE
225                   , error1
226 #endif
227         );
228         return 0;
229 abort:
230         RSEQ_INJECT_FAILED
231         return -1;
232 #ifdef RSEQ_COMPARE_TWICE
233 error1:
234         rseq_bug("cpu_id comparison failed");
235 #endif
236 }
237 
238 static inline __attribute__((always_inline))
239 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
240                                  intptr_t *v2, intptr_t newv2,
241                                  intptr_t newv, int cpu)
242 {
243         RSEQ_INJECT_C(9)
244 
245         __asm__ __volatile__ goto (
246                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
247                 /* Start rseq by storing table entry pointer into rseq_cs. */
248                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
249                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
250                 RSEQ_INJECT_ASM(3)
251                 "cmpq %[v], %[expect]\n\t"
252                 "jnz %l[cmpfail]\n\t"
253                 RSEQ_INJECT_ASM(4)
254 #ifdef RSEQ_COMPARE_TWICE
255                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
256                 "cmpq %[v], %[expect]\n\t"
257                 "jnz %l[error2]\n\t"
258 #endif
259                 /* try store */
260                 "movq %[newv2], %[v2]\n\t"
261                 RSEQ_INJECT_ASM(5)
262                 /* final store */
263                 "movq %[newv], %[v]\n\t"
264                 "2:\n\t"
265                 RSEQ_INJECT_ASM(6)
266                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
267                 : /* gcc asm goto does not allow outputs */
268                 : [cpu_id]              "r" (cpu),
269                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
270                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
271                   /* try store input */
272                   [v2]                  "m" (*v2),
273                   [newv2]               "r" (newv2),
274                   /* final store input */
275                   [v]                   "m" (*v),
276                   [expect]              "r" (expect),
277                   [newv]                "r" (newv)
278                 : "memory", "cc", "rax"
279                   RSEQ_INJECT_CLOBBER
280                 : abort, cmpfail
281 #ifdef RSEQ_COMPARE_TWICE
282                   , error1, error2
283 #endif
284         );
285         return 0;
286 abort:
287         RSEQ_INJECT_FAILED
288         return -1;
289 cmpfail:
290         return 1;
291 #ifdef RSEQ_COMPARE_TWICE
292 error1:
293         rseq_bug("cpu_id comparison failed");
294 error2:
295         rseq_bug("expected value comparison failed");
296 #endif
297 }
298 
299 /* x86-64 is TSO. */
300 static inline __attribute__((always_inline))
301 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
302                                          intptr_t *v2, intptr_t newv2,
303                                          intptr_t newv, int cpu)
304 {
305         return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
306 }
307 
308 static inline __attribute__((always_inline))
309 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
310                               intptr_t *v2, intptr_t expect2,
311                               intptr_t newv, int cpu)
312 {
313         RSEQ_INJECT_C(9)
314 
315         __asm__ __volatile__ goto (
316                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
317                 /* Start rseq by storing table entry pointer into rseq_cs. */
318                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
319                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
320                 RSEQ_INJECT_ASM(3)
321                 "cmpq %[v], %[expect]\n\t"
322                 "jnz %l[cmpfail]\n\t"
323                 RSEQ_INJECT_ASM(4)
324                 "cmpq %[v2], %[expect2]\n\t"
325                 "jnz %l[cmpfail]\n\t"
326                 RSEQ_INJECT_ASM(5)
327 #ifdef RSEQ_COMPARE_TWICE
328                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
329                 "cmpq %[v], %[expect]\n\t"
330                 "jnz %l[error2]\n\t"
331                 "cmpq %[v2], %[expect2]\n\t"
332                 "jnz %l[error3]\n\t"
333 #endif
334                 /* final store */
335                 "movq %[newv], %[v]\n\t"
336                 "2:\n\t"
337                 RSEQ_INJECT_ASM(6)
338                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
339                 : /* gcc asm goto does not allow outputs */
340                 : [cpu_id]              "r" (cpu),
341                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
342                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
343                   /* cmp2 input */
344                   [v2]                  "m" (*v2),
345                   [expect2]             "r" (expect2),
346                   /* final store input */
347                   [v]                   "m" (*v),
348                   [expect]              "r" (expect),
349                   [newv]                "r" (newv)
350                 : "memory", "cc", "rax"
351                   RSEQ_INJECT_CLOBBER
352                 : abort, cmpfail
353 #ifdef RSEQ_COMPARE_TWICE
354                   , error1, error2, error3
355 #endif
356         );
357         return 0;
358 abort:
359         RSEQ_INJECT_FAILED
360         return -1;
361 cmpfail:
362         return 1;
363 #ifdef RSEQ_COMPARE_TWICE
364 error1:
365         rseq_bug("cpu_id comparison failed");
366 error2:
367         rseq_bug("1st expected value comparison failed");
368 error3:
369         rseq_bug("2nd expected value comparison failed");
370 #endif
371 }
372 
373 static inline __attribute__((always_inline))
374 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
375                                  void *dst, void *src, size_t len,
376                                  intptr_t newv, int cpu)
377 {
378         uint64_t rseq_scratch[3];
379 
380         RSEQ_INJECT_C(9)
381 
382         __asm__ __volatile__ goto (
383                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
384                 "movq %[src], %[rseq_scratch0]\n\t"
385                 "movq %[dst], %[rseq_scratch1]\n\t"
386                 "movq %[len], %[rseq_scratch2]\n\t"
387                 /* Start rseq by storing table entry pointer into rseq_cs. */
388                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
389                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
390                 RSEQ_INJECT_ASM(3)
391                 "cmpq %[v], %[expect]\n\t"
392                 "jnz 5f\n\t"
393                 RSEQ_INJECT_ASM(4)
394 #ifdef RSEQ_COMPARE_TWICE
395                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
396                 "cmpq %[v], %[expect]\n\t"
397                 "jnz 7f\n\t"
398 #endif
399                 /* try memcpy */
400                 "test %[len], %[len]\n\t" \
401                 "jz 333f\n\t" \
402                 "222:\n\t" \
403                 "movb (%[src]), %%al\n\t" \
404                 "movb %%al, (%[dst])\n\t" \
405                 "inc %[src]\n\t" \
406                 "inc %[dst]\n\t" \
407                 "dec %[len]\n\t" \
408                 "jnz 222b\n\t" \
409                 "333:\n\t" \
410                 RSEQ_INJECT_ASM(5)
411                 /* final store */
412                 "movq %[newv], %[v]\n\t"
413                 "2:\n\t"
414                 RSEQ_INJECT_ASM(6)
415                 /* teardown */
416                 "movq %[rseq_scratch2], %[len]\n\t"
417                 "movq %[rseq_scratch1], %[dst]\n\t"
418                 "movq %[rseq_scratch0], %[src]\n\t"
419                 RSEQ_ASM_DEFINE_ABORT(4,
420                         "movq %[rseq_scratch2], %[len]\n\t"
421                         "movq %[rseq_scratch1], %[dst]\n\t"
422                         "movq %[rseq_scratch0], %[src]\n\t",
423                         abort)
424                 RSEQ_ASM_DEFINE_CMPFAIL(5,
425                         "movq %[rseq_scratch2], %[len]\n\t"
426                         "movq %[rseq_scratch1], %[dst]\n\t"
427                         "movq %[rseq_scratch0], %[src]\n\t",
428                         cmpfail)
429 #ifdef RSEQ_COMPARE_TWICE
430                 RSEQ_ASM_DEFINE_CMPFAIL(6,
431                         "movq %[rseq_scratch2], %[len]\n\t"
432                         "movq %[rseq_scratch1], %[dst]\n\t"
433                         "movq %[rseq_scratch0], %[src]\n\t",
434                         error1)
435                 RSEQ_ASM_DEFINE_CMPFAIL(7,
436                         "movq %[rseq_scratch2], %[len]\n\t"
437                         "movq %[rseq_scratch1], %[dst]\n\t"
438                         "movq %[rseq_scratch0], %[src]\n\t",
439                         error2)
440 #endif
441                 : /* gcc asm goto does not allow outputs */
442                 : [cpu_id]              "r" (cpu),
443                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
444                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
445                   /* final store input */
446                   [v]                   "m" (*v),
447                   [expect]              "r" (expect),
448                   [newv]                "r" (newv),
449                   /* try memcpy input */
450                   [dst]                 "r" (dst),
451                   [src]                 "r" (src),
452                   [len]                 "r" (len),
453                   [rseq_scratch0]       "m" (rseq_scratch[0]),
454                   [rseq_scratch1]       "m" (rseq_scratch[1]),
455                   [rseq_scratch2]       "m" (rseq_scratch[2])
456                 : "memory", "cc", "rax"
457                   RSEQ_INJECT_CLOBBER
458                 : abort, cmpfail
459 #ifdef RSEQ_COMPARE_TWICE
460                   , error1, error2
461 #endif
462         );
463         return 0;
464 abort:
465         RSEQ_INJECT_FAILED
466         return -1;
467 cmpfail:
468         return 1;
469 #ifdef RSEQ_COMPARE_TWICE
470 error1:
471         rseq_bug("cpu_id comparison failed");
472 error2:
473         rseq_bug("expected value comparison failed");
474 #endif
475 }
476 
477 /* x86-64 is TSO. */
478 static inline __attribute__((always_inline))
479 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
480                                          void *dst, void *src, size_t len,
481                                          intptr_t newv, int cpu)
482 {
483         return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
484                                             newv, cpu);
485 }
486 
487 #endif /* !RSEQ_SKIP_FASTPATH */
488 
489 #elif __i386__
490 
491 #define rseq_smp_mb()   \
492         __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
493 #define rseq_smp_rmb()  \
494         __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
495 #define rseq_smp_wmb()  \
496         __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
497 
498 #define rseq_smp_load_acquire(p)                                        \
499 __extension__ ({                                                        \
500         __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
501         rseq_smp_mb();                                                  \
502         ____p1;                                                         \
503 })
504 
505 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
506 
507 #define rseq_smp_store_release(p, v)                                    \
508 do {                                                                    \
509         rseq_smp_mb();                                                  \
510         RSEQ_WRITE_ONCE(*p, v);                                         \
511 } while (0)
512 
513 #ifdef RSEQ_SKIP_FASTPATH
514 #include "rseq-skip.h"
515 #else /* !RSEQ_SKIP_FASTPATH */
516 
517 /*
518  * Use eax as scratch register and take memory operands as input to
519  * lessen register pressure. Especially needed when compiling in O0.
520  */
521 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
522                                 start_ip, post_commit_offset, abort_ip) \
523                 ".pushsection __rseq_table, \"aw\"\n\t"                 \
524                 ".balign 32\n\t"                                        \
525                 __rseq_str(label) ":\n\t"                               \
526                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
527                 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
528                 ".popsection\n\t"
529 
530 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
531         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
532                                 (post_commit_ip - start_ip), abort_ip)
533 
534 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
535                 RSEQ_INJECT_ASM(1)                                      \
536                 "movl $" __rseq_str(cs_label) ", %[rseq_cs]\n\t"        \
537                 __rseq_str(label) ":\n\t"
538 
539 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
540                 RSEQ_INJECT_ASM(2)                                      \
541                 "cmpl %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
542                 "jnz " __rseq_str(label) "\n\t"
543 
544 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
545                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
546                 /* Disassembler-friendly signature: nopl <sig>. */      \
547                 ".byte 0x0f, 0x1f, 0x05\n\t"                            \
548                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
549                 __rseq_str(label) ":\n\t"                               \
550                 teardown                                                \
551                 "jmp %l[" __rseq_str(abort_label) "]\n\t"               \
552                 ".popsection\n\t"
553 
554 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
555                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
556                 __rseq_str(label) ":\n\t"                               \
557                 teardown                                                \
558                 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t"             \
559                 ".popsection\n\t"
560 
561 static inline __attribute__((always_inline))
562 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
563 {
564         RSEQ_INJECT_C(9)
565 
566         __asm__ __volatile__ goto (
567                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
568                 /* Start rseq by storing table entry pointer into rseq_cs. */
569                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
570                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
571                 RSEQ_INJECT_ASM(3)
572                 "cmpl %[v], %[expect]\n\t"
573                 "jnz %l[cmpfail]\n\t"
574                 RSEQ_INJECT_ASM(4)
575 #ifdef RSEQ_COMPARE_TWICE
576                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
577                 "cmpl %[v], %[expect]\n\t"
578                 "jnz %l[error2]\n\t"
579 #endif
580                 /* final store */
581                 "movl %[newv], %[v]\n\t"
582                 "2:\n\t"
583                 RSEQ_INJECT_ASM(5)
584                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
585                 : /* gcc asm goto does not allow outputs */
586                 : [cpu_id]              "r" (cpu),
587                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
588                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
589                   [v]                   "m" (*v),
590                   [expect]              "r" (expect),
591                   [newv]                "r" (newv)
592                 : "memory", "cc", "eax"
593                   RSEQ_INJECT_CLOBBER
594                 : abort, cmpfail
595 #ifdef RSEQ_COMPARE_TWICE
596                   , error1, error2
597 #endif
598         );
599         return 0;
600 abort:
601         RSEQ_INJECT_FAILED
602         return -1;
603 cmpfail:
604         return 1;
605 #ifdef RSEQ_COMPARE_TWICE
606 error1:
607         rseq_bug("cpu_id comparison failed");
608 error2:
609         rseq_bug("expected value comparison failed");
610 #endif
611 }
612 
613 /*
614  * Compare @v against @expectnot. When it does _not_ match, load @v
615  * into @load, and store the content of *@v + voffp into @v.
616  */
617 static inline __attribute__((always_inline))
618 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
619                                off_t voffp, intptr_t *load, int cpu)
620 {
621         RSEQ_INJECT_C(9)
622 
623         __asm__ __volatile__ goto (
624                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
625                 /* Start rseq by storing table entry pointer into rseq_cs. */
626                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
627                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
628                 RSEQ_INJECT_ASM(3)
629                 "movl %[v], %%ebx\n\t"
630                 "cmpl %%ebx, %[expectnot]\n\t"
631                 "je %l[cmpfail]\n\t"
632                 RSEQ_INJECT_ASM(4)
633 #ifdef RSEQ_COMPARE_TWICE
634                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
635                 "movl %[v], %%ebx\n\t"
636                 "cmpl %%ebx, %[expectnot]\n\t"
637                 "je %l[error2]\n\t"
638 #endif
639                 "movl %%ebx, %[load]\n\t"
640                 "addl %[voffp], %%ebx\n\t"
641                 "movl (%%ebx), %%ebx\n\t"
642                 /* final store */
643                 "movl %%ebx, %[v]\n\t"
644                 "2:\n\t"
645                 RSEQ_INJECT_ASM(5)
646                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
647                 : /* gcc asm goto does not allow outputs */
648                 : [cpu_id]              "r" (cpu),
649                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
650                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
651                   /* final store input */
652                   [v]                   "m" (*v),
653                   [expectnot]           "r" (expectnot),
654                   [voffp]               "ir" (voffp),
655                   [load]                "m" (*load)
656                 : "memory", "cc", "eax", "ebx"
657                   RSEQ_INJECT_CLOBBER
658                 : abort, cmpfail
659 #ifdef RSEQ_COMPARE_TWICE
660                   , error1, error2
661 #endif
662         );
663         return 0;
664 abort:
665         RSEQ_INJECT_FAILED
666         return -1;
667 cmpfail:
668         return 1;
669 #ifdef RSEQ_COMPARE_TWICE
670 error1:
671         rseq_bug("cpu_id comparison failed");
672 error2:
673         rseq_bug("expected value comparison failed");
674 #endif
675 }
676 
677 static inline __attribute__((always_inline))
678 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
679 {
680         RSEQ_INJECT_C(9)
681 
682         __asm__ __volatile__ goto (
683                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
684                 /* Start rseq by storing table entry pointer into rseq_cs. */
685                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
686                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
687                 RSEQ_INJECT_ASM(3)
688 #ifdef RSEQ_COMPARE_TWICE
689                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
690 #endif
691                 /* final store */
692                 "addl %[count], %[v]\n\t"
693                 "2:\n\t"
694                 RSEQ_INJECT_ASM(4)
695                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
696                 : /* gcc asm goto does not allow outputs */
697                 : [cpu_id]              "r" (cpu),
698                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
699                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
700                   /* final store input */
701                   [v]                   "m" (*v),
702                   [count]               "ir" (count)
703                 : "memory", "cc", "eax"
704                   RSEQ_INJECT_CLOBBER
705                 : abort
706 #ifdef RSEQ_COMPARE_TWICE
707                   , error1
708 #endif
709         );
710         return 0;
711 abort:
712         RSEQ_INJECT_FAILED
713         return -1;
714 #ifdef RSEQ_COMPARE_TWICE
715 error1:
716         rseq_bug("cpu_id comparison failed");
717 #endif
718 }
719 
720 static inline __attribute__((always_inline))
721 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
722                                  intptr_t *v2, intptr_t newv2,
723                                  intptr_t newv, int cpu)
724 {
725         RSEQ_INJECT_C(9)
726 
727         __asm__ __volatile__ goto (
728                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
729                 /* Start rseq by storing table entry pointer into rseq_cs. */
730                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
731                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
732                 RSEQ_INJECT_ASM(3)
733                 "cmpl %[v], %[expect]\n\t"
734                 "jnz %l[cmpfail]\n\t"
735                 RSEQ_INJECT_ASM(4)
736 #ifdef RSEQ_COMPARE_TWICE
737                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
738                 "cmpl %[v], %[expect]\n\t"
739                 "jnz %l[error2]\n\t"
740 #endif
741                 /* try store */
742                 "movl %[newv2], %%eax\n\t"
743                 "movl %%eax, %[v2]\n\t"
744                 RSEQ_INJECT_ASM(5)
745                 /* final store */
746                 "movl %[newv], %[v]\n\t"
747                 "2:\n\t"
748                 RSEQ_INJECT_ASM(6)
749                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
750                 : /* gcc asm goto does not allow outputs */
751                 : [cpu_id]              "r" (cpu),
752                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
753                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
754                   /* try store input */
755                   [v2]                  "m" (*v2),
756                   [newv2]               "m" (newv2),
757                   /* final store input */
758                   [v]                   "m" (*v),
759                   [expect]              "r" (expect),
760                   [newv]                "r" (newv)
761                 : "memory", "cc", "eax"
762                   RSEQ_INJECT_CLOBBER
763                 : abort, cmpfail
764 #ifdef RSEQ_COMPARE_TWICE
765                   , error1, error2
766 #endif
767         );
768         return 0;
769 abort:
770         RSEQ_INJECT_FAILED
771         return -1;
772 cmpfail:
773         return 1;
774 #ifdef RSEQ_COMPARE_TWICE
775 error1:
776         rseq_bug("cpu_id comparison failed");
777 error2:
778         rseq_bug("expected value comparison failed");
779 #endif
780 }
781 
782 static inline __attribute__((always_inline))
783 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
784                                          intptr_t *v2, intptr_t newv2,
785                                          intptr_t newv, int cpu)
786 {
787         RSEQ_INJECT_C(9)
788 
789         __asm__ __volatile__ goto (
790                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
791                 /* Start rseq by storing table entry pointer into rseq_cs. */
792                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
793                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
794                 RSEQ_INJECT_ASM(3)
795                 "movl %[expect], %%eax\n\t"
796                 "cmpl %[v], %%eax\n\t"
797                 "jnz %l[cmpfail]\n\t"
798                 RSEQ_INJECT_ASM(4)
799 #ifdef RSEQ_COMPARE_TWICE
800                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
801                 "movl %[expect], %%eax\n\t"
802                 "cmpl %[v], %%eax\n\t"
803                 "jnz %l[error2]\n\t"
804 #endif
805                 /* try store */
806                 "movl %[newv2], %[v2]\n\t"
807                 RSEQ_INJECT_ASM(5)
808                 "lock; addl $0,-128(%%esp)\n\t"
809                 /* final store */
810                 "movl %[newv], %[v]\n\t"
811                 "2:\n\t"
812                 RSEQ_INJECT_ASM(6)
813                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
814                 : /* gcc asm goto does not allow outputs */
815                 : [cpu_id]              "r" (cpu),
816                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
817                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
818                   /* try store input */
819                   [v2]                  "m" (*v2),
820                   [newv2]               "r" (newv2),
821                   /* final store input */
822                   [v]                   "m" (*v),
823                   [expect]              "m" (expect),
824                   [newv]                "r" (newv)
825                 : "memory", "cc", "eax"
826                   RSEQ_INJECT_CLOBBER
827                 : abort, cmpfail
828 #ifdef RSEQ_COMPARE_TWICE
829                   , error1, error2
830 #endif
831         );
832         return 0;
833 abort:
834         RSEQ_INJECT_FAILED
835         return -1;
836 cmpfail:
837         return 1;
838 #ifdef RSEQ_COMPARE_TWICE
839 error1:
840         rseq_bug("cpu_id comparison failed");
841 error2:
842         rseq_bug("expected value comparison failed");
843 #endif
844 
845 }
846 
847 static inline __attribute__((always_inline))
848 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
849                               intptr_t *v2, intptr_t expect2,
850                               intptr_t newv, int cpu)
851 {
852         RSEQ_INJECT_C(9)
853 
854         __asm__ __volatile__ goto (
855                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
856                 /* Start rseq by storing table entry pointer into rseq_cs. */
857                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
858                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
859                 RSEQ_INJECT_ASM(3)
860                 "cmpl %[v], %[expect]\n\t"
861                 "jnz %l[cmpfail]\n\t"
862                 RSEQ_INJECT_ASM(4)
863                 "cmpl %[expect2], %[v2]\n\t"
864                 "jnz %l[cmpfail]\n\t"
865                 RSEQ_INJECT_ASM(5)
866 #ifdef RSEQ_COMPARE_TWICE
867                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
868                 "cmpl %[v], %[expect]\n\t"
869                 "jnz %l[error2]\n\t"
870                 "cmpl %[expect2], %[v2]\n\t"
871                 "jnz %l[error3]\n\t"
872 #endif
873                 "movl %[newv], %%eax\n\t"
874                 /* final store */
875                 "movl %%eax, %[v]\n\t"
876                 "2:\n\t"
877                 RSEQ_INJECT_ASM(6)
878                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
879                 : /* gcc asm goto does not allow outputs */
880                 : [cpu_id]              "r" (cpu),
881                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
882                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
883                   /* cmp2 input */
884                   [v2]                  "m" (*v2),
885                   [expect2]             "r" (expect2),
886                   /* final store input */
887                   [v]                   "m" (*v),
888                   [expect]              "r" (expect),
889                   [newv]                "m" (newv)
890                 : "memory", "cc", "eax"
891                   RSEQ_INJECT_CLOBBER
892                 : abort, cmpfail
893 #ifdef RSEQ_COMPARE_TWICE
894                   , error1, error2, error3
895 #endif
896         );
897         return 0;
898 abort:
899         RSEQ_INJECT_FAILED
900         return -1;
901 cmpfail:
902         return 1;
903 #ifdef RSEQ_COMPARE_TWICE
904 error1:
905         rseq_bug("cpu_id comparison failed");
906 error2:
907         rseq_bug("1st expected value comparison failed");
908 error3:
909         rseq_bug("2nd expected value comparison failed");
910 #endif
911 }
912 
913 /* TODO: implement a faster memcpy. */
914 static inline __attribute__((always_inline))
915 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
916                                  void *dst, void *src, size_t len,
917                                  intptr_t newv, int cpu)
918 {
919         uint32_t rseq_scratch[3];
920 
921         RSEQ_INJECT_C(9)
922 
923         __asm__ __volatile__ goto (
924                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
925                 "movl %[src], %[rseq_scratch0]\n\t"
926                 "movl %[dst], %[rseq_scratch1]\n\t"
927                 "movl %[len], %[rseq_scratch2]\n\t"
928                 /* Start rseq by storing table entry pointer into rseq_cs. */
929                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
930                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
931                 RSEQ_INJECT_ASM(3)
932                 "movl %[expect], %%eax\n\t"
933                 "cmpl %%eax, %[v]\n\t"
934                 "jnz 5f\n\t"
935                 RSEQ_INJECT_ASM(4)
936 #ifdef RSEQ_COMPARE_TWICE
937                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
938                 "movl %[expect], %%eax\n\t"
939                 "cmpl %%eax, %[v]\n\t"
940                 "jnz 7f\n\t"
941 #endif
942                 /* try memcpy */
943                 "test %[len], %[len]\n\t" \
944                 "jz 333f\n\t" \
945                 "222:\n\t" \
946                 "movb (%[src]), %%al\n\t" \
947                 "movb %%al, (%[dst])\n\t" \
948                 "inc %[src]\n\t" \
949                 "inc %[dst]\n\t" \
950                 "dec %[len]\n\t" \
951                 "jnz 222b\n\t" \
952                 "333:\n\t" \
953                 RSEQ_INJECT_ASM(5)
954                 "movl %[newv], %%eax\n\t"
955                 /* final store */
956                 "movl %%eax, %[v]\n\t"
957                 "2:\n\t"
958                 RSEQ_INJECT_ASM(6)
959                 /* teardown */
960                 "movl %[rseq_scratch2], %[len]\n\t"
961                 "movl %[rseq_scratch1], %[dst]\n\t"
962                 "movl %[rseq_scratch0], %[src]\n\t"
963                 RSEQ_ASM_DEFINE_ABORT(4,
964                         "movl %[rseq_scratch2], %[len]\n\t"
965                         "movl %[rseq_scratch1], %[dst]\n\t"
966                         "movl %[rseq_scratch0], %[src]\n\t",
967                         abort)
968                 RSEQ_ASM_DEFINE_CMPFAIL(5,
969                         "movl %[rseq_scratch2], %[len]\n\t"
970                         "movl %[rseq_scratch1], %[dst]\n\t"
971                         "movl %[rseq_scratch0], %[src]\n\t",
972                         cmpfail)
973 #ifdef RSEQ_COMPARE_TWICE
974                 RSEQ_ASM_DEFINE_CMPFAIL(6,
975                         "movl %[rseq_scratch2], %[len]\n\t"
976                         "movl %[rseq_scratch1], %[dst]\n\t"
977                         "movl %[rseq_scratch0], %[src]\n\t",
978                         error1)
979                 RSEQ_ASM_DEFINE_CMPFAIL(7,
980                         "movl %[rseq_scratch2], %[len]\n\t"
981                         "movl %[rseq_scratch1], %[dst]\n\t"
982                         "movl %[rseq_scratch0], %[src]\n\t",
983                         error2)
984 #endif
985                 : /* gcc asm goto does not allow outputs */
986                 : [cpu_id]              "r" (cpu),
987                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
988                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
989                   /* final store input */
990                   [v]                   "m" (*v),
991                   [expect]              "m" (expect),
992                   [newv]                "m" (newv),
993                   /* try memcpy input */
994                   [dst]                 "r" (dst),
995                   [src]                 "r" (src),
996                   [len]                 "r" (len),
997                   [rseq_scratch0]       "m" (rseq_scratch[0]),
998                   [rseq_scratch1]       "m" (rseq_scratch[1]),
999                   [rseq_scratch2]       "m" (rseq_scratch[2])
1000                 : "memory", "cc", "eax"
1001                   RSEQ_INJECT_CLOBBER
1002                 : abort, cmpfail
1003 #ifdef RSEQ_COMPARE_TWICE
1004                   , error1, error2
1005 #endif
1006         );
1007         return 0;
1008 abort:
1009         RSEQ_INJECT_FAILED
1010         return -1;
1011 cmpfail:
1012         return 1;
1013 #ifdef RSEQ_COMPARE_TWICE
1014 error1:
1015         rseq_bug("cpu_id comparison failed");
1016 error2:
1017         rseq_bug("expected value comparison failed");
1018 #endif
1019 }
1020 
1021 /* TODO: implement a faster memcpy. */
1022 static inline __attribute__((always_inline))
1023 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
1024                                          void *dst, void *src, size_t len,
1025                                          intptr_t newv, int cpu)
1026 {
1027         uint32_t rseq_scratch[3];
1028 
1029         RSEQ_INJECT_C(9)
1030 
1031         __asm__ __volatile__ goto (
1032                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1033                 "movl %[src], %[rseq_scratch0]\n\t"
1034                 "movl %[dst], %[rseq_scratch1]\n\t"
1035                 "movl %[len], %[rseq_scratch2]\n\t"
1036                 /* Start rseq by storing table entry pointer into rseq_cs. */
1037                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
1038                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
1039                 RSEQ_INJECT_ASM(3)
1040                 "movl %[expect], %%eax\n\t"
1041                 "cmpl %%eax, %[v]\n\t"
1042                 "jnz 5f\n\t"
1043                 RSEQ_INJECT_ASM(4)
1044 #ifdef RSEQ_COMPARE_TWICE
1045                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
1046                 "movl %[expect], %%eax\n\t"
1047                 "cmpl %%eax, %[v]\n\t"
1048                 "jnz 7f\n\t"
1049 #endif
1050                 /* try memcpy */
1051                 "test %[len], %[len]\n\t" \
1052                 "jz 333f\n\t" \
1053                 "222:\n\t" \
1054                 "movb (%[src]), %%al\n\t" \
1055                 "movb %%al, (%[dst])\n\t" \
1056                 "inc %[src]\n\t" \
1057                 "inc %[dst]\n\t" \
1058                 "dec %[len]\n\t" \
1059                 "jnz 222b\n\t" \
1060                 "333:\n\t" \
1061                 RSEQ_INJECT_ASM(5)
1062                 "lock; addl $0,-128(%%esp)\n\t"
1063                 "movl %[newv], %%eax\n\t"
1064                 /* final store */
1065                 "movl %%eax, %[v]\n\t"
1066                 "2:\n\t"
1067                 RSEQ_INJECT_ASM(6)
1068                 /* teardown */
1069                 "movl %[rseq_scratch2], %[len]\n\t"
1070                 "movl %[rseq_scratch1], %[dst]\n\t"
1071                 "movl %[rseq_scratch0], %[src]\n\t"
1072                 RSEQ_ASM_DEFINE_ABORT(4,
1073                         "movl %[rseq_scratch2], %[len]\n\t"
1074                         "movl %[rseq_scratch1], %[dst]\n\t"
1075                         "movl %[rseq_scratch0], %[src]\n\t",
1076                         abort)
1077                 RSEQ_ASM_DEFINE_CMPFAIL(5,
1078                         "movl %[rseq_scratch2], %[len]\n\t"
1079                         "movl %[rseq_scratch1], %[dst]\n\t"
1080                         "movl %[rseq_scratch0], %[src]\n\t",
1081                         cmpfail)
1082 #ifdef RSEQ_COMPARE_TWICE
1083                 RSEQ_ASM_DEFINE_CMPFAIL(6,
1084                         "movl %[rseq_scratch2], %[len]\n\t"
1085                         "movl %[rseq_scratch1], %[dst]\n\t"
1086                         "movl %[rseq_scratch0], %[src]\n\t",
1087                         error1)
1088                 RSEQ_ASM_DEFINE_CMPFAIL(7,
1089                         "movl %[rseq_scratch2], %[len]\n\t"
1090                         "movl %[rseq_scratch1], %[dst]\n\t"
1091                         "movl %[rseq_scratch0], %[src]\n\t",
1092                         error2)
1093 #endif
1094                 : /* gcc asm goto does not allow outputs */
1095                 : [cpu_id]              "r" (cpu),
1096                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
1097                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
1098                   /* final store input */
1099                   [v]                   "m" (*v),
1100                   [expect]              "m" (expect),
1101                   [newv]                "m" (newv),
1102                   /* try memcpy input */
1103                   [dst]                 "r" (dst),
1104                   [src]                 "r" (src),
1105                   [len]                 "r" (len),
1106                   [rseq_scratch0]       "m" (rseq_scratch[0]),
1107                   [rseq_scratch1]       "m" (rseq_scratch[1]),
1108                   [rseq_scratch2]       "m" (rseq_scratch[2])
1109                 : "memory", "cc", "eax"
1110                   RSEQ_INJECT_CLOBBER
1111                 : abort, cmpfail
1112 #ifdef RSEQ_COMPARE_TWICE
1113                   , error1, error2
1114 #endif
1115         );
1116         return 0;
1117 abort:
1118         RSEQ_INJECT_FAILED
1119         return -1;
1120 cmpfail:
1121         return 1;
1122 #ifdef RSEQ_COMPARE_TWICE
1123 error1:
1124         rseq_bug("cpu_id comparison failed");
1125 error2:
1126         rseq_bug("expected value comparison failed");
1127 #endif
1128 }
1129 
1130 #endif /* !RSEQ_SKIP_FASTPATH */
1131 
1132 #endif
1133 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp