~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/include/linux/atomic-fallback.h

Version: ~ [ linux-5.18 ] ~ [ linux-5.17.9 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.41 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.117 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.195 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.244 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.280 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.315 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 
  3 // Generated by scripts/atomic/gen-atomic-fallback.sh
  4 // DO NOT MODIFY THIS FILE DIRECTLY
  5 
  6 #ifndef _LINUX_ATOMIC_FALLBACK_H
  7 #define _LINUX_ATOMIC_FALLBACK_H
  8 
  9 #ifndef xchg_relaxed
 10 #define xchg_relaxed            xchg
 11 #define xchg_acquire            xchg
 12 #define xchg_release            xchg
 13 #else /* xchg_relaxed */
 14 
 15 #ifndef xchg_acquire
 16 #define xchg_acquire(...) \
 17         __atomic_op_acquire(xchg, __VA_ARGS__)
 18 #endif
 19 
 20 #ifndef xchg_release
 21 #define xchg_release(...) \
 22         __atomic_op_release(xchg, __VA_ARGS__)
 23 #endif
 24 
 25 #ifndef xchg
 26 #define xchg(...) \
 27         __atomic_op_fence(xchg, __VA_ARGS__)
 28 #endif
 29 
 30 #endif /* xchg_relaxed */
 31 
 32 #ifndef cmpxchg_relaxed
 33 #define cmpxchg_relaxed         cmpxchg
 34 #define cmpxchg_acquire         cmpxchg
 35 #define cmpxchg_release         cmpxchg
 36 #else /* cmpxchg_relaxed */
 37 
 38 #ifndef cmpxchg_acquire
 39 #define cmpxchg_acquire(...) \
 40         __atomic_op_acquire(cmpxchg, __VA_ARGS__)
 41 #endif
 42 
 43 #ifndef cmpxchg_release
 44 #define cmpxchg_release(...) \
 45         __atomic_op_release(cmpxchg, __VA_ARGS__)
 46 #endif
 47 
 48 #ifndef cmpxchg
 49 #define cmpxchg(...) \
 50         __atomic_op_fence(cmpxchg, __VA_ARGS__)
 51 #endif
 52 
 53 #endif /* cmpxchg_relaxed */
 54 
 55 #ifndef cmpxchg64_relaxed
 56 #define cmpxchg64_relaxed               cmpxchg64
 57 #define cmpxchg64_acquire               cmpxchg64
 58 #define cmpxchg64_release               cmpxchg64
 59 #else /* cmpxchg64_relaxed */
 60 
 61 #ifndef cmpxchg64_acquire
 62 #define cmpxchg64_acquire(...) \
 63         __atomic_op_acquire(cmpxchg64, __VA_ARGS__)
 64 #endif
 65 
 66 #ifndef cmpxchg64_release
 67 #define cmpxchg64_release(...) \
 68         __atomic_op_release(cmpxchg64, __VA_ARGS__)
 69 #endif
 70 
 71 #ifndef cmpxchg64
 72 #define cmpxchg64(...) \
 73         __atomic_op_fence(cmpxchg64, __VA_ARGS__)
 74 #endif
 75 
 76 #endif /* cmpxchg64_relaxed */
 77 
 78 #ifndef atomic_read_acquire
 79 static inline int
 80 atomic_read_acquire(const atomic_t *v)
 81 {
 82         return smp_load_acquire(&(v)->counter);
 83 }
 84 #define atomic_read_acquire atomic_read_acquire
 85 #endif
 86 
 87 #ifndef atomic_set_release
 88 static inline void
 89 atomic_set_release(atomic_t *v, int i)
 90 {
 91         smp_store_release(&(v)->counter, i);
 92 }
 93 #define atomic_set_release atomic_set_release
 94 #endif
 95 
 96 #ifndef atomic_add_return_relaxed
 97 #define atomic_add_return_acquire atomic_add_return
 98 #define atomic_add_return_release atomic_add_return
 99 #define atomic_add_return_relaxed atomic_add_return
100 #else /* atomic_add_return_relaxed */
101 
102 #ifndef atomic_add_return_acquire
103 static inline int
104 atomic_add_return_acquire(int i, atomic_t *v)
105 {
106         int ret = atomic_add_return_relaxed(i, v);
107         __atomic_acquire_fence();
108         return ret;
109 }
110 #define atomic_add_return_acquire atomic_add_return_acquire
111 #endif
112 
113 #ifndef atomic_add_return_release
114 static inline int
115 atomic_add_return_release(int i, atomic_t *v)
116 {
117         __atomic_release_fence();
118         return atomic_add_return_relaxed(i, v);
119 }
120 #define atomic_add_return_release atomic_add_return_release
121 #endif
122 
123 #ifndef atomic_add_return
124 static inline int
125 atomic_add_return(int i, atomic_t *v)
126 {
127         int ret;
128         __atomic_pre_full_fence();
129         ret = atomic_add_return_relaxed(i, v);
130         __atomic_post_full_fence();
131         return ret;
132 }
133 #define atomic_add_return atomic_add_return
134 #endif
135 
136 #endif /* atomic_add_return_relaxed */
137 
138 #ifndef atomic_fetch_add_relaxed
139 #define atomic_fetch_add_acquire atomic_fetch_add
140 #define atomic_fetch_add_release atomic_fetch_add
141 #define atomic_fetch_add_relaxed atomic_fetch_add
142 #else /* atomic_fetch_add_relaxed */
143 
144 #ifndef atomic_fetch_add_acquire
145 static inline int
146 atomic_fetch_add_acquire(int i, atomic_t *v)
147 {
148         int ret = atomic_fetch_add_relaxed(i, v);
149         __atomic_acquire_fence();
150         return ret;
151 }
152 #define atomic_fetch_add_acquire atomic_fetch_add_acquire
153 #endif
154 
155 #ifndef atomic_fetch_add_release
156 static inline int
157 atomic_fetch_add_release(int i, atomic_t *v)
158 {
159         __atomic_release_fence();
160         return atomic_fetch_add_relaxed(i, v);
161 }
162 #define atomic_fetch_add_release atomic_fetch_add_release
163 #endif
164 
165 #ifndef atomic_fetch_add
166 static inline int
167 atomic_fetch_add(int i, atomic_t *v)
168 {
169         int ret;
170         __atomic_pre_full_fence();
171         ret = atomic_fetch_add_relaxed(i, v);
172         __atomic_post_full_fence();
173         return ret;
174 }
175 #define atomic_fetch_add atomic_fetch_add
176 #endif
177 
178 #endif /* atomic_fetch_add_relaxed */
179 
180 #ifndef atomic_sub_return_relaxed
181 #define atomic_sub_return_acquire atomic_sub_return
182 #define atomic_sub_return_release atomic_sub_return
183 #define atomic_sub_return_relaxed atomic_sub_return
184 #else /* atomic_sub_return_relaxed */
185 
186 #ifndef atomic_sub_return_acquire
187 static inline int
188 atomic_sub_return_acquire(int i, atomic_t *v)
189 {
190         int ret = atomic_sub_return_relaxed(i, v);
191         __atomic_acquire_fence();
192         return ret;
193 }
194 #define atomic_sub_return_acquire atomic_sub_return_acquire
195 #endif
196 
197 #ifndef atomic_sub_return_release
198 static inline int
199 atomic_sub_return_release(int i, atomic_t *v)
200 {
201         __atomic_release_fence();
202         return atomic_sub_return_relaxed(i, v);
203 }
204 #define atomic_sub_return_release atomic_sub_return_release
205 #endif
206 
207 #ifndef atomic_sub_return
208 static inline int
209 atomic_sub_return(int i, atomic_t *v)
210 {
211         int ret;
212         __atomic_pre_full_fence();
213         ret = atomic_sub_return_relaxed(i, v);
214         __atomic_post_full_fence();
215         return ret;
216 }
217 #define atomic_sub_return atomic_sub_return
218 #endif
219 
220 #endif /* atomic_sub_return_relaxed */
221 
222 #ifndef atomic_fetch_sub_relaxed
223 #define atomic_fetch_sub_acquire atomic_fetch_sub
224 #define atomic_fetch_sub_release atomic_fetch_sub
225 #define atomic_fetch_sub_relaxed atomic_fetch_sub
226 #else /* atomic_fetch_sub_relaxed */
227 
228 #ifndef atomic_fetch_sub_acquire
229 static inline int
230 atomic_fetch_sub_acquire(int i, atomic_t *v)
231 {
232         int ret = atomic_fetch_sub_relaxed(i, v);
233         __atomic_acquire_fence();
234         return ret;
235 }
236 #define atomic_fetch_sub_acquire atomic_fetch_sub_acquire
237 #endif
238 
239 #ifndef atomic_fetch_sub_release
240 static inline int
241 atomic_fetch_sub_release(int i, atomic_t *v)
242 {
243         __atomic_release_fence();
244         return atomic_fetch_sub_relaxed(i, v);
245 }
246 #define atomic_fetch_sub_release atomic_fetch_sub_release
247 #endif
248 
249 #ifndef atomic_fetch_sub
250 static inline int
251 atomic_fetch_sub(int i, atomic_t *v)
252 {
253         int ret;
254         __atomic_pre_full_fence();
255         ret = atomic_fetch_sub_relaxed(i, v);
256         __atomic_post_full_fence();
257         return ret;
258 }
259 #define atomic_fetch_sub atomic_fetch_sub
260 #endif
261 
262 #endif /* atomic_fetch_sub_relaxed */
263 
264 #ifndef atomic_inc
265 static inline void
266 atomic_inc(atomic_t *v)
267 {
268         atomic_add(1, v);
269 }
270 #define atomic_inc atomic_inc
271 #endif
272 
273 #ifndef atomic_inc_return_relaxed
274 #ifdef atomic_inc_return
275 #define atomic_inc_return_acquire atomic_inc_return
276 #define atomic_inc_return_release atomic_inc_return
277 #define atomic_inc_return_relaxed atomic_inc_return
278 #endif /* atomic_inc_return */
279 
280 #ifndef atomic_inc_return
281 static inline int
282 atomic_inc_return(atomic_t *v)
283 {
284         return atomic_add_return(1, v);
285 }
286 #define atomic_inc_return atomic_inc_return
287 #endif
288 
289 #ifndef atomic_inc_return_acquire
290 static inline int
291 atomic_inc_return_acquire(atomic_t *v)
292 {
293         return atomic_add_return_acquire(1, v);
294 }
295 #define atomic_inc_return_acquire atomic_inc_return_acquire
296 #endif
297 
298 #ifndef atomic_inc_return_release
299 static inline int
300 atomic_inc_return_release(atomic_t *v)
301 {
302         return atomic_add_return_release(1, v);
303 }
304 #define atomic_inc_return_release atomic_inc_return_release
305 #endif
306 
307 #ifndef atomic_inc_return_relaxed
308 static inline int
309 atomic_inc_return_relaxed(atomic_t *v)
310 {
311         return atomic_add_return_relaxed(1, v);
312 }
313 #define atomic_inc_return_relaxed atomic_inc_return_relaxed
314 #endif
315 
316 #else /* atomic_inc_return_relaxed */
317 
318 #ifndef atomic_inc_return_acquire
319 static inline int
320 atomic_inc_return_acquire(atomic_t *v)
321 {
322         int ret = atomic_inc_return_relaxed(v);
323         __atomic_acquire_fence();
324         return ret;
325 }
326 #define atomic_inc_return_acquire atomic_inc_return_acquire
327 #endif
328 
329 #ifndef atomic_inc_return_release
330 static inline int
331 atomic_inc_return_release(atomic_t *v)
332 {
333         __atomic_release_fence();
334         return atomic_inc_return_relaxed(v);
335 }
336 #define atomic_inc_return_release atomic_inc_return_release
337 #endif
338 
339 #ifndef atomic_inc_return
340 static inline int
341 atomic_inc_return(atomic_t *v)
342 {
343         int ret;
344         __atomic_pre_full_fence();
345         ret = atomic_inc_return_relaxed(v);
346         __atomic_post_full_fence();
347         return ret;
348 }
349 #define atomic_inc_return atomic_inc_return
350 #endif
351 
352 #endif /* atomic_inc_return_relaxed */
353 
354 #ifndef atomic_fetch_inc_relaxed
355 #ifdef atomic_fetch_inc
356 #define atomic_fetch_inc_acquire atomic_fetch_inc
357 #define atomic_fetch_inc_release atomic_fetch_inc
358 #define atomic_fetch_inc_relaxed atomic_fetch_inc
359 #endif /* atomic_fetch_inc */
360 
361 #ifndef atomic_fetch_inc
362 static inline int
363 atomic_fetch_inc(atomic_t *v)
364 {
365         return atomic_fetch_add(1, v);
366 }
367 #define atomic_fetch_inc atomic_fetch_inc
368 #endif
369 
370 #ifndef atomic_fetch_inc_acquire
371 static inline int
372 atomic_fetch_inc_acquire(atomic_t *v)
373 {
374         return atomic_fetch_add_acquire(1, v);
375 }
376 #define atomic_fetch_inc_acquire atomic_fetch_inc_acquire
377 #endif
378 
379 #ifndef atomic_fetch_inc_release
380 static inline int
381 atomic_fetch_inc_release(atomic_t *v)
382 {
383         return atomic_fetch_add_release(1, v);
384 }
385 #define atomic_fetch_inc_release atomic_fetch_inc_release
386 #endif
387 
388 #ifndef atomic_fetch_inc_relaxed
389 static inline int
390 atomic_fetch_inc_relaxed(atomic_t *v)
391 {
392         return atomic_fetch_add_relaxed(1, v);
393 }
394 #define atomic_fetch_inc_relaxed atomic_fetch_inc_relaxed
395 #endif
396 
397 #else /* atomic_fetch_inc_relaxed */
398 
399 #ifndef atomic_fetch_inc_acquire
400 static inline int
401 atomic_fetch_inc_acquire(atomic_t *v)
402 {
403         int ret = atomic_fetch_inc_relaxed(v);
404         __atomic_acquire_fence();
405         return ret;
406 }
407 #define atomic_fetch_inc_acquire atomic_fetch_inc_acquire
408 #endif
409 
410 #ifndef atomic_fetch_inc_release
411 static inline int
412 atomic_fetch_inc_release(atomic_t *v)
413 {
414         __atomic_release_fence();
415         return atomic_fetch_inc_relaxed(v);
416 }
417 #define atomic_fetch_inc_release atomic_fetch_inc_release
418 #endif
419 
420 #ifndef atomic_fetch_inc
421 static inline int
422 atomic_fetch_inc(atomic_t *v)
423 {
424         int ret;
425         __atomic_pre_full_fence();
426         ret = atomic_fetch_inc_relaxed(v);
427         __atomic_post_full_fence();
428         return ret;
429 }
430 #define atomic_fetch_inc atomic_fetch_inc
431 #endif
432 
433 #endif /* atomic_fetch_inc_relaxed */
434 
435 #ifndef atomic_dec
436 static inline void
437 atomic_dec(atomic_t *v)
438 {
439         atomic_sub(1, v);
440 }
441 #define atomic_dec atomic_dec
442 #endif
443 
444 #ifndef atomic_dec_return_relaxed
445 #ifdef atomic_dec_return
446 #define atomic_dec_return_acquire atomic_dec_return
447 #define atomic_dec_return_release atomic_dec_return
448 #define atomic_dec_return_relaxed atomic_dec_return
449 #endif /* atomic_dec_return */
450 
451 #ifndef atomic_dec_return
452 static inline int
453 atomic_dec_return(atomic_t *v)
454 {
455         return atomic_sub_return(1, v);
456 }
457 #define atomic_dec_return atomic_dec_return
458 #endif
459 
460 #ifndef atomic_dec_return_acquire
461 static inline int
462 atomic_dec_return_acquire(atomic_t *v)
463 {
464         return atomic_sub_return_acquire(1, v);
465 }
466 #define atomic_dec_return_acquire atomic_dec_return_acquire
467 #endif
468 
469 #ifndef atomic_dec_return_release
470 static inline int
471 atomic_dec_return_release(atomic_t *v)
472 {
473         return atomic_sub_return_release(1, v);
474 }
475 #define atomic_dec_return_release atomic_dec_return_release
476 #endif
477 
478 #ifndef atomic_dec_return_relaxed
479 static inline int
480 atomic_dec_return_relaxed(atomic_t *v)
481 {
482         return atomic_sub_return_relaxed(1, v);
483 }
484 #define atomic_dec_return_relaxed atomic_dec_return_relaxed
485 #endif
486 
487 #else /* atomic_dec_return_relaxed */
488 
489 #ifndef atomic_dec_return_acquire
490 static inline int
491 atomic_dec_return_acquire(atomic_t *v)
492 {
493         int ret = atomic_dec_return_relaxed(v);
494         __atomic_acquire_fence();
495         return ret;
496 }
497 #define atomic_dec_return_acquire atomic_dec_return_acquire
498 #endif
499 
500 #ifndef atomic_dec_return_release
501 static inline int
502 atomic_dec_return_release(atomic_t *v)
503 {
504         __atomic_release_fence();
505         return atomic_dec_return_relaxed(v);
506 }
507 #define atomic_dec_return_release atomic_dec_return_release
508 #endif
509 
510 #ifndef atomic_dec_return
511 static inline int
512 atomic_dec_return(atomic_t *v)
513 {
514         int ret;
515         __atomic_pre_full_fence();
516         ret = atomic_dec_return_relaxed(v);
517         __atomic_post_full_fence();
518         return ret;
519 }
520 #define atomic_dec_return atomic_dec_return
521 #endif
522 
523 #endif /* atomic_dec_return_relaxed */
524 
525 #ifndef atomic_fetch_dec_relaxed
526 #ifdef atomic_fetch_dec
527 #define atomic_fetch_dec_acquire atomic_fetch_dec
528 #define atomic_fetch_dec_release atomic_fetch_dec
529 #define atomic_fetch_dec_relaxed atomic_fetch_dec
530 #endif /* atomic_fetch_dec */
531 
532 #ifndef atomic_fetch_dec
533 static inline int
534 atomic_fetch_dec(atomic_t *v)
535 {
536         return atomic_fetch_sub(1, v);
537 }
538 #define atomic_fetch_dec atomic_fetch_dec
539 #endif
540 
541 #ifndef atomic_fetch_dec_acquire
542 static inline int
543 atomic_fetch_dec_acquire(atomic_t *v)
544 {
545         return atomic_fetch_sub_acquire(1, v);
546 }
547 #define atomic_fetch_dec_acquire atomic_fetch_dec_acquire
548 #endif
549 
550 #ifndef atomic_fetch_dec_release
551 static inline int
552 atomic_fetch_dec_release(atomic_t *v)
553 {
554         return atomic_fetch_sub_release(1, v);
555 }
556 #define atomic_fetch_dec_release atomic_fetch_dec_release
557 #endif
558 
559 #ifndef atomic_fetch_dec_relaxed
560 static inline int
561 atomic_fetch_dec_relaxed(atomic_t *v)
562 {
563         return atomic_fetch_sub_relaxed(1, v);
564 }
565 #define atomic_fetch_dec_relaxed atomic_fetch_dec_relaxed
566 #endif
567 
568 #else /* atomic_fetch_dec_relaxed */
569 
570 #ifndef atomic_fetch_dec_acquire
571 static inline int
572 atomic_fetch_dec_acquire(atomic_t *v)
573 {
574         int ret = atomic_fetch_dec_relaxed(v);
575         __atomic_acquire_fence();
576         return ret;
577 }
578 #define atomic_fetch_dec_acquire atomic_fetch_dec_acquire
579 #endif
580 
581 #ifndef atomic_fetch_dec_release
582 static inline int
583 atomic_fetch_dec_release(atomic_t *v)
584 {
585         __atomic_release_fence();
586         return atomic_fetch_dec_relaxed(v);
587 }
588 #define atomic_fetch_dec_release atomic_fetch_dec_release
589 #endif
590 
591 #ifndef atomic_fetch_dec
592 static inline int
593 atomic_fetch_dec(atomic_t *v)
594 {
595         int ret;
596         __atomic_pre_full_fence();
597         ret = atomic_fetch_dec_relaxed(v);
598         __atomic_post_full_fence();
599         return ret;
600 }
601 #define atomic_fetch_dec atomic_fetch_dec
602 #endif
603 
604 #endif /* atomic_fetch_dec_relaxed */
605 
606 #ifndef atomic_fetch_and_relaxed
607 #define atomic_fetch_and_acquire atomic_fetch_and
608 #define atomic_fetch_and_release atomic_fetch_and
609 #define atomic_fetch_and_relaxed atomic_fetch_and
610 #else /* atomic_fetch_and_relaxed */
611 
612 #ifndef atomic_fetch_and_acquire
613 static inline int
614 atomic_fetch_and_acquire(int i, atomic_t *v)
615 {
616         int ret = atomic_fetch_and_relaxed(i, v);
617         __atomic_acquire_fence();
618         return ret;
619 }
620 #define atomic_fetch_and_acquire atomic_fetch_and_acquire
621 #endif
622 
623 #ifndef atomic_fetch_and_release
624 static inline int
625 atomic_fetch_and_release(int i, atomic_t *v)
626 {
627         __atomic_release_fence();
628         return atomic_fetch_and_relaxed(i, v);
629 }
630 #define atomic_fetch_and_release atomic_fetch_and_release
631 #endif
632 
633 #ifndef atomic_fetch_and
634 static inline int
635 atomic_fetch_and(int i, atomic_t *v)
636 {
637         int ret;
638         __atomic_pre_full_fence();
639         ret = atomic_fetch_and_relaxed(i, v);
640         __atomic_post_full_fence();
641         return ret;
642 }
643 #define atomic_fetch_and atomic_fetch_and
644 #endif
645 
646 #endif /* atomic_fetch_and_relaxed */
647 
648 #ifndef atomic_andnot
649 static inline void
650 atomic_andnot(int i, atomic_t *v)
651 {
652         atomic_and(~i, v);
653 }
654 #define atomic_andnot atomic_andnot
655 #endif
656 
657 #ifndef atomic_fetch_andnot_relaxed
658 #ifdef atomic_fetch_andnot
659 #define atomic_fetch_andnot_acquire atomic_fetch_andnot
660 #define atomic_fetch_andnot_release atomic_fetch_andnot
661 #define atomic_fetch_andnot_relaxed atomic_fetch_andnot
662 #endif /* atomic_fetch_andnot */
663 
664 #ifndef atomic_fetch_andnot
665 static inline int
666 atomic_fetch_andnot(int i, atomic_t *v)
667 {
668         return atomic_fetch_and(~i, v);
669 }
670 #define atomic_fetch_andnot atomic_fetch_andnot
671 #endif
672 
673 #ifndef atomic_fetch_andnot_acquire
674 static inline int
675 atomic_fetch_andnot_acquire(int i, atomic_t *v)
676 {
677         return atomic_fetch_and_acquire(~i, v);
678 }
679 #define atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire
680 #endif
681 
682 #ifndef atomic_fetch_andnot_release
683 static inline int
684 atomic_fetch_andnot_release(int i, atomic_t *v)
685 {
686         return atomic_fetch_and_release(~i, v);
687 }
688 #define atomic_fetch_andnot_release atomic_fetch_andnot_release
689 #endif
690 
691 #ifndef atomic_fetch_andnot_relaxed
692 static inline int
693 atomic_fetch_andnot_relaxed(int i, atomic_t *v)
694 {
695         return atomic_fetch_and_relaxed(~i, v);
696 }
697 #define atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed
698 #endif
699 
700 #else /* atomic_fetch_andnot_relaxed */
701 
702 #ifndef atomic_fetch_andnot_acquire
703 static inline int
704 atomic_fetch_andnot_acquire(int i, atomic_t *v)
705 {
706         int ret = atomic_fetch_andnot_relaxed(i, v);
707         __atomic_acquire_fence();
708         return ret;
709 }
710 #define atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire
711 #endif
712 
713 #ifndef atomic_fetch_andnot_release
714 static inline int
715 atomic_fetch_andnot_release(int i, atomic_t *v)
716 {
717         __atomic_release_fence();
718         return atomic_fetch_andnot_relaxed(i, v);
719 }
720 #define atomic_fetch_andnot_release atomic_fetch_andnot_release
721 #endif
722 
723 #ifndef atomic_fetch_andnot
724 static inline int
725 atomic_fetch_andnot(int i, atomic_t *v)
726 {
727         int ret;
728         __atomic_pre_full_fence();
729         ret = atomic_fetch_andnot_relaxed(i, v);
730         __atomic_post_full_fence();
731         return ret;
732 }
733 #define atomic_fetch_andnot atomic_fetch_andnot
734 #endif
735 
736 #endif /* atomic_fetch_andnot_relaxed */
737 
738 #ifndef atomic_fetch_or_relaxed
739 #define atomic_fetch_or_acquire atomic_fetch_or
740 #define atomic_fetch_or_release atomic_fetch_or
741 #define atomic_fetch_or_relaxed atomic_fetch_or
742 #else /* atomic_fetch_or_relaxed */
743 
744 #ifndef atomic_fetch_or_acquire
745 static inline int
746 atomic_fetch_or_acquire(int i, atomic_t *v)
747 {
748         int ret = atomic_fetch_or_relaxed(i, v);
749         __atomic_acquire_fence();
750         return ret;
751 }
752 #define atomic_fetch_or_acquire atomic_fetch_or_acquire
753 #endif
754 
755 #ifndef atomic_fetch_or_release
756 static inline int
757 atomic_fetch_or_release(int i, atomic_t *v)
758 {
759         __atomic_release_fence();
760         return atomic_fetch_or_relaxed(i, v);
761 }
762 #define atomic_fetch_or_release atomic_fetch_or_release
763 #endif
764 
765 #ifndef atomic_fetch_or
766 static inline int
767 atomic_fetch_or(int i, atomic_t *v)
768 {
769         int ret;
770         __atomic_pre_full_fence();
771         ret = atomic_fetch_or_relaxed(i, v);
772         __atomic_post_full_fence();
773         return ret;
774 }
775 #define atomic_fetch_or atomic_fetch_or
776 #endif
777 
778 #endif /* atomic_fetch_or_relaxed */
779 
780 #ifndef atomic_fetch_xor_relaxed
781 #define atomic_fetch_xor_acquire atomic_fetch_xor
782 #define atomic_fetch_xor_release atomic_fetch_xor
783 #define atomic_fetch_xor_relaxed atomic_fetch_xor
784 #else /* atomic_fetch_xor_relaxed */
785 
786 #ifndef atomic_fetch_xor_acquire
787 static inline int
788 atomic_fetch_xor_acquire(int i, atomic_t *v)
789 {
790         int ret = atomic_fetch_xor_relaxed(i, v);
791         __atomic_acquire_fence();
792         return ret;
793 }
794 #define atomic_fetch_xor_acquire atomic_fetch_xor_acquire
795 #endif
796 
797 #ifndef atomic_fetch_xor_release
798 static inline int
799 atomic_fetch_xor_release(int i, atomic_t *v)
800 {
801         __atomic_release_fence();
802         return atomic_fetch_xor_relaxed(i, v);
803 }
804 #define atomic_fetch_xor_release atomic_fetch_xor_release
805 #endif
806 
807 #ifndef atomic_fetch_xor
808 static inline int
809 atomic_fetch_xor(int i, atomic_t *v)
810 {
811         int ret;
812         __atomic_pre_full_fence();
813         ret = atomic_fetch_xor_relaxed(i, v);
814         __atomic_post_full_fence();
815         return ret;
816 }
817 #define atomic_fetch_xor atomic_fetch_xor
818 #endif
819 
820 #endif /* atomic_fetch_xor_relaxed */
821 
822 #ifndef atomic_xchg_relaxed
823 #define atomic_xchg_acquire atomic_xchg
824 #define atomic_xchg_release atomic_xchg
825 #define atomic_xchg_relaxed atomic_xchg
826 #else /* atomic_xchg_relaxed */
827 
828 #ifndef atomic_xchg_acquire
829 static inline int
830 atomic_xchg_acquire(atomic_t *v, int i)
831 {
832         int ret = atomic_xchg_relaxed(v, i);
833         __atomic_acquire_fence();
834         return ret;
835 }
836 #define atomic_xchg_acquire atomic_xchg_acquire
837 #endif
838 
839 #ifndef atomic_xchg_release
840 static inline int
841 atomic_xchg_release(atomic_t *v, int i)
842 {
843         __atomic_release_fence();
844         return atomic_xchg_relaxed(v, i);
845 }
846 #define atomic_xchg_release atomic_xchg_release
847 #endif
848 
849 #ifndef atomic_xchg
850 static inline int
851 atomic_xchg(atomic_t *v, int i)
852 {
853         int ret;
854         __atomic_pre_full_fence();
855         ret = atomic_xchg_relaxed(v, i);
856         __atomic_post_full_fence();
857         return ret;
858 }
859 #define atomic_xchg atomic_xchg
860 #endif
861 
862 #endif /* atomic_xchg_relaxed */
863 
864 #ifndef atomic_cmpxchg_relaxed
865 #define atomic_cmpxchg_acquire atomic_cmpxchg
866 #define atomic_cmpxchg_release atomic_cmpxchg
867 #define atomic_cmpxchg_relaxed atomic_cmpxchg
868 #else /* atomic_cmpxchg_relaxed */
869 
870 #ifndef atomic_cmpxchg_acquire
871 static inline int
872 atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
873 {
874         int ret = atomic_cmpxchg_relaxed(v, old, new);
875         __atomic_acquire_fence();
876         return ret;
877 }
878 #define atomic_cmpxchg_acquire atomic_cmpxchg_acquire
879 #endif
880 
881 #ifndef atomic_cmpxchg_release
882 static inline int
883 atomic_cmpxchg_release(atomic_t *v, int old, int new)
884 {
885         __atomic_release_fence();
886         return atomic_cmpxchg_relaxed(v, old, new);
887 }
888 #define atomic_cmpxchg_release atomic_cmpxchg_release
889 #endif
890 
891 #ifndef atomic_cmpxchg
892 static inline int
893 atomic_cmpxchg(atomic_t *v, int old, int new)
894 {
895         int ret;
896         __atomic_pre_full_fence();
897         ret = atomic_cmpxchg_relaxed(v, old, new);
898         __atomic_post_full_fence();
899         return ret;
900 }
901 #define atomic_cmpxchg atomic_cmpxchg
902 #endif
903 
904 #endif /* atomic_cmpxchg_relaxed */
905 
906 #ifndef atomic_try_cmpxchg_relaxed
907 #ifdef atomic_try_cmpxchg
908 #define atomic_try_cmpxchg_acquire atomic_try_cmpxchg
909 #define atomic_try_cmpxchg_release atomic_try_cmpxchg
910 #define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg
911 #endif /* atomic_try_cmpxchg */
912 
913 #ifndef atomic_try_cmpxchg
914 static inline bool
915 atomic_try_cmpxchg(atomic_t *v, int *old, int new)
916 {
917         int r, o = *old;
918         r = atomic_cmpxchg(v, o, new);
919         if (unlikely(r != o))
920                 *old = r;
921         return likely(r == o);
922 }
923 #define atomic_try_cmpxchg atomic_try_cmpxchg
924 #endif
925 
926 #ifndef atomic_try_cmpxchg_acquire
927 static inline bool
928 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
929 {
930         int r, o = *old;
931         r = atomic_cmpxchg_acquire(v, o, new);
932         if (unlikely(r != o))
933                 *old = r;
934         return likely(r == o);
935 }
936 #define atomic_try_cmpxchg_acquire atomic_try_cmpxchg_acquire
937 #endif
938 
939 #ifndef atomic_try_cmpxchg_release
940 static inline bool
941 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
942 {
943         int r, o = *old;
944         r = atomic_cmpxchg_release(v, o, new);
945         if (unlikely(r != o))
946                 *old = r;
947         return likely(r == o);
948 }
949 #define atomic_try_cmpxchg_release atomic_try_cmpxchg_release
950 #endif
951 
952 #ifndef atomic_try_cmpxchg_relaxed
953 static inline bool
954 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
955 {
956         int r, o = *old;
957         r = atomic_cmpxchg_relaxed(v, o, new);
958         if (unlikely(r != o))
959                 *old = r;
960         return likely(r == o);
961 }
962 #define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg_relaxed
963 #endif
964 
965 #else /* atomic_try_cmpxchg_relaxed */
966 
967 #ifndef atomic_try_cmpxchg_acquire
968 static inline bool
969 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
970 {
971         bool ret = atomic_try_cmpxchg_relaxed(v, old, new);
972         __atomic_acquire_fence();
973         return ret;
974 }
975 #define atomic_try_cmpxchg_acquire atomic_try_cmpxchg_acquire
976 #endif
977 
978 #ifndef atomic_try_cmpxchg_release
979 static inline bool
980 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
981 {
982         __atomic_release_fence();
983         return atomic_try_cmpxchg_relaxed(v, old, new);
984 }
985 #define atomic_try_cmpxchg_release atomic_try_cmpxchg_release
986 #endif
987 
988 #ifndef atomic_try_cmpxchg
989 static inline bool
990 atomic_try_cmpxchg(atomic_t *v, int *old, int new)
991 {
992         bool ret;
993         __atomic_pre_full_fence();
994         ret = atomic_try_cmpxchg_relaxed(v, old, new);
995         __atomic_post_full_fence();
996         return ret;
997 }
998 #define atomic_try_cmpxchg atomic_try_cmpxchg
999 #endif
1000 
1001 #endif /* atomic_try_cmpxchg_relaxed */
1002 
1003 #ifndef atomic_sub_and_test
1004 /**
1005  * atomic_sub_and_test - subtract value from variable and test result
1006  * @i: integer value to subtract
1007  * @v: pointer of type atomic_t
1008  *
1009  * Atomically subtracts @i from @v and returns
1010  * true if the result is zero, or false for all
1011  * other cases.
1012  */
1013 static inline bool
1014 atomic_sub_and_test(int i, atomic_t *v)
1015 {
1016         return atomic_sub_return(i, v) == 0;
1017 }
1018 #define atomic_sub_and_test atomic_sub_and_test
1019 #endif
1020 
1021 #ifndef atomic_dec_and_test
1022 /**
1023  * atomic_dec_and_test - decrement and test
1024  * @v: pointer of type atomic_t
1025  *
1026  * Atomically decrements @v by 1 and
1027  * returns true if the result is 0, or false for all other
1028  * cases.
1029  */
1030 static inline bool
1031 atomic_dec_and_test(atomic_t *v)
1032 {
1033         return atomic_dec_return(v) == 0;
1034 }
1035 #define atomic_dec_and_test atomic_dec_and_test
1036 #endif
1037 
1038 #ifndef atomic_inc_and_test
1039 /**
1040  * atomic_inc_and_test - increment and test
1041  * @v: pointer of type atomic_t
1042  *
1043  * Atomically increments @v by 1
1044  * and returns true if the result is zero, or false for all
1045  * other cases.
1046  */
1047 static inline bool
1048 atomic_inc_and_test(atomic_t *v)
1049 {
1050         return atomic_inc_return(v) == 0;
1051 }
1052 #define atomic_inc_and_test atomic_inc_and_test
1053 #endif
1054 
1055 #ifndef atomic_add_negative
1056 /**
1057  * atomic_add_negative - add and test if negative
1058  * @i: integer value to add
1059  * @v: pointer of type atomic_t
1060  *
1061  * Atomically adds @i to @v and returns true
1062  * if the result is negative, or false when
1063  * result is greater than or equal to zero.
1064  */
1065 static inline bool
1066 atomic_add_negative(int i, atomic_t *v)
1067 {
1068         return atomic_add_return(i, v) < 0;
1069 }
1070 #define atomic_add_negative atomic_add_negative
1071 #endif
1072 
1073 #ifndef atomic_fetch_add_unless
1074 /**
1075  * atomic_fetch_add_unless - add unless the number is already a given value
1076  * @v: pointer of type atomic_t
1077  * @a: the amount to add to v...
1078  * @u: ...unless v is equal to u.
1079  *
1080  * Atomically adds @a to @v, so long as @v was not already @u.
1081  * Returns original value of @v
1082  */
1083 static inline int
1084 atomic_fetch_add_unless(atomic_t *v, int a, int u)
1085 {
1086         int c = atomic_read(v);
1087 
1088         do {
1089                 if (unlikely(c == u))
1090                         break;
1091         } while (!atomic_try_cmpxchg(v, &c, c + a));
1092 
1093         return c;
1094 }
1095 #define atomic_fetch_add_unless atomic_fetch_add_unless
1096 #endif
1097 
1098 #ifndef atomic_add_unless
1099 /**
1100  * atomic_add_unless - add unless the number is already a given value
1101  * @v: pointer of type atomic_t
1102  * @a: the amount to add to v...
1103  * @u: ...unless v is equal to u.
1104  *
1105  * Atomically adds @a to @v, if @v was not already @u.
1106  * Returns true if the addition was done.
1107  */
1108 static inline bool
1109 atomic_add_unless(atomic_t *v, int a, int u)
1110 {
1111         return atomic_fetch_add_unless(v, a, u) != u;
1112 }
1113 #define atomic_add_unless atomic_add_unless
1114 #endif
1115 
1116 #ifndef atomic_inc_not_zero
1117 /**
1118  * atomic_inc_not_zero - increment unless the number is zero
1119  * @v: pointer of type atomic_t
1120  *
1121  * Atomically increments @v by 1, if @v is non-zero.
1122  * Returns true if the increment was done.
1123  */
1124 static inline bool
1125 atomic_inc_not_zero(atomic_t *v)
1126 {
1127         return atomic_add_unless(v, 1, 0);
1128 }
1129 #define atomic_inc_not_zero atomic_inc_not_zero
1130 #endif
1131 
1132 #ifndef atomic_inc_unless_negative
1133 static inline bool
1134 atomic_inc_unless_negative(atomic_t *v)
1135 {
1136         int c = atomic_read(v);
1137 
1138         do {
1139                 if (unlikely(c < 0))
1140                         return false;
1141         } while (!atomic_try_cmpxchg(v, &c, c + 1));
1142 
1143         return true;
1144 }
1145 #define atomic_inc_unless_negative atomic_inc_unless_negative
1146 #endif
1147 
1148 #ifndef atomic_dec_unless_positive
1149 static inline bool
1150 atomic_dec_unless_positive(atomic_t *v)
1151 {
1152         int c = atomic_read(v);
1153 
1154         do {
1155                 if (unlikely(c > 0))
1156                         return false;
1157         } while (!atomic_try_cmpxchg(v, &c, c - 1));
1158 
1159         return true;
1160 }
1161 #define atomic_dec_unless_positive atomic_dec_unless_positive
1162 #endif
1163 
1164 #ifndef atomic_dec_if_positive
1165 static inline int
1166 atomic_dec_if_positive(atomic_t *v)
1167 {
1168         int dec, c = atomic_read(v);
1169 
1170         do {
1171                 dec = c - 1;
1172                 if (unlikely(dec < 0))
1173                         break;
1174         } while (!atomic_try_cmpxchg(v, &c, dec));
1175 
1176         return dec;
1177 }
1178 #define atomic_dec_if_positive atomic_dec_if_positive
1179 #endif
1180 
1181 #define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
1182 #define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
1183 
1184 #ifdef CONFIG_GENERIC_ATOMIC64
1185 #include <asm-generic/atomic64.h>
1186 #endif
1187 
1188 #ifndef atomic64_read_acquire
1189 static inline s64
1190 atomic64_read_acquire(const atomic64_t *v)
1191 {
1192         return smp_load_acquire(&(v)->counter);
1193 }
1194 #define atomic64_read_acquire atomic64_read_acquire
1195 #endif
1196 
1197 #ifndef atomic64_set_release
1198 static inline void
1199 atomic64_set_release(atomic64_t *v, s64 i)
1200 {
1201         smp_store_release(&(v)->counter, i);
1202 }
1203 #define atomic64_set_release atomic64_set_release
1204 #endif
1205 
1206 #ifndef atomic64_add_return_relaxed
1207 #define atomic64_add_return_acquire atomic64_add_return
1208 #define atomic64_add_return_release atomic64_add_return
1209 #define atomic64_add_return_relaxed atomic64_add_return
1210 #else /* atomic64_add_return_relaxed */
1211 
1212 #ifndef atomic64_add_return_acquire
1213 static inline s64
1214 atomic64_add_return_acquire(s64 i, atomic64_t *v)
1215 {
1216         s64 ret = atomic64_add_return_relaxed(i, v);
1217         __atomic_acquire_fence();
1218         return ret;
1219 }
1220 #define atomic64_add_return_acquire atomic64_add_return_acquire
1221 #endif
1222 
1223 #ifndef atomic64_add_return_release
1224 static inline s64
1225 atomic64_add_return_release(s64 i, atomic64_t *v)
1226 {
1227         __atomic_release_fence();
1228         return atomic64_add_return_relaxed(i, v);
1229 }
1230 #define atomic64_add_return_release atomic64_add_return_release
1231 #endif
1232 
1233 #ifndef atomic64_add_return
1234 static inline s64
1235 atomic64_add_return(s64 i, atomic64_t *v)
1236 {
1237         s64 ret;
1238         __atomic_pre_full_fence();
1239         ret = atomic64_add_return_relaxed(i, v);
1240         __atomic_post_full_fence();
1241         return ret;
1242 }
1243 #define atomic64_add_return atomic64_add_return
1244 #endif
1245 
1246 #endif /* atomic64_add_return_relaxed */
1247 
1248 #ifndef atomic64_fetch_add_relaxed
1249 #define atomic64_fetch_add_acquire atomic64_fetch_add
1250 #define atomic64_fetch_add_release atomic64_fetch_add
1251 #define atomic64_fetch_add_relaxed atomic64_fetch_add
1252 #else /* atomic64_fetch_add_relaxed */
1253 
1254 #ifndef atomic64_fetch_add_acquire
1255 static inline s64
1256 atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1257 {
1258         s64 ret = atomic64_fetch_add_relaxed(i, v);
1259         __atomic_acquire_fence();
1260         return ret;
1261 }
1262 #define atomic64_fetch_add_acquire atomic64_fetch_add_acquire
1263 #endif
1264 
1265 #ifndef atomic64_fetch_add_release
1266 static inline s64
1267 atomic64_fetch_add_release(s64 i, atomic64_t *v)
1268 {
1269         __atomic_release_fence();
1270         return atomic64_fetch_add_relaxed(i, v);
1271 }
1272 #define atomic64_fetch_add_release atomic64_fetch_add_release
1273 #endif
1274 
1275 #ifndef atomic64_fetch_add
1276 static inline s64
1277 atomic64_fetch_add(s64 i, atomic64_t *v)
1278 {
1279         s64 ret;
1280         __atomic_pre_full_fence();
1281         ret = atomic64_fetch_add_relaxed(i, v);
1282         __atomic_post_full_fence();
1283         return ret;
1284 }
1285 #define atomic64_fetch_add atomic64_fetch_add
1286 #endif
1287 
1288 #endif /* atomic64_fetch_add_relaxed */
1289 
1290 #ifndef atomic64_sub_return_relaxed
1291 #define atomic64_sub_return_acquire atomic64_sub_return
1292 #define atomic64_sub_return_release atomic64_sub_return
1293 #define atomic64_sub_return_relaxed atomic64_sub_return
1294 #else /* atomic64_sub_return_relaxed */
1295 
1296 #ifndef atomic64_sub_return_acquire
1297 static inline s64
1298 atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1299 {
1300         s64 ret = atomic64_sub_return_relaxed(i, v);
1301         __atomic_acquire_fence();
1302         return ret;
1303 }
1304 #define atomic64_sub_return_acquire atomic64_sub_return_acquire
1305 #endif
1306 
1307 #ifndef atomic64_sub_return_release
1308 static inline s64
1309 atomic64_sub_return_release(s64 i, atomic64_t *v)
1310 {
1311         __atomic_release_fence();
1312         return atomic64_sub_return_relaxed(i, v);
1313 }
1314 #define atomic64_sub_return_release atomic64_sub_return_release
1315 #endif
1316 
1317 #ifndef atomic64_sub_return
1318 static inline s64
1319 atomic64_sub_return(s64 i, atomic64_t *v)
1320 {
1321         s64 ret;
1322         __atomic_pre_full_fence();
1323         ret = atomic64_sub_return_relaxed(i, v);
1324         __atomic_post_full_fence();
1325         return ret;
1326 }
1327 #define atomic64_sub_return atomic64_sub_return
1328 #endif
1329 
1330 #endif /* atomic64_sub_return_relaxed */
1331 
1332 #ifndef atomic64_fetch_sub_relaxed
1333 #define atomic64_fetch_sub_acquire atomic64_fetch_sub
1334 #define atomic64_fetch_sub_release atomic64_fetch_sub
1335 #define atomic64_fetch_sub_relaxed atomic64_fetch_sub
1336 #else /* atomic64_fetch_sub_relaxed */
1337 
1338 #ifndef atomic64_fetch_sub_acquire
1339 static inline s64
1340 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1341 {
1342         s64 ret = atomic64_fetch_sub_relaxed(i, v);
1343         __atomic_acquire_fence();
1344         return ret;
1345 }
1346 #define atomic64_fetch_sub_acquire atomic64_fetch_sub_acquire
1347 #endif
1348 
1349 #ifndef atomic64_fetch_sub_release
1350 static inline s64
1351 atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1352 {
1353         __atomic_release_fence();
1354         return atomic64_fetch_sub_relaxed(i, v);
1355 }
1356 #define atomic64_fetch_sub_release atomic64_fetch_sub_release
1357 #endif
1358 
1359 #ifndef atomic64_fetch_sub
1360 static inline s64
1361 atomic64_fetch_sub(s64 i, atomic64_t *v)
1362 {
1363         s64 ret;
1364         __atomic_pre_full_fence();
1365         ret = atomic64_fetch_sub_relaxed(i, v);
1366         __atomic_post_full_fence();
1367         return ret;
1368 }
1369 #define atomic64_fetch_sub atomic64_fetch_sub
1370 #endif
1371 
1372 #endif /* atomic64_fetch_sub_relaxed */
1373 
1374 #ifndef atomic64_inc
1375 static inline void
1376 atomic64_inc(atomic64_t *v)
1377 {
1378         atomic64_add(1, v);
1379 }
1380 #define atomic64_inc atomic64_inc
1381 #endif
1382 
1383 #ifndef atomic64_inc_return_relaxed
1384 #ifdef atomic64_inc_return
1385 #define atomic64_inc_return_acquire atomic64_inc_return
1386 #define atomic64_inc_return_release atomic64_inc_return
1387 #define atomic64_inc_return_relaxed atomic64_inc_return
1388 #endif /* atomic64_inc_return */
1389 
1390 #ifndef atomic64_inc_return
1391 static inline s64
1392 atomic64_inc_return(atomic64_t *v)
1393 {
1394         return atomic64_add_return(1, v);
1395 }
1396 #define atomic64_inc_return atomic64_inc_return
1397 #endif
1398 
1399 #ifndef atomic64_inc_return_acquire
1400 static inline s64
1401 atomic64_inc_return_acquire(atomic64_t *v)
1402 {
1403         return atomic64_add_return_acquire(1, v);
1404 }
1405 #define atomic64_inc_return_acquire atomic64_inc_return_acquire
1406 #endif
1407 
1408 #ifndef atomic64_inc_return_release
1409 static inline s64
1410 atomic64_inc_return_release(atomic64_t *v)
1411 {
1412         return atomic64_add_return_release(1, v);
1413 }
1414 #define atomic64_inc_return_release atomic64_inc_return_release
1415 #endif
1416 
1417 #ifndef atomic64_inc_return_relaxed
1418 static inline s64
1419 atomic64_inc_return_relaxed(atomic64_t *v)
1420 {
1421         return atomic64_add_return_relaxed(1, v);
1422 }
1423 #define atomic64_inc_return_relaxed atomic64_inc_return_relaxed
1424 #endif
1425 
1426 #else /* atomic64_inc_return_relaxed */
1427 
1428 #ifndef atomic64_inc_return_acquire
1429 static inline s64
1430 atomic64_inc_return_acquire(atomic64_t *v)
1431 {
1432         s64 ret = atomic64_inc_return_relaxed(v);
1433         __atomic_acquire_fence();
1434         return ret;
1435 }
1436 #define atomic64_inc_return_acquire atomic64_inc_return_acquire
1437 #endif
1438 
1439 #ifndef atomic64_inc_return_release
1440 static inline s64
1441 atomic64_inc_return_release(atomic64_t *v)
1442 {
1443         __atomic_release_fence();
1444         return atomic64_inc_return_relaxed(v);
1445 }
1446 #define atomic64_inc_return_release atomic64_inc_return_release
1447 #endif
1448 
1449 #ifndef atomic64_inc_return
1450 static inline s64
1451 atomic64_inc_return(atomic64_t *v)
1452 {
1453         s64 ret;
1454         __atomic_pre_full_fence();
1455         ret = atomic64_inc_return_relaxed(v);
1456         __atomic_post_full_fence();
1457         return ret;
1458 }
1459 #define atomic64_inc_return atomic64_inc_return
1460 #endif
1461 
1462 #endif /* atomic64_inc_return_relaxed */
1463 
1464 #ifndef atomic64_fetch_inc_relaxed
1465 #ifdef atomic64_fetch_inc
1466 #define atomic64_fetch_inc_acquire atomic64_fetch_inc
1467 #define atomic64_fetch_inc_release atomic64_fetch_inc
1468 #define atomic64_fetch_inc_relaxed atomic64_fetch_inc
1469 #endif /* atomic64_fetch_inc */
1470 
1471 #ifndef atomic64_fetch_inc
1472 static inline s64
1473 atomic64_fetch_inc(atomic64_t *v)
1474 {
1475         return atomic64_fetch_add(1, v);
1476 }
1477 #define atomic64_fetch_inc atomic64_fetch_inc
1478 #endif
1479 
1480 #ifndef atomic64_fetch_inc_acquire
1481 static inline s64
1482 atomic64_fetch_inc_acquire(atomic64_t *v)
1483 {
1484         return atomic64_fetch_add_acquire(1, v);
1485 }
1486 #define atomic64_fetch_inc_acquire atomic64_fetch_inc_acquire
1487 #endif
1488 
1489 #ifndef atomic64_fetch_inc_release
1490 static inline s64
1491 atomic64_fetch_inc_release(atomic64_t *v)
1492 {
1493         return atomic64_fetch_add_release(1, v);
1494 }
1495 #define atomic64_fetch_inc_release atomic64_fetch_inc_release
1496 #endif
1497 
1498 #ifndef atomic64_fetch_inc_relaxed
1499 static inline s64
1500 atomic64_fetch_inc_relaxed(atomic64_t *v)
1501 {
1502         return atomic64_fetch_add_relaxed(1, v);
1503 }
1504 #define atomic64_fetch_inc_relaxed atomic64_fetch_inc_relaxed
1505 #endif
1506 
1507 #else /* atomic64_fetch_inc_relaxed */
1508 
1509 #ifndef atomic64_fetch_inc_acquire
1510 static inline s64
1511 atomic64_fetch_inc_acquire(atomic64_t *v)
1512 {
1513         s64 ret = atomic64_fetch_inc_relaxed(v);
1514         __atomic_acquire_fence();
1515         return ret;
1516 }
1517 #define atomic64_fetch_inc_acquire atomic64_fetch_inc_acquire
1518 #endif
1519 
1520 #ifndef atomic64_fetch_inc_release
1521 static inline s64
1522 atomic64_fetch_inc_release(atomic64_t *v)
1523 {
1524         __atomic_release_fence();
1525         return atomic64_fetch_inc_relaxed(v);
1526 }
1527 #define atomic64_fetch_inc_release atomic64_fetch_inc_release
1528 #endif
1529 
1530 #ifndef atomic64_fetch_inc
1531 static inline s64
1532 atomic64_fetch_inc(atomic64_t *v)
1533 {
1534         s64 ret;
1535         __atomic_pre_full_fence();
1536         ret = atomic64_fetch_inc_relaxed(v);
1537         __atomic_post_full_fence();
1538         return ret;
1539 }
1540 #define atomic64_fetch_inc atomic64_fetch_inc
1541 #endif
1542 
1543 #endif /* atomic64_fetch_inc_relaxed */
1544 
1545 #ifndef atomic64_dec
1546 static inline void
1547 atomic64_dec(atomic64_t *v)
1548 {
1549         atomic64_sub(1, v);
1550 }
1551 #define atomic64_dec atomic64_dec
1552 #endif
1553 
1554 #ifndef atomic64_dec_return_relaxed
1555 #ifdef atomic64_dec_return
1556 #define atomic64_dec_return_acquire atomic64_dec_return
1557 #define atomic64_dec_return_release atomic64_dec_return
1558 #define atomic64_dec_return_relaxed atomic64_dec_return
1559 #endif /* atomic64_dec_return */
1560 
1561 #ifndef atomic64_dec_return
1562 static inline s64
1563 atomic64_dec_return(atomic64_t *v)
1564 {
1565         return atomic64_sub_return(1, v);
1566 }
1567 #define atomic64_dec_return atomic64_dec_return
1568 #endif
1569 
1570 #ifndef atomic64_dec_return_acquire
1571 static inline s64
1572 atomic64_dec_return_acquire(atomic64_t *v)
1573 {
1574         return atomic64_sub_return_acquire(1, v);
1575 }
1576 #define atomic64_dec_return_acquire atomic64_dec_return_acquire
1577 #endif
1578 
1579 #ifndef atomic64_dec_return_release
1580 static inline s64
1581 atomic64_dec_return_release(atomic64_t *v)
1582 {
1583         return atomic64_sub_return_release(1, v);
1584 }
1585 #define atomic64_dec_return_release atomic64_dec_return_release
1586 #endif
1587 
1588 #ifndef atomic64_dec_return_relaxed
1589 static inline s64
1590 atomic64_dec_return_relaxed(atomic64_t *v)
1591 {
1592         return atomic64_sub_return_relaxed(1, v);
1593 }
1594 #define atomic64_dec_return_relaxed atomic64_dec_return_relaxed
1595 #endif
1596 
1597 #else /* atomic64_dec_return_relaxed */
1598 
1599 #ifndef atomic64_dec_return_acquire
1600 static inline s64
1601 atomic64_dec_return_acquire(atomic64_t *v)
1602 {
1603         s64 ret = atomic64_dec_return_relaxed(v);
1604         __atomic_acquire_fence();
1605         return ret;
1606 }
1607 #define atomic64_dec_return_acquire atomic64_dec_return_acquire
1608 #endif
1609 
1610 #ifndef atomic64_dec_return_release
1611 static inline s64
1612 atomic64_dec_return_release(atomic64_t *v)
1613 {
1614         __atomic_release_fence();
1615         return atomic64_dec_return_relaxed(v);
1616 }
1617 #define atomic64_dec_return_release atomic64_dec_return_release
1618 #endif
1619 
1620 #ifndef atomic64_dec_return
1621 static inline s64
1622 atomic64_dec_return(atomic64_t *v)
1623 {
1624         s64 ret;
1625         __atomic_pre_full_fence();
1626         ret = atomic64_dec_return_relaxed(v);
1627         __atomic_post_full_fence();
1628         return ret;
1629 }
1630 #define atomic64_dec_return atomic64_dec_return
1631 #endif
1632 
1633 #endif /* atomic64_dec_return_relaxed */
1634 
1635 #ifndef atomic64_fetch_dec_relaxed
1636 #ifdef atomic64_fetch_dec
1637 #define atomic64_fetch_dec_acquire atomic64_fetch_dec
1638 #define atomic64_fetch_dec_release atomic64_fetch_dec
1639 #define atomic64_fetch_dec_relaxed atomic64_fetch_dec
1640 #endif /* atomic64_fetch_dec */
1641 
1642 #ifndef atomic64_fetch_dec
1643 static inline s64
1644 atomic64_fetch_dec(atomic64_t *v)
1645 {
1646         return atomic64_fetch_sub(1, v);
1647 }
1648 #define atomic64_fetch_dec atomic64_fetch_dec
1649 #endif
1650 
1651 #ifndef atomic64_fetch_dec_acquire
1652 static inline s64
1653 atomic64_fetch_dec_acquire(atomic64_t *v)
1654 {
1655         return atomic64_fetch_sub_acquire(1, v);
1656 }
1657 #define atomic64_fetch_dec_acquire atomic64_fetch_dec_acquire
1658 #endif
1659 
1660 #ifndef atomic64_fetch_dec_release
1661 static inline s64
1662 atomic64_fetch_dec_release(atomic64_t *v)
1663 {
1664         return atomic64_fetch_sub_release(1, v);
1665 }
1666 #define atomic64_fetch_dec_release atomic64_fetch_dec_release
1667 #endif
1668 
1669 #ifndef atomic64_fetch_dec_relaxed
1670 static inline s64
1671 atomic64_fetch_dec_relaxed(atomic64_t *v)
1672 {
1673         return atomic64_fetch_sub_relaxed(1, v);
1674 }
1675 #define atomic64_fetch_dec_relaxed atomic64_fetch_dec_relaxed
1676 #endif
1677 
1678 #else /* atomic64_fetch_dec_relaxed */
1679 
1680 #ifndef atomic64_fetch_dec_acquire
1681 static inline s64
1682 atomic64_fetch_dec_acquire(atomic64_t *v)
1683 {
1684         s64 ret = atomic64_fetch_dec_relaxed(v);
1685         __atomic_acquire_fence();
1686         return ret;
1687 }
1688 #define atomic64_fetch_dec_acquire atomic64_fetch_dec_acquire
1689 #endif
1690 
1691 #ifndef atomic64_fetch_dec_release
1692 static inline s64
1693 atomic64_fetch_dec_release(atomic64_t *v)
1694 {
1695         __atomic_release_fence();
1696         return atomic64_fetch_dec_relaxed(v);
1697 }
1698 #define atomic64_fetch_dec_release atomic64_fetch_dec_release
1699 #endif
1700 
1701 #ifndef atomic64_fetch_dec
1702 static inline s64
1703 atomic64_fetch_dec(atomic64_t *v)
1704 {
1705         s64 ret;
1706         __atomic_pre_full_fence();
1707         ret = atomic64_fetch_dec_relaxed(v);
1708         __atomic_post_full_fence();
1709         return ret;
1710 }
1711 #define atomic64_fetch_dec atomic64_fetch_dec
1712 #endif
1713 
1714 #endif /* atomic64_fetch_dec_relaxed */
1715 
1716 #ifndef atomic64_fetch_and_relaxed
1717 #define atomic64_fetch_and_acquire atomic64_fetch_and
1718 #define atomic64_fetch_and_release atomic64_fetch_and
1719 #define atomic64_fetch_and_relaxed atomic64_fetch_and
1720 #else /* atomic64_fetch_and_relaxed */
1721 
1722 #ifndef atomic64_fetch_and_acquire
1723 static inline s64
1724 atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
1725 {
1726         s64 ret = atomic64_fetch_and_relaxed(i, v);
1727         __atomic_acquire_fence();
1728         return ret;
1729 }
1730 #define atomic64_fetch_and_acquire atomic64_fetch_and_acquire
1731 #endif
1732 
1733 #ifndef atomic64_fetch_and_release
1734 static inline s64
1735 atomic64_fetch_and_release(s64 i, atomic64_t *v)
1736 {
1737         __atomic_release_fence();
1738         return atomic64_fetch_and_relaxed(i, v);
1739 }
1740 #define atomic64_fetch_and_release atomic64_fetch_and_release
1741 #endif
1742 
1743 #ifndef atomic64_fetch_and
1744 static inline s64
1745 atomic64_fetch_and(s64 i, atomic64_t *v)
1746 {
1747         s64 ret;
1748         __atomic_pre_full_fence();
1749         ret = atomic64_fetch_and_relaxed(i, v);
1750         __atomic_post_full_fence();
1751         return ret;
1752 }
1753 #define atomic64_fetch_and atomic64_fetch_and
1754 #endif
1755 
1756 #endif /* atomic64_fetch_and_relaxed */
1757 
1758 #ifndef atomic64_andnot
1759 static inline void
1760 atomic64_andnot(s64 i, atomic64_t *v)
1761 {
1762         atomic64_and(~i, v);
1763 }
1764 #define atomic64_andnot atomic64_andnot
1765 #endif
1766 
1767 #ifndef atomic64_fetch_andnot_relaxed
1768 #ifdef atomic64_fetch_andnot
1769 #define atomic64_fetch_andnot_acquire atomic64_fetch_andnot
1770 #define atomic64_fetch_andnot_release atomic64_fetch_andnot
1771 #define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot
1772 #endif /* atomic64_fetch_andnot */
1773 
1774 #ifndef atomic64_fetch_andnot
1775 static inline s64
1776 atomic64_fetch_andnot(s64 i, atomic64_t *v)
1777 {
1778         return atomic64_fetch_and(~i, v);
1779 }
1780 #define atomic64_fetch_andnot atomic64_fetch_andnot
1781 #endif
1782 
1783 #ifndef atomic64_fetch_andnot_acquire
1784 static inline s64
1785 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1786 {
1787         return atomic64_fetch_and_acquire(~i, v);
1788 }
1789 #define atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire
1790 #endif
1791 
1792 #ifndef atomic64_fetch_andnot_release
1793 static inline s64
1794 atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1795 {
1796         return atomic64_fetch_and_release(~i, v);
1797 }
1798 #define atomic64_fetch_andnot_release atomic64_fetch_andnot_release
1799 #endif
1800 
1801 #ifndef atomic64_fetch_andnot_relaxed
1802 static inline s64
1803 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1804 {
1805         return atomic64_fetch_and_relaxed(~i, v);
1806 }
1807 #define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed
1808 #endif
1809 
1810 #else /* atomic64_fetch_andnot_relaxed */
1811 
1812 #ifndef atomic64_fetch_andnot_acquire
1813 static inline s64
1814 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1815 {
1816         s64 ret = atomic64_fetch_andnot_relaxed(i, v);
1817         __atomic_acquire_fence();
1818         return ret;
1819 }
1820 #define atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire
1821 #endif
1822 
1823 #ifndef atomic64_fetch_andnot_release
1824 static inline s64
1825 atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1826 {
1827         __atomic_release_fence();
1828         return atomic64_fetch_andnot_relaxed(i, v);
1829 }
1830 #define atomic64_fetch_andnot_release atomic64_fetch_andnot_release
1831 #endif
1832 
1833 #ifndef atomic64_fetch_andnot
1834 static inline s64
1835 atomic64_fetch_andnot(s64 i, atomic64_t *v)
1836 {
1837         s64 ret;
1838         __atomic_pre_full_fence();
1839         ret = atomic64_fetch_andnot_relaxed(i, v);
1840         __atomic_post_full_fence();
1841         return ret;
1842 }
1843 #define atomic64_fetch_andnot atomic64_fetch_andnot
1844 #endif
1845 
1846 #endif /* atomic64_fetch_andnot_relaxed */
1847 
1848 #ifndef atomic64_fetch_or_relaxed
1849 #define atomic64_fetch_or_acquire atomic64_fetch_or
1850 #define atomic64_fetch_or_release atomic64_fetch_or
1851 #define atomic64_fetch_or_relaxed atomic64_fetch_or
1852 #else /* atomic64_fetch_or_relaxed */
1853 
1854 #ifndef atomic64_fetch_or_acquire
1855 static inline s64
1856 atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1857 {
1858         s64 ret = atomic64_fetch_or_relaxed(i, v);
1859         __atomic_acquire_fence();
1860         return ret;
1861 }
1862 #define atomic64_fetch_or_acquire atomic64_fetch_or_acquire
1863 #endif
1864 
1865 #ifndef atomic64_fetch_or_release
1866 static inline s64
1867 atomic64_fetch_or_release(s64 i, atomic64_t *v)
1868 {
1869         __atomic_release_fence();
1870         return atomic64_fetch_or_relaxed(i, v);
1871 }
1872 #define atomic64_fetch_or_release atomic64_fetch_or_release
1873 #endif
1874 
1875 #ifndef atomic64_fetch_or
1876 static inline s64
1877 atomic64_fetch_or(s64 i, atomic64_t *v)
1878 {
1879         s64 ret;
1880         __atomic_pre_full_fence();
1881         ret = atomic64_fetch_or_relaxed(i, v);
1882         __atomic_post_full_fence();
1883         return ret;
1884 }
1885 #define atomic64_fetch_or atomic64_fetch_or
1886 #endif
1887 
1888 #endif /* atomic64_fetch_or_relaxed */
1889 
1890 #ifndef atomic64_fetch_xor_relaxed
1891 #define atomic64_fetch_xor_acquire atomic64_fetch_xor
1892 #define atomic64_fetch_xor_release atomic64_fetch_xor
1893 #define atomic64_fetch_xor_relaxed atomic64_fetch_xor
1894 #else /* atomic64_fetch_xor_relaxed */
1895 
1896 #ifndef atomic64_fetch_xor_acquire
1897 static inline s64
1898 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1899 {
1900         s64 ret = atomic64_fetch_xor_relaxed(i, v);
1901         __atomic_acquire_fence();
1902         return ret;
1903 }
1904 #define atomic64_fetch_xor_acquire atomic64_fetch_xor_acquire
1905 #endif
1906 
1907 #ifndef atomic64_fetch_xor_release
1908 static inline s64
1909 atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1910 {
1911         __atomic_release_fence();
1912         return atomic64_fetch_xor_relaxed(i, v);
1913 }
1914 #define atomic64_fetch_xor_release atomic64_fetch_xor_release
1915 #endif
1916 
1917 #ifndef atomic64_fetch_xor
1918 static inline s64
1919 atomic64_fetch_xor(s64 i, atomic64_t *v)
1920 {
1921         s64 ret;
1922         __atomic_pre_full_fence();
1923         ret = atomic64_fetch_xor_relaxed(i, v);
1924         __atomic_post_full_fence();
1925         return ret;
1926 }
1927 #define atomic64_fetch_xor atomic64_fetch_xor
1928 #endif
1929 
1930 #endif /* atomic64_fetch_xor_relaxed */
1931 
1932 #ifndef atomic64_xchg_relaxed
1933 #define atomic64_xchg_acquire atomic64_xchg
1934 #define atomic64_xchg_release atomic64_xchg
1935 #define atomic64_xchg_relaxed atomic64_xchg
1936 #else /* atomic64_xchg_relaxed */
1937 
1938 #ifndef atomic64_xchg_acquire
1939 static inline s64
1940 atomic64_xchg_acquire(atomic64_t *v, s64 i)
1941 {
1942         s64 ret = atomic64_xchg_relaxed(v, i);
1943         __atomic_acquire_fence();
1944         return ret;
1945 }
1946 #define atomic64_xchg_acquire atomic64_xchg_acquire
1947 #endif
1948 
1949 #ifndef atomic64_xchg_release
1950 static inline s64
1951 atomic64_xchg_release(atomic64_t *v, s64 i)
1952 {
1953         __atomic_release_fence();
1954         return atomic64_xchg_relaxed(v, i);
1955 }
1956 #define atomic64_xchg_release atomic64_xchg_release
1957 #endif
1958 
1959 #ifndef atomic64_xchg
1960 static inline s64
1961 atomic64_xchg(atomic64_t *v, s64 i)
1962 {
1963         s64 ret;
1964         __atomic_pre_full_fence();
1965         ret = atomic64_xchg_relaxed(v, i);
1966         __atomic_post_full_fence();
1967         return ret;
1968 }
1969 #define atomic64_xchg atomic64_xchg
1970 #endif
1971 
1972 #endif /* atomic64_xchg_relaxed */
1973 
1974 #ifndef atomic64_cmpxchg_relaxed
1975 #define atomic64_cmpxchg_acquire atomic64_cmpxchg
1976 #define atomic64_cmpxchg_release atomic64_cmpxchg
1977 #define atomic64_cmpxchg_relaxed atomic64_cmpxchg
1978 #else /* atomic64_cmpxchg_relaxed */
1979 
1980 #ifndef atomic64_cmpxchg_acquire
1981 static inline s64
1982 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1983 {
1984         s64 ret = atomic64_cmpxchg_relaxed(v, old, new);
1985         __atomic_acquire_fence();
1986         return ret;
1987 }
1988 #define atomic64_cmpxchg_acquire atomic64_cmpxchg_acquire
1989 #endif
1990 
1991 #ifndef atomic64_cmpxchg_release
1992 static inline s64
1993 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1994 {
1995         __atomic_release_fence();
1996         return atomic64_cmpxchg_relaxed(v, old, new);
1997 }
1998 #define atomic64_cmpxchg_release atomic64_cmpxchg_release
1999 #endif
2000 
2001 #ifndef atomic64_cmpxchg
2002 static inline s64
2003 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
2004 {
2005         s64 ret;
2006         __atomic_pre_full_fence();
2007         ret = atomic64_cmpxchg_relaxed(v, old, new);
2008         __atomic_post_full_fence();
2009         return ret;
2010 }
2011 #define atomic64_cmpxchg atomic64_cmpxchg
2012 #endif
2013 
2014 #endif /* atomic64_cmpxchg_relaxed */
2015 
2016 #ifndef atomic64_try_cmpxchg_relaxed
2017 #ifdef atomic64_try_cmpxchg
2018 #define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg
2019 #define atomic64_try_cmpxchg_release atomic64_try_cmpxchg
2020 #define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg
2021 #endif /* atomic64_try_cmpxchg */
2022 
2023 #ifndef atomic64_try_cmpxchg
2024 static inline bool
2025 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2026 {
2027         s64 r, o = *old;
2028         r = atomic64_cmpxchg(v, o, new);
2029         if (unlikely(r != o))
2030                 *old = r;
2031         return likely(r == o);
2032 }
2033 #define atomic64_try_cmpxchg atomic64_try_cmpxchg
2034 #endif
2035 
2036 #ifndef atomic64_try_cmpxchg_acquire
2037 static inline bool
2038 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2039 {
2040         s64 r, o = *old;
2041         r = atomic64_cmpxchg_acquire(v, o, new);
2042         if (unlikely(r != o))
2043                 *old = r;
2044         return likely(r == o);
2045 }
2046 #define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg_acquire
2047 #endif
2048 
2049 #ifndef atomic64_try_cmpxchg_release
2050 static inline bool
2051 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2052 {
2053         s64 r, o = *old;
2054         r = atomic64_cmpxchg_release(v, o, new);
2055         if (unlikely(r != o))
2056                 *old = r;
2057         return likely(r == o);
2058 }
2059 #define atomic64_try_cmpxchg_release atomic64_try_cmpxchg_release
2060 #endif
2061 
2062 #ifndef atomic64_try_cmpxchg_relaxed
2063 static inline bool
2064 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
2065 {
2066         s64 r, o = *old;
2067         r = atomic64_cmpxchg_relaxed(v, o, new);
2068         if (unlikely(r != o))
2069                 *old = r;
2070         return likely(r == o);
2071 }
2072 #define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg_relaxed
2073 #endif
2074 
2075 #else /* atomic64_try_cmpxchg_relaxed */
2076 
2077 #ifndef atomic64_try_cmpxchg_acquire
2078 static inline bool
2079 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2080 {
2081         bool ret = atomic64_try_cmpxchg_relaxed(v, old, new);
2082         __atomic_acquire_fence();
2083         return ret;
2084 }
2085 #define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg_acquire
2086 #endif
2087 
2088 #ifndef atomic64_try_cmpxchg_release
2089 static inline bool
2090 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2091 {
2092         __atomic_release_fence();
2093         return atomic64_try_cmpxchg_relaxed(v, old, new);
2094 }
2095 #define atomic64_try_cmpxchg_release atomic64_try_cmpxchg_release
2096 #endif
2097 
2098 #ifndef atomic64_try_cmpxchg
2099 static inline bool
2100 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2101 {
2102         bool ret;
2103         __atomic_pre_full_fence();
2104         ret = atomic64_try_cmpxchg_relaxed(v, old, new);
2105         __atomic_post_full_fence();
2106         return ret;
2107 }
2108 #define atomic64_try_cmpxchg atomic64_try_cmpxchg
2109 #endif
2110 
2111 #endif /* atomic64_try_cmpxchg_relaxed */
2112 
2113 #ifndef atomic64_sub_and_test
2114 /**
2115  * atomic64_sub_and_test - subtract value from variable and test result
2116  * @i: integer value to subtract
2117  * @v: pointer of type atomic64_t
2118  *
2119  * Atomically subtracts @i from @v and returns
2120  * true if the result is zero, or false for all
2121  * other cases.
2122  */
2123 static inline bool
2124 atomic64_sub_and_test(s64 i, atomic64_t *v)
2125 {
2126         return atomic64_sub_return(i, v) == 0;
2127 }
2128 #define atomic64_sub_and_test atomic64_sub_and_test
2129 #endif
2130 
2131 #ifndef atomic64_dec_and_test
2132 /**
2133  * atomic64_dec_and_test - decrement and test
2134  * @v: pointer of type atomic64_t
2135  *
2136  * Atomically decrements @v by 1 and
2137  * returns true if the result is 0, or false for all other
2138  * cases.
2139  */
2140 static inline bool
2141 atomic64_dec_and_test(atomic64_t *v)
2142 {
2143         return atomic64_dec_return(v) == 0;
2144 }
2145 #define atomic64_dec_and_test atomic64_dec_and_test
2146 #endif
2147 
2148 #ifndef atomic64_inc_and_test
2149 /**
2150  * atomic64_inc_and_test - increment and test
2151  * @v: pointer of type atomic64_t
2152  *
2153  * Atomically increments @v by 1
2154  * and returns true if the result is zero, or false for all
2155  * other cases.
2156  */
2157 static inline bool
2158 atomic64_inc_and_test(atomic64_t *v)
2159 {
2160         return atomic64_inc_return(v) == 0;
2161 }
2162 #define atomic64_inc_and_test atomic64_inc_and_test
2163 #endif
2164 
2165 #ifndef atomic64_add_negative
2166 /**
2167  * atomic64_add_negative - add and test if negative
2168  * @i: integer value to add
2169  * @v: pointer of type atomic64_t
2170  *
2171  * Atomically adds @i to @v and returns true
2172  * if the result is negative, or false when
2173  * result is greater than or equal to zero.
2174  */
2175 static inline bool
2176 atomic64_add_negative(s64 i, atomic64_t *v)
2177 {
2178         return atomic64_add_return(i, v) < 0;
2179 }
2180 #define atomic64_add_negative atomic64_add_negative
2181 #endif
2182 
2183 #ifndef atomic64_fetch_add_unless
2184 /**
2185  * atomic64_fetch_add_unless - add unless the number is already a given value
2186  * @v: pointer of type atomic64_t
2187  * @a: the amount to add to v...
2188  * @u: ...unless v is equal to u.
2189  *
2190  * Atomically adds @a to @v, so long as @v was not already @u.
2191  * Returns original value of @v
2192  */
2193 static inline s64
2194 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
2195 {
2196         s64 c = atomic64_read(v);
2197 
2198         do {
2199                 if (unlikely(c == u))
2200                         break;
2201         } while (!atomic64_try_cmpxchg(v, &c, c + a));
2202 
2203         return c;
2204 }
2205 #define atomic64_fetch_add_unless atomic64_fetch_add_unless
2206 #endif
2207 
2208 #ifndef atomic64_add_unless
2209 /**
2210  * atomic64_add_unless - add unless the number is already a given value
2211  * @v: pointer of type atomic64_t
2212  * @a: the amount to add to v...
2213  * @u: ...unless v is equal to u.
2214  *
2215  * Atomically adds @a to @v, if @v was not already @u.
2216  * Returns true if the addition was done.
2217  */
2218 static inline bool
2219 atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
2220 {
2221         return atomic64_fetch_add_unless(v, a, u) != u;
2222 }
2223 #define atomic64_add_unless atomic64_add_unless
2224 #endif
2225 
2226 #ifndef atomic64_inc_not_zero
2227 /**
2228  * atomic64_inc_not_zero - increment unless the number is zero
2229  * @v: pointer of type atomic64_t
2230  *
2231  * Atomically increments @v by 1, if @v is non-zero.
2232  * Returns true if the increment was done.
2233  */
2234 static inline bool
2235 atomic64_inc_not_zero(atomic64_t *v)
2236 {
2237         return atomic64_add_unless(v, 1, 0);
2238 }
2239 #define atomic64_inc_not_zero atomic64_inc_not_zero
2240 #endif
2241 
2242 #ifndef atomic64_inc_unless_negative
2243 static inline bool
2244 atomic64_inc_unless_negative(atomic64_t *v)
2245 {
2246         s64 c = atomic64_read(v);
2247 
2248         do {
2249                 if (unlikely(c < 0))
2250                         return false;
2251         } while (!atomic64_try_cmpxchg(v, &c, c + 1));
2252 
2253         return true;
2254 }
2255 #define atomic64_inc_unless_negative atomic64_inc_unless_negative
2256 #endif
2257 
2258 #ifndef atomic64_dec_unless_positive
2259 static inline bool
2260 atomic64_dec_unless_positive(atomic64_t *v)
2261 {
2262         s64 c = atomic64_read(v);
2263 
2264         do {
2265                 if (unlikely(c > 0))
2266                         return false;
2267         } while (!atomic64_try_cmpxchg(v, &c, c - 1));
2268 
2269         return true;
2270 }
2271 #define atomic64_dec_unless_positive atomic64_dec_unless_positive
2272 #endif
2273 
2274 #ifndef atomic64_dec_if_positive
2275 static inline s64
2276 atomic64_dec_if_positive(atomic64_t *v)
2277 {
2278         s64 dec, c = atomic64_read(v);
2279 
2280         do {
2281                 dec = c - 1;
2282                 if (unlikely(dec < 0))
2283                         break;
2284         } while (!atomic64_try_cmpxchg(v, &c, dec));
2285 
2286         return dec;
2287 }
2288 #define atomic64_dec_if_positive atomic64_dec_if_positive
2289 #endif
2290 
2291 #define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
2292 #define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
2293 
2294 #endif /* _LINUX_ATOMIC_FALLBACK_H */
2295 // 25de4a2804d70f57e994fe3b419148658bb5378a
2296 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp