~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/include/linux/rcupdate.h

Version: ~ [ linux-5.15-rc5 ] ~ [ linux-5.14.11 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.72 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.152 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.210 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.250 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.286 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.288 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* SPDX-License-Identifier: GPL-2.0+ */
  2 /*
  3  * Read-Copy Update mechanism for mutual exclusion
  4  *
  5  * Copyright IBM Corporation, 2001
  6  *
  7  * Author: Dipankar Sarma <dipankar@in.ibm.com>
  8  *
  9  * Based on the original work by Paul McKenney <paulmck@vnet.ibm.com>
 10  * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
 11  * Papers:
 12  * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
 13  * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001)
 14  *
 15  * For detailed explanation of Read-Copy Update mechanism see -
 16  *              http://lse.sourceforge.net/locking/rcupdate.html
 17  *
 18  */
 19 
 20 #ifndef __LINUX_RCUPDATE_H
 21 #define __LINUX_RCUPDATE_H
 22 
 23 #include <linux/types.h>
 24 #include <linux/compiler.h>
 25 #include <linux/atomic.h>
 26 #include <linux/irqflags.h>
 27 #include <linux/preempt.h>
 28 #include <linux/bottom_half.h>
 29 #include <linux/lockdep.h>
 30 #include <asm/processor.h>
 31 #include <linux/cpumask.h>
 32 
 33 #define ULONG_CMP_GE(a, b)      (ULONG_MAX / 2 >= (a) - (b))
 34 #define ULONG_CMP_LT(a, b)      (ULONG_MAX / 2 < (a) - (b))
 35 #define ulong2long(a)           (*(long *)(&(a)))
 36 #define USHORT_CMP_GE(a, b)     (USHRT_MAX / 2 >= (unsigned short)((a) - (b)))
 37 #define USHORT_CMP_LT(a, b)     (USHRT_MAX / 2 < (unsigned short)((a) - (b)))
 38 
 39 /* Exported common interfaces */
 40 void call_rcu(struct rcu_head *head, rcu_callback_t func);
 41 void rcu_barrier_tasks(void);
 42 void rcu_barrier_tasks_rude(void);
 43 void synchronize_rcu(void);
 44 
 45 #ifdef CONFIG_PREEMPT_RCU
 46 
 47 void __rcu_read_lock(void);
 48 void __rcu_read_unlock(void);
 49 
 50 /*
 51  * Defined as a macro as it is a very low level header included from
 52  * areas that don't even know about current.  This gives the rcu_read_lock()
 53  * nesting depth, but makes sense only if CONFIG_PREEMPT_RCU -- in other
 54  * types of kernel builds, the rcu_read_lock() nesting depth is unknowable.
 55  */
 56 #define rcu_preempt_depth() READ_ONCE(current->rcu_read_lock_nesting)
 57 
 58 #else /* #ifdef CONFIG_PREEMPT_RCU */
 59 
 60 #ifdef CONFIG_TINY_RCU
 61 #define rcu_read_unlock_strict() do { } while (0)
 62 #else
 63 void rcu_read_unlock_strict(void);
 64 #endif
 65 
 66 static inline void __rcu_read_lock(void)
 67 {
 68         preempt_disable();
 69 }
 70 
 71 static inline void __rcu_read_unlock(void)
 72 {
 73         preempt_enable();
 74         rcu_read_unlock_strict();
 75 }
 76 
 77 static inline int rcu_preempt_depth(void)
 78 {
 79         return 0;
 80 }
 81 
 82 #endif /* #else #ifdef CONFIG_PREEMPT_RCU */
 83 
 84 /* Internal to kernel */
 85 void rcu_init(void);
 86 extern int rcu_scheduler_active __read_mostly;
 87 void rcu_sched_clock_irq(int user);
 88 void rcu_report_dead(unsigned int cpu);
 89 void rcutree_migrate_callbacks(int cpu);
 90 
 91 #ifdef CONFIG_TASKS_RCU_GENERIC
 92 void rcu_init_tasks_generic(void);
 93 #else
 94 static inline void rcu_init_tasks_generic(void) { }
 95 #endif
 96 
 97 #ifdef CONFIG_RCU_STALL_COMMON
 98 void rcu_sysrq_start(void);
 99 void rcu_sysrq_end(void);
100 #else /* #ifdef CONFIG_RCU_STALL_COMMON */
101 static inline void rcu_sysrq_start(void) { }
102 static inline void rcu_sysrq_end(void) { }
103 #endif /* #else #ifdef CONFIG_RCU_STALL_COMMON */
104 
105 #ifdef CONFIG_NO_HZ_FULL
106 void rcu_user_enter(void);
107 void rcu_user_exit(void);
108 #else
109 static inline void rcu_user_enter(void) { }
110 static inline void rcu_user_exit(void) { }
111 #endif /* CONFIG_NO_HZ_FULL */
112 
113 #ifdef CONFIG_RCU_NOCB_CPU
114 void rcu_init_nohz(void);
115 int rcu_nocb_cpu_offload(int cpu);
116 int rcu_nocb_cpu_deoffload(int cpu);
117 void rcu_nocb_flush_deferred_wakeup(void);
118 #else /* #ifdef CONFIG_RCU_NOCB_CPU */
119 static inline void rcu_init_nohz(void) { }
120 static inline int rcu_nocb_cpu_offload(int cpu) { return -EINVAL; }
121 static inline int rcu_nocb_cpu_deoffload(int cpu) { return 0; }
122 static inline void rcu_nocb_flush_deferred_wakeup(void) { }
123 #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
124 
125 /**
126  * RCU_NONIDLE - Indicate idle-loop code that needs RCU readers
127  * @a: Code that RCU needs to pay attention to.
128  *
129  * RCU read-side critical sections are forbidden in the inner idle loop,
130  * that is, between the rcu_idle_enter() and the rcu_idle_exit() -- RCU
131  * will happily ignore any such read-side critical sections.  However,
132  * things like powertop need tracepoints in the inner idle loop.
133  *
134  * This macro provides the way out:  RCU_NONIDLE(do_something_with_RCU())
135  * will tell RCU that it needs to pay attention, invoke its argument
136  * (in this example, calling the do_something_with_RCU() function),
137  * and then tell RCU to go back to ignoring this CPU.  It is permissible
138  * to nest RCU_NONIDLE() wrappers, but not indefinitely (but the limit is
139  * on the order of a million or so, even on 32-bit systems).  It is
140  * not legal to block within RCU_NONIDLE(), nor is it permissible to
141  * transfer control either into or out of RCU_NONIDLE()'s statement.
142  */
143 #define RCU_NONIDLE(a) \
144         do { \
145                 rcu_irq_enter_irqson(); \
146                 do { a; } while (0); \
147                 rcu_irq_exit_irqson(); \
148         } while (0)
149 
150 /*
151  * Note a quasi-voluntary context switch for RCU-tasks's benefit.
152  * This is a macro rather than an inline function to avoid #include hell.
153  */
154 #ifdef CONFIG_TASKS_RCU_GENERIC
155 
156 # ifdef CONFIG_TASKS_RCU
157 # define rcu_tasks_classic_qs(t, preempt)                               \
158         do {                                                            \
159                 if (!(preempt) && READ_ONCE((t)->rcu_tasks_holdout))    \
160                         WRITE_ONCE((t)->rcu_tasks_holdout, false);      \
161         } while (0)
162 void call_rcu_tasks(struct rcu_head *head, rcu_callback_t func);
163 void synchronize_rcu_tasks(void);
164 # else
165 # define rcu_tasks_classic_qs(t, preempt) do { } while (0)
166 # define call_rcu_tasks call_rcu
167 # define synchronize_rcu_tasks synchronize_rcu
168 # endif
169 
170 # ifdef CONFIG_TASKS_TRACE_RCU
171 # define rcu_tasks_trace_qs(t)                                          \
172         do {                                                            \
173                 if (!likely(READ_ONCE((t)->trc_reader_checked)) &&      \
174                     !unlikely(READ_ONCE((t)->trc_reader_nesting))) {    \
175                         smp_store_release(&(t)->trc_reader_checked, true); \
176                         smp_mb(); /* Readers partitioned by store. */   \
177                 }                                                       \
178         } while (0)
179 # else
180 # define rcu_tasks_trace_qs(t) do { } while (0)
181 # endif
182 
183 #define rcu_tasks_qs(t, preempt)                                        \
184 do {                                                                    \
185         rcu_tasks_classic_qs((t), (preempt));                           \
186         rcu_tasks_trace_qs((t));                                        \
187 } while (0)
188 
189 # ifdef CONFIG_TASKS_RUDE_RCU
190 void call_rcu_tasks_rude(struct rcu_head *head, rcu_callback_t func);
191 void synchronize_rcu_tasks_rude(void);
192 # endif
193 
194 #define rcu_note_voluntary_context_switch(t) rcu_tasks_qs(t, false)
195 void exit_tasks_rcu_start(void);
196 void exit_tasks_rcu_finish(void);
197 #else /* #ifdef CONFIG_TASKS_RCU_GENERIC */
198 #define rcu_tasks_qs(t, preempt) do { } while (0)
199 #define rcu_note_voluntary_context_switch(t) do { } while (0)
200 #define call_rcu_tasks call_rcu
201 #define synchronize_rcu_tasks synchronize_rcu
202 static inline void exit_tasks_rcu_start(void) { }
203 static inline void exit_tasks_rcu_finish(void) { }
204 #endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */
205 
206 /**
207  * cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU
208  *
209  * This macro resembles cond_resched(), except that it is defined to
210  * report potential quiescent states to RCU-tasks even if the cond_resched()
211  * machinery were to be shut off, as some advocate for PREEMPTION kernels.
212  */
213 #define cond_resched_tasks_rcu_qs() \
214 do { \
215         rcu_tasks_qs(current, false); \
216         cond_resched(); \
217 } while (0)
218 
219 /*
220  * Infrastructure to implement the synchronize_() primitives in
221  * TREE_RCU and rcu_barrier_() primitives in TINY_RCU.
222  */
223 
224 #if defined(CONFIG_TREE_RCU)
225 #include <linux/rcutree.h>
226 #elif defined(CONFIG_TINY_RCU)
227 #include <linux/rcutiny.h>
228 #else
229 #error "Unknown RCU implementation specified to kernel configuration"
230 #endif
231 
232 /*
233  * The init_rcu_head_on_stack() and destroy_rcu_head_on_stack() calls
234  * are needed for dynamic initialization and destruction of rcu_head
235  * on the stack, and init_rcu_head()/destroy_rcu_head() are needed for
236  * dynamic initialization and destruction of statically allocated rcu_head
237  * structures.  However, rcu_head structures allocated dynamically in the
238  * heap don't need any initialization.
239  */
240 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
241 void init_rcu_head(struct rcu_head *head);
242 void destroy_rcu_head(struct rcu_head *head);
243 void init_rcu_head_on_stack(struct rcu_head *head);
244 void destroy_rcu_head_on_stack(struct rcu_head *head);
245 #else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
246 static inline void init_rcu_head(struct rcu_head *head) { }
247 static inline void destroy_rcu_head(struct rcu_head *head) { }
248 static inline void init_rcu_head_on_stack(struct rcu_head *head) { }
249 static inline void destroy_rcu_head_on_stack(struct rcu_head *head) { }
250 #endif  /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
251 
252 #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU)
253 bool rcu_lockdep_current_cpu_online(void);
254 #else /* #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */
255 static inline bool rcu_lockdep_current_cpu_online(void) { return true; }
256 #endif /* #else #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */
257 
258 extern struct lockdep_map rcu_lock_map;
259 extern struct lockdep_map rcu_bh_lock_map;
260 extern struct lockdep_map rcu_sched_lock_map;
261 extern struct lockdep_map rcu_callback_map;
262 
263 #ifdef CONFIG_DEBUG_LOCK_ALLOC
264 
265 static inline void rcu_lock_acquire(struct lockdep_map *map)
266 {
267         lock_acquire(map, 0, 0, 2, 0, NULL, _THIS_IP_);
268 }
269 
270 static inline void rcu_lock_release(struct lockdep_map *map)
271 {
272         lock_release(map, _THIS_IP_);
273 }
274 
275 int debug_lockdep_rcu_enabled(void);
276 int rcu_read_lock_held(void);
277 int rcu_read_lock_bh_held(void);
278 int rcu_read_lock_sched_held(void);
279 int rcu_read_lock_any_held(void);
280 
281 #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
282 
283 # define rcu_lock_acquire(a)            do { } while (0)
284 # define rcu_lock_release(a)            do { } while (0)
285 
286 static inline int rcu_read_lock_held(void)
287 {
288         return 1;
289 }
290 
291 static inline int rcu_read_lock_bh_held(void)
292 {
293         return 1;
294 }
295 
296 static inline int rcu_read_lock_sched_held(void)
297 {
298         return !preemptible();
299 }
300 
301 static inline int rcu_read_lock_any_held(void)
302 {
303         return !preemptible();
304 }
305 
306 #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
307 
308 #ifdef CONFIG_PROVE_RCU
309 
310 /**
311  * RCU_LOCKDEP_WARN - emit lockdep splat if specified condition is met
312  * @c: condition to check
313  * @s: informative message
314  */
315 #define RCU_LOCKDEP_WARN(c, s)                                          \
316         do {                                                            \
317                 static bool __section(".data.unlikely") __warned;       \
318                 if ((c) && debug_lockdep_rcu_enabled() && !__warned) {  \
319                         __warned = true;                                \
320                         lockdep_rcu_suspicious(__FILE__, __LINE__, s);  \
321                 }                                                       \
322         } while (0)
323 
324 #if defined(CONFIG_PROVE_RCU) && !defined(CONFIG_PREEMPT_RCU)
325 static inline void rcu_preempt_sleep_check(void)
326 {
327         RCU_LOCKDEP_WARN(lock_is_held(&rcu_lock_map),
328                          "Illegal context switch in RCU read-side critical section");
329 }
330 #else /* #ifdef CONFIG_PROVE_RCU */
331 static inline void rcu_preempt_sleep_check(void) { }
332 #endif /* #else #ifdef CONFIG_PROVE_RCU */
333 
334 #define rcu_sleep_check()                                               \
335         do {                                                            \
336                 rcu_preempt_sleep_check();                              \
337                 if (!IS_ENABLED(CONFIG_PREEMPT_RT))                     \
338                     RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map),    \
339                                  "Illegal context switch in RCU-bh read-side critical section"); \
340                 RCU_LOCKDEP_WARN(lock_is_held(&rcu_sched_lock_map),     \
341                                  "Illegal context switch in RCU-sched read-side critical section"); \
342         } while (0)
343 
344 #else /* #ifdef CONFIG_PROVE_RCU */
345 
346 #define RCU_LOCKDEP_WARN(c, s) do { } while (0 && (c))
347 #define rcu_sleep_check() do { } while (0)
348 
349 #endif /* #else #ifdef CONFIG_PROVE_RCU */
350 
351 /*
352  * Helper functions for rcu_dereference_check(), rcu_dereference_protected()
353  * and rcu_assign_pointer().  Some of these could be folded into their
354  * callers, but they are left separate in order to ease introduction of
355  * multiple pointers markings to match different RCU implementations
356  * (e.g., __srcu), should this make sense in the future.
357  */
358 
359 #ifdef __CHECKER__
360 #define rcu_check_sparse(p, space) \
361         ((void)(((typeof(*p) space *)p) == p))
362 #else /* #ifdef __CHECKER__ */
363 #define rcu_check_sparse(p, space)
364 #endif /* #else #ifdef __CHECKER__ */
365 
366 /**
367  * unrcu_pointer - mark a pointer as not being RCU protected
368  * @p: pointer needing to lose its __rcu property
369  *
370  * Converts @p from an __rcu pointer to a __kernel pointer.
371  * This allows an __rcu pointer to be used with xchg() and friends.
372  */
373 #define unrcu_pointer(p)                                                \
374 ({                                                                      \
375         typeof(*p) *_________p1 = (typeof(*p) *__force)(p);             \
376         rcu_check_sparse(p, __rcu);                                     \
377         ((typeof(*p) __force __kernel *)(_________p1));                 \
378 })
379 
380 #define __rcu_access_pointer(p, space) \
381 ({ \
382         typeof(*p) *_________p1 = (typeof(*p) *__force)READ_ONCE(p); \
383         rcu_check_sparse(p, space); \
384         ((typeof(*p) __force __kernel *)(_________p1)); \
385 })
386 #define __rcu_dereference_check(p, c, space) \
387 ({ \
388         /* Dependency order vs. p above. */ \
389         typeof(*p) *________p1 = (typeof(*p) *__force)READ_ONCE(p); \
390         RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_check() usage"); \
391         rcu_check_sparse(p, space); \
392         ((typeof(*p) __force __kernel *)(________p1)); \
393 })
394 #define __rcu_dereference_protected(p, c, space) \
395 ({ \
396         RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_protected() usage"); \
397         rcu_check_sparse(p, space); \
398         ((typeof(*p) __force __kernel *)(p)); \
399 })
400 #define rcu_dereference_raw(p) \
401 ({ \
402         /* Dependency order vs. p above. */ \
403         typeof(p) ________p1 = READ_ONCE(p); \
404         ((typeof(*p) __force __kernel *)(________p1)); \
405 })
406 
407 /**
408  * RCU_INITIALIZER() - statically initialize an RCU-protected global variable
409  * @v: The value to statically initialize with.
410  */
411 #define RCU_INITIALIZER(v) (typeof(*(v)) __force __rcu *)(v)
412 
413 /**
414  * rcu_assign_pointer() - assign to RCU-protected pointer
415  * @p: pointer to assign to
416  * @v: value to assign (publish)
417  *
418  * Assigns the specified value to the specified RCU-protected
419  * pointer, ensuring that any concurrent RCU readers will see
420  * any prior initialization.
421  *
422  * Inserts memory barriers on architectures that require them
423  * (which is most of them), and also prevents the compiler from
424  * reordering the code that initializes the structure after the pointer
425  * assignment.  More importantly, this call documents which pointers
426  * will be dereferenced by RCU read-side code.
427  *
428  * In some special cases, you may use RCU_INIT_POINTER() instead
429  * of rcu_assign_pointer().  RCU_INIT_POINTER() is a bit faster due
430  * to the fact that it does not constrain either the CPU or the compiler.
431  * That said, using RCU_INIT_POINTER() when you should have used
432  * rcu_assign_pointer() is a very bad thing that results in
433  * impossible-to-diagnose memory corruption.  So please be careful.
434  * See the RCU_INIT_POINTER() comment header for details.
435  *
436  * Note that rcu_assign_pointer() evaluates each of its arguments only
437  * once, appearances notwithstanding.  One of the "extra" evaluations
438  * is in typeof() and the other visible only to sparse (__CHECKER__),
439  * neither of which actually execute the argument.  As with most cpp
440  * macros, this execute-arguments-only-once property is important, so
441  * please be careful when making changes to rcu_assign_pointer() and the
442  * other macros that it invokes.
443  */
444 #define rcu_assign_pointer(p, v)                                              \
445 do {                                                                          \
446         uintptr_t _r_a_p__v = (uintptr_t)(v);                                 \
447         rcu_check_sparse(p, __rcu);                                           \
448                                                                               \
449         if (__builtin_constant_p(v) && (_r_a_p__v) == (uintptr_t)NULL)        \
450                 WRITE_ONCE((p), (typeof(p))(_r_a_p__v));                      \
451         else                                                                  \
452                 smp_store_release(&p, RCU_INITIALIZER((typeof(p))_r_a_p__v)); \
453 } while (0)
454 
455 /**
456  * rcu_replace_pointer() - replace an RCU pointer, returning its old value
457  * @rcu_ptr: RCU pointer, whose old value is returned
458  * @ptr: regular pointer
459  * @c: the lockdep conditions under which the dereference will take place
460  *
461  * Perform a replacement, where @rcu_ptr is an RCU-annotated
462  * pointer and @c is the lockdep argument that is passed to the
463  * rcu_dereference_protected() call used to read that pointer.  The old
464  * value of @rcu_ptr is returned, and @rcu_ptr is set to @ptr.
465  */
466 #define rcu_replace_pointer(rcu_ptr, ptr, c)                            \
467 ({                                                                      \
468         typeof(ptr) __tmp = rcu_dereference_protected((rcu_ptr), (c));  \
469         rcu_assign_pointer((rcu_ptr), (ptr));                           \
470         __tmp;                                                          \
471 })
472 
473 /**
474  * rcu_access_pointer() - fetch RCU pointer with no dereferencing
475  * @p: The pointer to read
476  *
477  * Return the value of the specified RCU-protected pointer, but omit the
478  * lockdep checks for being in an RCU read-side critical section.  This is
479  * useful when the value of this pointer is accessed, but the pointer is
480  * not dereferenced, for example, when testing an RCU-protected pointer
481  * against NULL.  Although rcu_access_pointer() may also be used in cases
482  * where update-side locks prevent the value of the pointer from changing,
483  * you should instead use rcu_dereference_protected() for this use case.
484  *
485  * It is also permissible to use rcu_access_pointer() when read-side
486  * access to the pointer was removed at least one grace period ago, as
487  * is the case in the context of the RCU callback that is freeing up
488  * the data, or after a synchronize_rcu() returns.  This can be useful
489  * when tearing down multi-linked structures after a grace period
490  * has elapsed.
491  */
492 #define rcu_access_pointer(p) __rcu_access_pointer((p), __rcu)
493 
494 /**
495  * rcu_dereference_check() - rcu_dereference with debug checking
496  * @p: The pointer to read, prior to dereferencing
497  * @c: The conditions under which the dereference will take place
498  *
499  * Do an rcu_dereference(), but check that the conditions under which the
500  * dereference will take place are correct.  Typically the conditions
501  * indicate the various locking conditions that should be held at that
502  * point.  The check should return true if the conditions are satisfied.
503  * An implicit check for being in an RCU read-side critical section
504  * (rcu_read_lock()) is included.
505  *
506  * For example:
507  *
508  *      bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock));
509  *
510  * could be used to indicate to lockdep that foo->bar may only be dereferenced
511  * if either rcu_read_lock() is held, or that the lock required to replace
512  * the bar struct at foo->bar is held.
513  *
514  * Note that the list of conditions may also include indications of when a lock
515  * need not be held, for example during initialisation or destruction of the
516  * target struct:
517  *
518  *      bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock) ||
519  *                                            atomic_read(&foo->usage) == 0);
520  *
521  * Inserts memory barriers on architectures that require them
522  * (currently only the Alpha), prevents the compiler from refetching
523  * (and from merging fetches), and, more importantly, documents exactly
524  * which pointers are protected by RCU and checks that the pointer is
525  * annotated as __rcu.
526  */
527 #define rcu_dereference_check(p, c) \
528         __rcu_dereference_check((p), (c) || rcu_read_lock_held(), __rcu)
529 
530 /**
531  * rcu_dereference_bh_check() - rcu_dereference_bh with debug checking
532  * @p: The pointer to read, prior to dereferencing
533  * @c: The conditions under which the dereference will take place
534  *
535  * This is the RCU-bh counterpart to rcu_dereference_check().  However,
536  * please note that starting in v5.0 kernels, vanilla RCU grace periods
537  * wait for local_bh_disable() regions of code in addition to regions of
538  * code demarked by rcu_read_lock() and rcu_read_unlock().  This means
539  * that synchronize_rcu(), call_rcu, and friends all take not only
540  * rcu_read_lock() but also rcu_read_lock_bh() into account.
541  */
542 #define rcu_dereference_bh_check(p, c) \
543         __rcu_dereference_check((p), (c) || rcu_read_lock_bh_held(), __rcu)
544 
545 /**
546  * rcu_dereference_sched_check() - rcu_dereference_sched with debug checking
547  * @p: The pointer to read, prior to dereferencing
548  * @c: The conditions under which the dereference will take place
549  *
550  * This is the RCU-sched counterpart to rcu_dereference_check().
551  * However, please note that starting in v5.0 kernels, vanilla RCU grace
552  * periods wait for preempt_disable() regions of code in addition to
553  * regions of code demarked by rcu_read_lock() and rcu_read_unlock().
554  * This means that synchronize_rcu(), call_rcu, and friends all take not
555  * only rcu_read_lock() but also rcu_read_lock_sched() into account.
556  */
557 #define rcu_dereference_sched_check(p, c) \
558         __rcu_dereference_check((p), (c) || rcu_read_lock_sched_held(), \
559                                 __rcu)
560 
561 /*
562  * The tracing infrastructure traces RCU (we want that), but unfortunately
563  * some of the RCU checks causes tracing to lock up the system.
564  *
565  * The no-tracing version of rcu_dereference_raw() must not call
566  * rcu_read_lock_held().
567  */
568 #define rcu_dereference_raw_check(p) __rcu_dereference_check((p), 1, __rcu)
569 
570 /**
571  * rcu_dereference_protected() - fetch RCU pointer when updates prevented
572  * @p: The pointer to read, prior to dereferencing
573  * @c: The conditions under which the dereference will take place
574  *
575  * Return the value of the specified RCU-protected pointer, but omit
576  * the READ_ONCE().  This is useful in cases where update-side locks
577  * prevent the value of the pointer from changing.  Please note that this
578  * primitive does *not* prevent the compiler from repeating this reference
579  * or combining it with other references, so it should not be used without
580  * protection of appropriate locks.
581  *
582  * This function is only for update-side use.  Using this function
583  * when protected only by rcu_read_lock() will result in infrequent
584  * but very ugly failures.
585  */
586 #define rcu_dereference_protected(p, c) \
587         __rcu_dereference_protected((p), (c), __rcu)
588 
589 
590 /**
591  * rcu_dereference() - fetch RCU-protected pointer for dereferencing
592  * @p: The pointer to read, prior to dereferencing
593  *
594  * This is a simple wrapper around rcu_dereference_check().
595  */
596 #define rcu_dereference(p) rcu_dereference_check(p, 0)
597 
598 /**
599  * rcu_dereference_bh() - fetch an RCU-bh-protected pointer for dereferencing
600  * @p: The pointer to read, prior to dereferencing
601  *
602  * Makes rcu_dereference_check() do the dirty work.
603  */
604 #define rcu_dereference_bh(p) rcu_dereference_bh_check(p, 0)
605 
606 /**
607  * rcu_dereference_sched() - fetch RCU-sched-protected pointer for dereferencing
608  * @p: The pointer to read, prior to dereferencing
609  *
610  * Makes rcu_dereference_check() do the dirty work.
611  */
612 #define rcu_dereference_sched(p) rcu_dereference_sched_check(p, 0)
613 
614 /**
615  * rcu_pointer_handoff() - Hand off a pointer from RCU to other mechanism
616  * @p: The pointer to hand off
617  *
618  * This is simply an identity function, but it documents where a pointer
619  * is handed off from RCU to some other synchronization mechanism, for
620  * example, reference counting or locking.  In C11, it would map to
621  * kill_dependency().  It could be used as follows::
622  *
623  *      rcu_read_lock();
624  *      p = rcu_dereference(gp);
625  *      long_lived = is_long_lived(p);
626  *      if (long_lived) {
627  *              if (!atomic_inc_not_zero(p->refcnt))
628  *                      long_lived = false;
629  *              else
630  *                      p = rcu_pointer_handoff(p);
631  *      }
632  *      rcu_read_unlock();
633  */
634 #define rcu_pointer_handoff(p) (p)
635 
636 /**
637  * rcu_read_lock() - mark the beginning of an RCU read-side critical section
638  *
639  * When synchronize_rcu() is invoked on one CPU while other CPUs
640  * are within RCU read-side critical sections, then the
641  * synchronize_rcu() is guaranteed to block until after all the other
642  * CPUs exit their critical sections.  Similarly, if call_rcu() is invoked
643  * on one CPU while other CPUs are within RCU read-side critical
644  * sections, invocation of the corresponding RCU callback is deferred
645  * until after the all the other CPUs exit their critical sections.
646  *
647  * In v5.0 and later kernels, synchronize_rcu() and call_rcu() also
648  * wait for regions of code with preemption disabled, including regions of
649  * code with interrupts or softirqs disabled.  In pre-v5.0 kernels, which
650  * define synchronize_sched(), only code enclosed within rcu_read_lock()
651  * and rcu_read_unlock() are guaranteed to be waited for.
652  *
653  * Note, however, that RCU callbacks are permitted to run concurrently
654  * with new RCU read-side critical sections.  One way that this can happen
655  * is via the following sequence of events: (1) CPU 0 enters an RCU
656  * read-side critical section, (2) CPU 1 invokes call_rcu() to register
657  * an RCU callback, (3) CPU 0 exits the RCU read-side critical section,
658  * (4) CPU 2 enters a RCU read-side critical section, (5) the RCU
659  * callback is invoked.  This is legal, because the RCU read-side critical
660  * section that was running concurrently with the call_rcu() (and which
661  * therefore might be referencing something that the corresponding RCU
662  * callback would free up) has completed before the corresponding
663  * RCU callback is invoked.
664  *
665  * RCU read-side critical sections may be nested.  Any deferred actions
666  * will be deferred until the outermost RCU read-side critical section
667  * completes.
668  *
669  * You can avoid reading and understanding the next paragraph by
670  * following this rule: don't put anything in an rcu_read_lock() RCU
671  * read-side critical section that would block in a !PREEMPTION kernel.
672  * But if you want the full story, read on!
673  *
674  * In non-preemptible RCU implementations (pure TREE_RCU and TINY_RCU),
675  * it is illegal to block while in an RCU read-side critical section.
676  * In preemptible RCU implementations (PREEMPT_RCU) in CONFIG_PREEMPTION
677  * kernel builds, RCU read-side critical sections may be preempted,
678  * but explicit blocking is illegal.  Finally, in preemptible RCU
679  * implementations in real-time (with -rt patchset) kernel builds, RCU
680  * read-side critical sections may be preempted and they may also block, but
681  * only when acquiring spinlocks that are subject to priority inheritance.
682  */
683 static __always_inline void rcu_read_lock(void)
684 {
685         __rcu_read_lock();
686         __acquire(RCU);
687         rcu_lock_acquire(&rcu_lock_map);
688         RCU_LOCKDEP_WARN(!rcu_is_watching(),
689                          "rcu_read_lock() used illegally while idle");
690 }
691 
692 /*
693  * So where is rcu_write_lock()?  It does not exist, as there is no
694  * way for writers to lock out RCU readers.  This is a feature, not
695  * a bug -- this property is what provides RCU's performance benefits.
696  * Of course, writers must coordinate with each other.  The normal
697  * spinlock primitives work well for this, but any other technique may be
698  * used as well.  RCU does not care how the writers keep out of each
699  * others' way, as long as they do so.
700  */
701 
702 /**
703  * rcu_read_unlock() - marks the end of an RCU read-side critical section.
704  *
705  * In almost all situations, rcu_read_unlock() is immune from deadlock.
706  * In recent kernels that have consolidated synchronize_sched() and
707  * synchronize_rcu_bh() into synchronize_rcu(), this deadlock immunity
708  * also extends to the scheduler's runqueue and priority-inheritance
709  * spinlocks, courtesy of the quiescent-state deferral that is carried
710  * out when rcu_read_unlock() is invoked with interrupts disabled.
711  *
712  * See rcu_read_lock() for more information.
713  */
714 static inline void rcu_read_unlock(void)
715 {
716         RCU_LOCKDEP_WARN(!rcu_is_watching(),
717                          "rcu_read_unlock() used illegally while idle");
718         __release(RCU);
719         __rcu_read_unlock();
720         rcu_lock_release(&rcu_lock_map); /* Keep acq info for rls diags. */
721 }
722 
723 /**
724  * rcu_read_lock_bh() - mark the beginning of an RCU-bh critical section
725  *
726  * This is equivalent to rcu_read_lock(), but also disables softirqs.
727  * Note that anything else that disables softirqs can also serve as an RCU
728  * read-side critical section.  However, please note that this equivalence
729  * applies only to v5.0 and later.  Before v5.0, rcu_read_lock() and
730  * rcu_read_lock_bh() were unrelated.
731  *
732  * Note that rcu_read_lock_bh() and the matching rcu_read_unlock_bh()
733  * must occur in the same context, for example, it is illegal to invoke
734  * rcu_read_unlock_bh() from one task if the matching rcu_read_lock_bh()
735  * was invoked from some other task.
736  */
737 static inline void rcu_read_lock_bh(void)
738 {
739         local_bh_disable();
740         __acquire(RCU_BH);
741         rcu_lock_acquire(&rcu_bh_lock_map);
742         RCU_LOCKDEP_WARN(!rcu_is_watching(),
743                          "rcu_read_lock_bh() used illegally while idle");
744 }
745 
746 /**
747  * rcu_read_unlock_bh() - marks the end of a softirq-only RCU critical section
748  *
749  * See rcu_read_lock_bh() for more information.
750  */
751 static inline void rcu_read_unlock_bh(void)
752 {
753         RCU_LOCKDEP_WARN(!rcu_is_watching(),
754                          "rcu_read_unlock_bh() used illegally while idle");
755         rcu_lock_release(&rcu_bh_lock_map);
756         __release(RCU_BH);
757         local_bh_enable();
758 }
759 
760 /**
761  * rcu_read_lock_sched() - mark the beginning of a RCU-sched critical section
762  *
763  * This is equivalent to rcu_read_lock(), but also disables preemption.
764  * Read-side critical sections can also be introduced by anything else that
765  * disables preemption, including local_irq_disable() and friends.  However,
766  * please note that the equivalence to rcu_read_lock() applies only to
767  * v5.0 and later.  Before v5.0, rcu_read_lock() and rcu_read_lock_sched()
768  * were unrelated.
769  *
770  * Note that rcu_read_lock_sched() and the matching rcu_read_unlock_sched()
771  * must occur in the same context, for example, it is illegal to invoke
772  * rcu_read_unlock_sched() from process context if the matching
773  * rcu_read_lock_sched() was invoked from an NMI handler.
774  */
775 static inline void rcu_read_lock_sched(void)
776 {
777         preempt_disable();
778         __acquire(RCU_SCHED);
779         rcu_lock_acquire(&rcu_sched_lock_map);
780         RCU_LOCKDEP_WARN(!rcu_is_watching(),
781                          "rcu_read_lock_sched() used illegally while idle");
782 }
783 
784 /* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
785 static inline notrace void rcu_read_lock_sched_notrace(void)
786 {
787         preempt_disable_notrace();
788         __acquire(RCU_SCHED);
789 }
790 
791 /**
792  * rcu_read_unlock_sched() - marks the end of a RCU-classic critical section
793  *
794  * See rcu_read_lock_sched() for more information.
795  */
796 static inline void rcu_read_unlock_sched(void)
797 {
798         RCU_LOCKDEP_WARN(!rcu_is_watching(),
799                          "rcu_read_unlock_sched() used illegally while idle");
800         rcu_lock_release(&rcu_sched_lock_map);
801         __release(RCU_SCHED);
802         preempt_enable();
803 }
804 
805 /* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
806 static inline notrace void rcu_read_unlock_sched_notrace(void)
807 {
808         __release(RCU_SCHED);
809         preempt_enable_notrace();
810 }
811 
812 /**
813  * RCU_INIT_POINTER() - initialize an RCU protected pointer
814  * @p: The pointer to be initialized.
815  * @v: The value to initialized the pointer to.
816  *
817  * Initialize an RCU-protected pointer in special cases where readers
818  * do not need ordering constraints on the CPU or the compiler.  These
819  * special cases are:
820  *
821  * 1.   This use of RCU_INIT_POINTER() is NULLing out the pointer *or*
822  * 2.   The caller has taken whatever steps are required to prevent
823  *      RCU readers from concurrently accessing this pointer *or*
824  * 3.   The referenced data structure has already been exposed to
825  *      readers either at compile time or via rcu_assign_pointer() *and*
826  *
827  *      a.      You have not made *any* reader-visible changes to
828  *              this structure since then *or*
829  *      b.      It is OK for readers accessing this structure from its
830  *              new location to see the old state of the structure.  (For
831  *              example, the changes were to statistical counters or to
832  *              other state where exact synchronization is not required.)
833  *
834  * Failure to follow these rules governing use of RCU_INIT_POINTER() will
835  * result in impossible-to-diagnose memory corruption.  As in the structures
836  * will look OK in crash dumps, but any concurrent RCU readers might
837  * see pre-initialized values of the referenced data structure.  So
838  * please be very careful how you use RCU_INIT_POINTER()!!!
839  *
840  * If you are creating an RCU-protected linked structure that is accessed
841  * by a single external-to-structure RCU-protected pointer, then you may
842  * use RCU_INIT_POINTER() to initialize the internal RCU-protected
843  * pointers, but you must use rcu_assign_pointer() to initialize the
844  * external-to-structure pointer *after* you have completely initialized
845  * the reader-accessible portions of the linked structure.
846  *
847  * Note that unlike rcu_assign_pointer(), RCU_INIT_POINTER() provides no
848  * ordering guarantees for either the CPU or the compiler.
849  */
850 #define RCU_INIT_POINTER(p, v) \
851         do { \
852                 rcu_check_sparse(p, __rcu); \
853                 WRITE_ONCE(p, RCU_INITIALIZER(v)); \
854         } while (0)
855 
856 /**
857  * RCU_POINTER_INITIALIZER() - statically initialize an RCU protected pointer
858  * @p: The pointer to be initialized.
859  * @v: The value to initialized the pointer to.
860  *
861  * GCC-style initialization for an RCU-protected pointer in a structure field.
862  */
863 #define RCU_POINTER_INITIALIZER(p, v) \
864                 .p = RCU_INITIALIZER(v)
865 
866 /*
867  * Does the specified offset indicate that the corresponding rcu_head
868  * structure can be handled by kvfree_rcu()?
869  */
870 #define __is_kvfree_rcu_offset(offset) ((offset) < 4096)
871 
872 /**
873  * kfree_rcu() - kfree an object after a grace period.
874  * @ptr: pointer to kfree for both single- and double-argument invocations.
875  * @rhf: the name of the struct rcu_head within the type of @ptr,
876  *       but only for double-argument invocations.
877  *
878  * Many rcu callbacks functions just call kfree() on the base structure.
879  * These functions are trivial, but their size adds up, and furthermore
880  * when they are used in a kernel module, that module must invoke the
881  * high-latency rcu_barrier() function at module-unload time.
882  *
883  * The kfree_rcu() function handles this issue.  Rather than encoding a
884  * function address in the embedded rcu_head structure, kfree_rcu() instead
885  * encodes the offset of the rcu_head structure within the base structure.
886  * Because the functions are not allowed in the low-order 4096 bytes of
887  * kernel virtual memory, offsets up to 4095 bytes can be accommodated.
888  * If the offset is larger than 4095 bytes, a compile-time error will
889  * be generated in kvfree_rcu_arg_2(). If this error is triggered, you can
890  * either fall back to use of call_rcu() or rearrange the structure to
891  * position the rcu_head structure into the first 4096 bytes.
892  *
893  * Note that the allowable offset might decrease in the future, for example,
894  * to allow something like kmem_cache_free_rcu().
895  *
896  * The BUILD_BUG_ON check must not involve any function calls, hence the
897  * checks are done in macros here.
898  */
899 #define kfree_rcu(ptr, rhf...) kvfree_rcu(ptr, ## rhf)
900 
901 /**
902  * kvfree_rcu() - kvfree an object after a grace period.
903  *
904  * This macro consists of one or two arguments and it is
905  * based on whether an object is head-less or not. If it
906  * has a head then a semantic stays the same as it used
907  * to be before:
908  *
909  *     kvfree_rcu(ptr, rhf);
910  *
911  * where @ptr is a pointer to kvfree(), @rhf is the name
912  * of the rcu_head structure within the type of @ptr.
913  *
914  * When it comes to head-less variant, only one argument
915  * is passed and that is just a pointer which has to be
916  * freed after a grace period. Therefore the semantic is
917  *
918  *     kvfree_rcu(ptr);
919  *
920  * where @ptr is a pointer to kvfree().
921  *
922  * Please note, head-less way of freeing is permitted to
923  * use from a context that has to follow might_sleep()
924  * annotation. Otherwise, please switch and embed the
925  * rcu_head structure within the type of @ptr.
926  */
927 #define kvfree_rcu(...) KVFREE_GET_MACRO(__VA_ARGS__,           \
928         kvfree_rcu_arg_2, kvfree_rcu_arg_1)(__VA_ARGS__)
929 
930 #define KVFREE_GET_MACRO(_1, _2, NAME, ...) NAME
931 #define kvfree_rcu_arg_2(ptr, rhf)                                      \
932 do {                                                                    \
933         typeof (ptr) ___p = (ptr);                                      \
934                                                                         \
935         if (___p) {                                                                     \
936                 BUILD_BUG_ON(!__is_kvfree_rcu_offset(offsetof(typeof(*(ptr)), rhf)));   \
937                 kvfree_call_rcu(&((___p)->rhf), (rcu_callback_t)(unsigned long)         \
938                         (offsetof(typeof(*(ptr)), rhf)));                               \
939         }                                                                               \
940 } while (0)
941 
942 #define kvfree_rcu_arg_1(ptr)                                   \
943 do {                                                            \
944         typeof(ptr) ___p = (ptr);                               \
945                                                                 \
946         if (___p)                                               \
947                 kvfree_call_rcu(NULL, (rcu_callback_t) (___p)); \
948 } while (0)
949 
950 /*
951  * Place this after a lock-acquisition primitive to guarantee that
952  * an UNLOCK+LOCK pair acts as a full barrier.  This guarantee applies
953  * if the UNLOCK and LOCK are executed by the same CPU or if the
954  * UNLOCK and LOCK operate on the same lock variable.
955  */
956 #ifdef CONFIG_ARCH_WEAK_RELEASE_ACQUIRE
957 #define smp_mb__after_unlock_lock()     smp_mb()  /* Full ordering for lock. */
958 #else /* #ifdef CONFIG_ARCH_WEAK_RELEASE_ACQUIRE */
959 #define smp_mb__after_unlock_lock()     do { } while (0)
960 #endif /* #else #ifdef CONFIG_ARCH_WEAK_RELEASE_ACQUIRE */
961 
962 
963 /* Has the specified rcu_head structure been handed to call_rcu()? */
964 
965 /**
966  * rcu_head_init - Initialize rcu_head for rcu_head_after_call_rcu()
967  * @rhp: The rcu_head structure to initialize.
968  *
969  * If you intend to invoke rcu_head_after_call_rcu() to test whether a
970  * given rcu_head structure has already been passed to call_rcu(), then
971  * you must also invoke this rcu_head_init() function on it just after
972  * allocating that structure.  Calls to this function must not race with
973  * calls to call_rcu(), rcu_head_after_call_rcu(), or callback invocation.
974  */
975 static inline void rcu_head_init(struct rcu_head *rhp)
976 {
977         rhp->func = (rcu_callback_t)~0L;
978 }
979 
980 /**
981  * rcu_head_after_call_rcu() - Has this rcu_head been passed to call_rcu()?
982  * @rhp: The rcu_head structure to test.
983  * @f: The function passed to call_rcu() along with @rhp.
984  *
985  * Returns @true if the @rhp has been passed to call_rcu() with @func,
986  * and @false otherwise.  Emits a warning in any other case, including
987  * the case where @rhp has already been invoked after a grace period.
988  * Calls to this function must not race with callback invocation.  One way
989  * to avoid such races is to enclose the call to rcu_head_after_call_rcu()
990  * in an RCU read-side critical section that includes a read-side fetch
991  * of the pointer to the structure containing @rhp.
992  */
993 static inline bool
994 rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f)
995 {
996         rcu_callback_t func = READ_ONCE(rhp->func);
997 
998         if (func == f)
999                 return true;
1000         WARN_ON_ONCE(func != (rcu_callback_t)~0L);
1001         return false;
1002 }
1003 
1004 /* kernel/ksysfs.c definitions */
1005 extern int rcu_expedited;
1006 extern int rcu_normal;
1007 
1008 #endif /* __LINUX_RCUPDATE_H */
1009 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp