~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/rcu/tree_plugin.h

Version: ~ [ linux-5.4-rc3 ] ~ [ linux-5.3.6 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.79 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.149 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.196 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.196 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.75 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Read-Copy Update mechanism for mutual exclusion (tree-based version)
  3  * Internal non-public definitions that provide either classic
  4  * or preemptible semantics.
  5  *
  6  * This program is free software; you can redistribute it and/or modify
  7  * it under the terms of the GNU General Public License as published by
  8  * the Free Software Foundation; either version 2 of the License, or
  9  * (at your option) any later version.
 10  *
 11  * This program is distributed in the hope that it will be useful,
 12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 14  * GNU General Public License for more details.
 15  *
 16  * You should have received a copy of the GNU General Public License
 17  * along with this program; if not, you can access it online at
 18  * http://www.gnu.org/licenses/gpl-2.0.html.
 19  *
 20  * Copyright Red Hat, 2009
 21  * Copyright IBM Corporation, 2009
 22  *
 23  * Author: Ingo Molnar <mingo@elte.hu>
 24  *         Paul E. McKenney <paulmck@linux.vnet.ibm.com>
 25  */
 26 
 27 #include <linux/delay.h>
 28 #include <linux/gfp.h>
 29 #include <linux/oom.h>
 30 #include <linux/smpboot.h>
 31 #include "../time/tick-internal.h"
 32 
 33 #ifdef CONFIG_RCU_BOOST
 34 
 35 #include "../locking/rtmutex_common.h"
 36 
 37 /*
 38  * Control variables for per-CPU and per-rcu_node kthreads.  These
 39  * handle all flavors of RCU.
 40  */
 41 static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);
 42 DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
 43 DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
 44 DEFINE_PER_CPU(char, rcu_cpu_has_work);
 45 
 46 #else /* #ifdef CONFIG_RCU_BOOST */
 47 
 48 /*
 49  * Some architectures do not define rt_mutexes, but if !CONFIG_RCU_BOOST,
 50  * all uses are in dead code.  Provide a definition to keep the compiler
 51  * happy, but add WARN_ON_ONCE() to complain if used in the wrong place.
 52  * This probably needs to be excluded from -rt builds.
 53  */
 54 #define rt_mutex_owner(a) ({ WARN_ON_ONCE(1); NULL; })
 55 
 56 #endif /* #else #ifdef CONFIG_RCU_BOOST */
 57 
 58 #ifdef CONFIG_RCU_NOCB_CPU
 59 static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */
 60 static bool have_rcu_nocb_mask;     /* Was rcu_nocb_mask allocated? */
 61 static bool __read_mostly rcu_nocb_poll;    /* Offload kthread are to poll. */
 62 #endif /* #ifdef CONFIG_RCU_NOCB_CPU */
 63 
 64 /*
 65  * Check the RCU kernel configuration parameters and print informative
 66  * messages about anything out of the ordinary.  If you like #ifdef, you
 67  * will love this function.
 68  */
 69 static void __init rcu_bootup_announce_oddness(void)
 70 {
 71         if (IS_ENABLED(CONFIG_RCU_TRACE))
 72                 pr_info("\tRCU debugfs-based tracing is enabled.\n");
 73         if ((IS_ENABLED(CONFIG_64BIT) && RCU_FANOUT != 64) ||
 74             (!IS_ENABLED(CONFIG_64BIT) && RCU_FANOUT != 32))
 75                 pr_info("\tCONFIG_RCU_FANOUT set to non-default value of %d\n",
 76                        RCU_FANOUT);
 77         if (rcu_fanout_exact)
 78                 pr_info("\tHierarchical RCU autobalancing is disabled.\n");
 79         if (IS_ENABLED(CONFIG_RCU_FAST_NO_HZ))
 80                 pr_info("\tRCU dyntick-idle grace-period acceleration is enabled.\n");
 81         if (IS_ENABLED(CONFIG_PROVE_RCU))
 82                 pr_info("\tRCU lockdep checking is enabled.\n");
 83         if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_RUNNABLE))
 84                 pr_info("\tRCU torture testing starts during boot.\n");
 85         if (RCU_NUM_LVLS >= 4)
 86                 pr_info("\tFour(or more)-level hierarchy is enabled.\n");
 87         if (RCU_FANOUT_LEAF != 16)
 88                 pr_info("\tBuild-time adjustment of leaf fanout to %d.\n",
 89                         RCU_FANOUT_LEAF);
 90         if (rcu_fanout_leaf != RCU_FANOUT_LEAF)
 91                 pr_info("\tBoot-time adjustment of leaf fanout to %d.\n", rcu_fanout_leaf);
 92         if (nr_cpu_ids != NR_CPUS)
 93                 pr_info("\tRCU restricting CPUs from NR_CPUS=%d to nr_cpu_ids=%d.\n", NR_CPUS, nr_cpu_ids);
 94         if (IS_ENABLED(CONFIG_RCU_BOOST))
 95                 pr_info("\tRCU kthread priority: %d.\n", kthread_prio);
 96 }
 97 
 98 #ifdef CONFIG_PREEMPT_RCU
 99 
100 RCU_STATE_INITIALIZER(rcu_preempt, 'p', call_rcu);
101 static struct rcu_state *const rcu_state_p = &rcu_preempt_state;
102 static struct rcu_data __percpu *const rcu_data_p = &rcu_preempt_data;
103 
104 static int rcu_preempted_readers_exp(struct rcu_node *rnp);
105 static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
106                                bool wake);
107 
108 /*
109  * Tell them what RCU they are running.
110  */
111 static void __init rcu_bootup_announce(void)
112 {
113         pr_info("Preemptible hierarchical RCU implementation.\n");
114         rcu_bootup_announce_oddness();
115 }
116 
117 /*
118  * Record a preemptible-RCU quiescent state for the specified CPU.  Note
119  * that this just means that the task currently running on the CPU is
120  * not in a quiescent state.  There might be any number of tasks blocked
121  * while in an RCU read-side critical section.
122  *
123  * As with the other rcu_*_qs() functions, callers to this function
124  * must disable preemption.
125  */
126 static void rcu_preempt_qs(void)
127 {
128         if (!__this_cpu_read(rcu_data_p->passed_quiesce)) {
129                 trace_rcu_grace_period(TPS("rcu_preempt"),
130                                        __this_cpu_read(rcu_data_p->gpnum),
131                                        TPS("cpuqs"));
132                 __this_cpu_write(rcu_data_p->passed_quiesce, 1);
133                 barrier(); /* Coordinate with rcu_preempt_check_callbacks(). */
134                 current->rcu_read_unlock_special.b.need_qs = false;
135         }
136 }
137 
138 /*
139  * We have entered the scheduler, and the current task might soon be
140  * context-switched away from.  If this task is in an RCU read-side
141  * critical section, we will no longer be able to rely on the CPU to
142  * record that fact, so we enqueue the task on the blkd_tasks list.
143  * The task will dequeue itself when it exits the outermost enclosing
144  * RCU read-side critical section.  Therefore, the current grace period
145  * cannot be permitted to complete until the blkd_tasks list entries
146  * predating the current grace period drain, in other words, until
147  * rnp->gp_tasks becomes NULL.
148  *
149  * Caller must disable preemption.
150  */
151 static void rcu_preempt_note_context_switch(void)
152 {
153         struct task_struct *t = current;
154         unsigned long flags;
155         struct rcu_data *rdp;
156         struct rcu_node *rnp;
157 
158         if (t->rcu_read_lock_nesting > 0 &&
159             !t->rcu_read_unlock_special.b.blocked) {
160 
161                 /* Possibly blocking in an RCU read-side critical section. */
162                 rdp = this_cpu_ptr(rcu_state_p->rda);
163                 rnp = rdp->mynode;
164                 raw_spin_lock_irqsave(&rnp->lock, flags);
165                 smp_mb__after_unlock_lock();
166                 t->rcu_read_unlock_special.b.blocked = true;
167                 t->rcu_blocked_node = rnp;
168 
169                 /*
170                  * If this CPU has already checked in, then this task
171                  * will hold up the next grace period rather than the
172                  * current grace period.  Queue the task accordingly.
173                  * If the task is queued for the current grace period
174                  * (i.e., this CPU has not yet passed through a quiescent
175                  * state for the current grace period), then as long
176                  * as that task remains queued, the current grace period
177                  * cannot end.  Note that there is some uncertainty as
178                  * to exactly when the current grace period started.
179                  * We take a conservative approach, which can result
180                  * in unnecessarily waiting on tasks that started very
181                  * slightly after the current grace period began.  C'est
182                  * la vie!!!
183                  *
184                  * But first, note that the current CPU must still be
185                  * on line!
186                  */
187                 WARN_ON_ONCE((rdp->grpmask & rcu_rnp_online_cpus(rnp)) == 0);
188                 WARN_ON_ONCE(!list_empty(&t->rcu_node_entry));
189                 if ((rnp->qsmask & rdp->grpmask) && rnp->gp_tasks != NULL) {
190                         list_add(&t->rcu_node_entry, rnp->gp_tasks->prev);
191                         rnp->gp_tasks = &t->rcu_node_entry;
192                         if (IS_ENABLED(CONFIG_RCU_BOOST) &&
193                             rnp->boost_tasks != NULL)
194                                 rnp->boost_tasks = rnp->gp_tasks;
195                 } else {
196                         list_add(&t->rcu_node_entry, &rnp->blkd_tasks);
197                         if (rnp->qsmask & rdp->grpmask)
198                                 rnp->gp_tasks = &t->rcu_node_entry;
199                 }
200                 trace_rcu_preempt_task(rdp->rsp->name,
201                                        t->pid,
202                                        (rnp->qsmask & rdp->grpmask)
203                                        ? rnp->gpnum
204                                        : rnp->gpnum + 1);
205                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
206         } else if (t->rcu_read_lock_nesting < 0 &&
207                    t->rcu_read_unlock_special.s) {
208 
209                 /*
210                  * Complete exit from RCU read-side critical section on
211                  * behalf of preempted instance of __rcu_read_unlock().
212                  */
213                 rcu_read_unlock_special(t);
214         }
215 
216         /*
217          * Either we were not in an RCU read-side critical section to
218          * begin with, or we have now recorded that critical section
219          * globally.  Either way, we can now note a quiescent state
220          * for this CPU.  Again, if we were in an RCU read-side critical
221          * section, and if that critical section was blocking the current
222          * grace period, then the fact that the task has been enqueued
223          * means that we continue to block the current grace period.
224          */
225         rcu_preempt_qs();
226 }
227 
228 /*
229  * Check for preempted RCU readers blocking the current grace period
230  * for the specified rcu_node structure.  If the caller needs a reliable
231  * answer, it must hold the rcu_node's ->lock.
232  */
233 static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
234 {
235         return rnp->gp_tasks != NULL;
236 }
237 
238 /*
239  * Advance a ->blkd_tasks-list pointer to the next entry, instead
240  * returning NULL if at the end of the list.
241  */
242 static struct list_head *rcu_next_node_entry(struct task_struct *t,
243                                              struct rcu_node *rnp)
244 {
245         struct list_head *np;
246 
247         np = t->rcu_node_entry.next;
248         if (np == &rnp->blkd_tasks)
249                 np = NULL;
250         return np;
251 }
252 
253 /*
254  * Return true if the specified rcu_node structure has tasks that were
255  * preempted within an RCU read-side critical section.
256  */
257 static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
258 {
259         return !list_empty(&rnp->blkd_tasks);
260 }
261 
262 /*
263  * Handle special cases during rcu_read_unlock(), such as needing to
264  * notify RCU core processing or task having blocked during the RCU
265  * read-side critical section.
266  */
267 void rcu_read_unlock_special(struct task_struct *t)
268 {
269         bool empty_exp;
270         bool empty_norm;
271         bool empty_exp_now;
272         unsigned long flags;
273         struct list_head *np;
274         bool drop_boost_mutex = false;
275         struct rcu_node *rnp;
276         union rcu_special special;
277 
278         /* NMI handlers cannot block and cannot safely manipulate state. */
279         if (in_nmi())
280                 return;
281 
282         local_irq_save(flags);
283 
284         /*
285          * If RCU core is waiting for this CPU to exit critical section,
286          * let it know that we have done so.  Because irqs are disabled,
287          * t->rcu_read_unlock_special cannot change.
288          */
289         special = t->rcu_read_unlock_special;
290         if (special.b.need_qs) {
291                 rcu_preempt_qs();
292                 t->rcu_read_unlock_special.b.need_qs = false;
293                 if (!t->rcu_read_unlock_special.s) {
294                         local_irq_restore(flags);
295                         return;
296                 }
297         }
298 
299         /* Hardware IRQ handlers cannot block, complain if they get here. */
300         if (in_irq() || in_serving_softirq()) {
301                 lockdep_rcu_suspicious(__FILE__, __LINE__,
302                                        "rcu_read_unlock() from irq or softirq with blocking in critical section!!!\n");
303                 pr_alert("->rcu_read_unlock_special: %#x (b: %d, nq: %d)\n",
304                          t->rcu_read_unlock_special.s,
305                          t->rcu_read_unlock_special.b.blocked,
306                          t->rcu_read_unlock_special.b.need_qs);
307                 local_irq_restore(flags);
308                 return;
309         }
310 
311         /* Clean up if blocked during RCU read-side critical section. */
312         if (special.b.blocked) {
313                 t->rcu_read_unlock_special.b.blocked = false;
314 
315                 /*
316                  * Remove this task from the list it blocked on.  The task
317                  * now remains queued on the rcu_node corresponding to
318                  * the CPU it first blocked on, so the first attempt to
319                  * acquire the task's rcu_node's ->lock will succeed.
320                  * Keep the loop and add a WARN_ON() out of sheer paranoia.
321                  */
322                 for (;;) {
323                         rnp = t->rcu_blocked_node;
324                         raw_spin_lock(&rnp->lock);  /* irqs already disabled. */
325                         smp_mb__after_unlock_lock();
326                         if (rnp == t->rcu_blocked_node)
327                                 break;
328                         WARN_ON_ONCE(1);
329                         raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
330                 }
331                 empty_norm = !rcu_preempt_blocked_readers_cgp(rnp);
332                 empty_exp = !rcu_preempted_readers_exp(rnp);
333                 smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */
334                 np = rcu_next_node_entry(t, rnp);
335                 list_del_init(&t->rcu_node_entry);
336                 t->rcu_blocked_node = NULL;
337                 trace_rcu_unlock_preempted_task(TPS("rcu_preempt"),
338                                                 rnp->gpnum, t->pid);
339                 if (&t->rcu_node_entry == rnp->gp_tasks)
340                         rnp->gp_tasks = np;
341                 if (&t->rcu_node_entry == rnp->exp_tasks)
342                         rnp->exp_tasks = np;
343                 if (IS_ENABLED(CONFIG_RCU_BOOST)) {
344                         if (&t->rcu_node_entry == rnp->boost_tasks)
345                                 rnp->boost_tasks = np;
346                         /* Snapshot ->boost_mtx ownership w/rnp->lock held. */
347                         drop_boost_mutex = rt_mutex_owner(&rnp->boost_mtx) == t;
348                 }
349 
350                 /*
351                  * If this was the last task on the current list, and if
352                  * we aren't waiting on any CPUs, report the quiescent state.
353                  * Note that rcu_report_unblock_qs_rnp() releases rnp->lock,
354                  * so we must take a snapshot of the expedited state.
355                  */
356                 empty_exp_now = !rcu_preempted_readers_exp(rnp);
357                 if (!empty_norm && !rcu_preempt_blocked_readers_cgp(rnp)) {
358                         trace_rcu_quiescent_state_report(TPS("preempt_rcu"),
359                                                          rnp->gpnum,
360                                                          0, rnp->qsmask,
361                                                          rnp->level,
362                                                          rnp->grplo,
363                                                          rnp->grphi,
364                                                          !!rnp->gp_tasks);
365                         rcu_report_unblock_qs_rnp(rcu_state_p, rnp, flags);
366                 } else {
367                         raw_spin_unlock_irqrestore(&rnp->lock, flags);
368                 }
369 
370                 /* Unboost if we were boosted. */
371                 if (IS_ENABLED(CONFIG_RCU_BOOST) && drop_boost_mutex)
372                         rt_mutex_unlock(&rnp->boost_mtx);
373 
374                 /*
375                  * If this was the last task on the expedited lists,
376                  * then we need to report up the rcu_node hierarchy.
377                  */
378                 if (!empty_exp && empty_exp_now)
379                         rcu_report_exp_rnp(rcu_state_p, rnp, true);
380         } else {
381                 local_irq_restore(flags);
382         }
383 }
384 
385 /*
386  * Dump detailed information for all tasks blocking the current RCU
387  * grace period on the specified rcu_node structure.
388  */
389 static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
390 {
391         unsigned long flags;
392         struct task_struct *t;
393 
394         raw_spin_lock_irqsave(&rnp->lock, flags);
395         if (!rcu_preempt_blocked_readers_cgp(rnp)) {
396                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
397                 return;
398         }
399         t = list_entry(rnp->gp_tasks->prev,
400                        struct task_struct, rcu_node_entry);
401         list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry)
402                 sched_show_task(t);
403         raw_spin_unlock_irqrestore(&rnp->lock, flags);
404 }
405 
406 /*
407  * Dump detailed information for all tasks blocking the current RCU
408  * grace period.
409  */
410 static void rcu_print_detail_task_stall(struct rcu_state *rsp)
411 {
412         struct rcu_node *rnp = rcu_get_root(rsp);
413 
414         rcu_print_detail_task_stall_rnp(rnp);
415         rcu_for_each_leaf_node(rsp, rnp)
416                 rcu_print_detail_task_stall_rnp(rnp);
417 }
418 
419 static void rcu_print_task_stall_begin(struct rcu_node *rnp)
420 {
421         pr_err("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):",
422                rnp->level, rnp->grplo, rnp->grphi);
423 }
424 
425 static void rcu_print_task_stall_end(void)
426 {
427         pr_cont("\n");
428 }
429 
430 /*
431  * Scan the current list of tasks blocked within RCU read-side critical
432  * sections, printing out the tid of each.
433  */
434 static int rcu_print_task_stall(struct rcu_node *rnp)
435 {
436         struct task_struct *t;
437         int ndetected = 0;
438 
439         if (!rcu_preempt_blocked_readers_cgp(rnp))
440                 return 0;
441         rcu_print_task_stall_begin(rnp);
442         t = list_entry(rnp->gp_tasks->prev,
443                        struct task_struct, rcu_node_entry);
444         list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
445                 pr_cont(" P%d", t->pid);
446                 ndetected++;
447         }
448         rcu_print_task_stall_end();
449         return ndetected;
450 }
451 
452 /*
453  * Check that the list of blocked tasks for the newly completed grace
454  * period is in fact empty.  It is a serious bug to complete a grace
455  * period that still has RCU readers blocked!  This function must be
456  * invoked -before- updating this rnp's ->gpnum, and the rnp's ->lock
457  * must be held by the caller.
458  *
459  * Also, if there are blocked tasks on the list, they automatically
460  * block the newly created grace period, so set up ->gp_tasks accordingly.
461  */
462 static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
463 {
464         WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp));
465         if (rcu_preempt_has_tasks(rnp))
466                 rnp->gp_tasks = rnp->blkd_tasks.next;
467         WARN_ON_ONCE(rnp->qsmask);
468 }
469 
470 /*
471  * Check for a quiescent state from the current CPU.  When a task blocks,
472  * the task is recorded in the corresponding CPU's rcu_node structure,
473  * which is checked elsewhere.
474  *
475  * Caller must disable hard irqs.
476  */
477 static void rcu_preempt_check_callbacks(void)
478 {
479         struct task_struct *t = current;
480 
481         if (t->rcu_read_lock_nesting == 0) {
482                 rcu_preempt_qs();
483                 return;
484         }
485         if (t->rcu_read_lock_nesting > 0 &&
486             __this_cpu_read(rcu_data_p->qs_pending) &&
487             !__this_cpu_read(rcu_data_p->passed_quiesce))
488                 t->rcu_read_unlock_special.b.need_qs = true;
489 }
490 
491 #ifdef CONFIG_RCU_BOOST
492 
493 static void rcu_preempt_do_callbacks(void)
494 {
495         rcu_do_batch(rcu_state_p, this_cpu_ptr(rcu_data_p));
496 }
497 
498 #endif /* #ifdef CONFIG_RCU_BOOST */
499 
500 /*
501  * Queue a preemptible-RCU callback for invocation after a grace period.
502  */
503 void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
504 {
505         __call_rcu(head, func, rcu_state_p, -1, 0);
506 }
507 EXPORT_SYMBOL_GPL(call_rcu);
508 
509 /**
510  * synchronize_rcu - wait until a grace period has elapsed.
511  *
512  * Control will return to the caller some time after a full grace
513  * period has elapsed, in other words after all currently executing RCU
514  * read-side critical sections have completed.  Note, however, that
515  * upon return from synchronize_rcu(), the caller might well be executing
516  * concurrently with new RCU read-side critical sections that began while
517  * synchronize_rcu() was waiting.  RCU read-side critical sections are
518  * delimited by rcu_read_lock() and rcu_read_unlock(), and may be nested.
519  *
520  * See the description of synchronize_sched() for more detailed information
521  * on memory ordering guarantees.
522  */
523 void synchronize_rcu(void)
524 {
525         RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
526                          lock_is_held(&rcu_lock_map) ||
527                          lock_is_held(&rcu_sched_lock_map),
528                          "Illegal synchronize_rcu() in RCU read-side critical section");
529         if (!rcu_scheduler_active)
530                 return;
531         if (rcu_gp_is_expedited())
532                 synchronize_rcu_expedited();
533         else
534                 wait_rcu_gp(call_rcu);
535 }
536 EXPORT_SYMBOL_GPL(synchronize_rcu);
537 
538 static DECLARE_WAIT_QUEUE_HEAD(sync_rcu_preempt_exp_wq);
539 
540 /*
541  * Return non-zero if there are any tasks in RCU read-side critical
542  * sections blocking the current preemptible-RCU expedited grace period.
543  * If there is no preemptible-RCU expedited grace period currently in
544  * progress, returns zero unconditionally.
545  */
546 static int rcu_preempted_readers_exp(struct rcu_node *rnp)
547 {
548         return rnp->exp_tasks != NULL;
549 }
550 
551 /*
552  * return non-zero if there is no RCU expedited grace period in progress
553  * for the specified rcu_node structure, in other words, if all CPUs and
554  * tasks covered by the specified rcu_node structure have done their bit
555  * for the current expedited grace period.  Works only for preemptible
556  * RCU -- other RCU implementation use other means.
557  *
558  * Caller must hold the root rcu_node's exp_funnel_mutex.
559  */
560 static int sync_rcu_preempt_exp_done(struct rcu_node *rnp)
561 {
562         return !rcu_preempted_readers_exp(rnp) &&
563                READ_ONCE(rnp->expmask) == 0;
564 }
565 
566 /*
567  * Report the exit from RCU read-side critical section for the last task
568  * that queued itself during or before the current expedited preemptible-RCU
569  * grace period.  This event is reported either to the rcu_node structure on
570  * which the task was queued or to one of that rcu_node structure's ancestors,
571  * recursively up the tree.  (Calm down, calm down, we do the recursion
572  * iteratively!)
573  *
574  * Caller must hold the root rcu_node's exp_funnel_mutex.
575  */
576 static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
577                                bool wake)
578 {
579         unsigned long flags;
580         unsigned long mask;
581 
582         raw_spin_lock_irqsave(&rnp->lock, flags);
583         smp_mb__after_unlock_lock();
584         for (;;) {
585                 if (!sync_rcu_preempt_exp_done(rnp)) {
586                         raw_spin_unlock_irqrestore(&rnp->lock, flags);
587                         break;
588                 }
589                 if (rnp->parent == NULL) {
590                         raw_spin_unlock_irqrestore(&rnp->lock, flags);
591                         if (wake) {
592                                 smp_mb(); /* EGP done before wake_up(). */
593                                 wake_up(&sync_rcu_preempt_exp_wq);
594                         }
595                         break;
596                 }
597                 mask = rnp->grpmask;
598                 raw_spin_unlock(&rnp->lock); /* irqs remain disabled */
599                 rnp = rnp->parent;
600                 raw_spin_lock(&rnp->lock); /* irqs already disabled */
601                 smp_mb__after_unlock_lock();
602                 rnp->expmask &= ~mask;
603         }
604 }
605 
606 /*
607  * Snapshot the tasks blocking the newly started preemptible-RCU expedited
608  * grace period for the specified rcu_node structure, phase 1.  If there
609  * are such tasks, set the ->expmask bits up the rcu_node tree and also
610  * set the ->expmask bits on the leaf rcu_node structures to tell phase 2
611  * that work is needed here.
612  *
613  * Caller must hold the root rcu_node's exp_funnel_mutex.
614  */
615 static void
616 sync_rcu_preempt_exp_init1(struct rcu_state *rsp, struct rcu_node *rnp)
617 {
618         unsigned long flags;
619         unsigned long mask;
620         struct rcu_node *rnp_up;
621 
622         raw_spin_lock_irqsave(&rnp->lock, flags);
623         smp_mb__after_unlock_lock();
624         WARN_ON_ONCE(rnp->expmask);
625         WARN_ON_ONCE(rnp->exp_tasks);
626         if (!rcu_preempt_has_tasks(rnp)) {
627                 /* No blocked tasks, nothing to do. */
628                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
629                 return;
630         }
631         /* Call for Phase 2 and propagate ->expmask bits up the tree. */
632         rnp->expmask = 1;
633         rnp_up = rnp;
634         while (rnp_up->parent) {
635                 mask = rnp_up->grpmask;
636                 rnp_up = rnp_up->parent;
637                 if (rnp_up->expmask & mask)
638                         break;
639                 raw_spin_lock(&rnp_up->lock); /* irqs already off */
640                 smp_mb__after_unlock_lock();
641                 rnp_up->expmask |= mask;
642                 raw_spin_unlock(&rnp_up->lock); /* irqs still off */
643         }
644         raw_spin_unlock_irqrestore(&rnp->lock, flags);
645 }
646 
647 /*
648  * Snapshot the tasks blocking the newly started preemptible-RCU expedited
649  * grace period for the specified rcu_node structure, phase 2.  If the
650  * leaf rcu_node structure has its ->expmask field set, check for tasks.
651  * If there are some, clear ->expmask and set ->exp_tasks accordingly,
652  * then initiate RCU priority boosting.  Otherwise, clear ->expmask and
653  * invoke rcu_report_exp_rnp() to clear out the upper-level ->expmask bits,
654  * enabling rcu_read_unlock_special() to do the bit-clearing.
655  *
656  * Caller must hold the root rcu_node's exp_funnel_mutex.
657  */
658 static void
659 sync_rcu_preempt_exp_init2(struct rcu_state *rsp, struct rcu_node *rnp)
660 {
661         unsigned long flags;
662 
663         raw_spin_lock_irqsave(&rnp->lock, flags);
664         smp_mb__after_unlock_lock();
665         if (!rnp->expmask) {
666                 /* Phase 1 didn't do anything, so Phase 2 doesn't either. */
667                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
668                 return;
669         }
670 
671         /* Phase 1 is over. */
672         rnp->expmask = 0;
673 
674         /*
675          * If there are still blocked tasks, set up ->exp_tasks so that
676          * rcu_read_unlock_special() will wake us and then boost them.
677          */
678         if (rcu_preempt_has_tasks(rnp)) {
679                 rnp->exp_tasks = rnp->blkd_tasks.next;
680                 rcu_initiate_boost(rnp, flags);  /* releases rnp->lock */
681                 return;
682         }
683 
684         /* No longer any blocked tasks, so undo bit setting. */
685         raw_spin_unlock_irqrestore(&rnp->lock, flags);
686         rcu_report_exp_rnp(rsp, rnp, false);
687 }
688 
689 /**
690  * synchronize_rcu_expedited - Brute-force RCU grace period
691  *
692  * Wait for an RCU-preempt grace period, but expedite it.  The basic
693  * idea is to invoke synchronize_sched_expedited() to push all the tasks to
694  * the ->blkd_tasks lists and wait for this list to drain.  This consumes
695  * significant time on all CPUs and is unfriendly to real-time workloads,
696  * so is thus not recommended for any sort of common-case code.
697  * In fact, if you are using synchronize_rcu_expedited() in a loop,
698  * please restructure your code to batch your updates, and then Use a
699  * single synchronize_rcu() instead.
700  */
701 void synchronize_rcu_expedited(void)
702 {
703         struct rcu_node *rnp;
704         struct rcu_node *rnp_unlock;
705         struct rcu_state *rsp = rcu_state_p;
706         unsigned long s;
707 
708         s = rcu_exp_gp_seq_snap(rsp);
709 
710         rnp_unlock = exp_funnel_lock(rsp, s);
711         if (rnp_unlock == NULL)
712                 return;  /* Someone else did our work for us. */
713 
714         rcu_exp_gp_seq_start(rsp);
715 
716         /* force all RCU readers onto ->blkd_tasks lists. */
717         synchronize_sched_expedited();
718 
719         /*
720          * Snapshot current state of ->blkd_tasks lists into ->expmask.
721          * Phase 1 sets bits and phase 2 permits rcu_read_unlock_special()
722          * to start clearing them.  Doing this in one phase leads to
723          * strange races between setting and clearing bits, so just say "no"!
724          */
725         rcu_for_each_leaf_node(rsp, rnp)
726                 sync_rcu_preempt_exp_init1(rsp, rnp);
727         rcu_for_each_leaf_node(rsp, rnp)
728                 sync_rcu_preempt_exp_init2(rsp, rnp);
729 
730         /* Wait for snapshotted ->blkd_tasks lists to drain. */
731         rnp = rcu_get_root(rsp);
732         wait_event(sync_rcu_preempt_exp_wq,
733                    sync_rcu_preempt_exp_done(rnp));
734 
735         /* Clean up and exit. */
736         rcu_exp_gp_seq_end(rsp);
737         mutex_unlock(&rnp_unlock->exp_funnel_mutex);
738 }
739 EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
740 
741 /**
742  * rcu_barrier - Wait until all in-flight call_rcu() callbacks complete.
743  *
744  * Note that this primitive does not necessarily wait for an RCU grace period
745  * to complete.  For example, if there are no RCU callbacks queued anywhere
746  * in the system, then rcu_barrier() is within its rights to return
747  * immediately, without waiting for anything, much less an RCU grace period.
748  */
749 void rcu_barrier(void)
750 {
751         _rcu_barrier(rcu_state_p);
752 }
753 EXPORT_SYMBOL_GPL(rcu_barrier);
754 
755 /*
756  * Initialize preemptible RCU's state structures.
757  */
758 static void __init __rcu_init_preempt(void)
759 {
760         rcu_init_one(rcu_state_p, rcu_data_p);
761 }
762 
763 /*
764  * Check for a task exiting while in a preemptible-RCU read-side
765  * critical section, clean up if so.  No need to issue warnings,
766  * as debug_check_no_locks_held() already does this if lockdep
767  * is enabled.
768  */
769 void exit_rcu(void)
770 {
771         struct task_struct *t = current;
772 
773         if (likely(list_empty(&current->rcu_node_entry)))
774                 return;
775         t->rcu_read_lock_nesting = 1;
776         barrier();
777         t->rcu_read_unlock_special.b.blocked = true;
778         __rcu_read_unlock();
779 }
780 
781 #else /* #ifdef CONFIG_PREEMPT_RCU */
782 
783 static struct rcu_state *const rcu_state_p = &rcu_sched_state;
784 static struct rcu_data __percpu *const rcu_data_p = &rcu_sched_data;
785 
786 /*
787  * Tell them what RCU they are running.
788  */
789 static void __init rcu_bootup_announce(void)
790 {
791         pr_info("Hierarchical RCU implementation.\n");
792         rcu_bootup_announce_oddness();
793 }
794 
795 /*
796  * Because preemptible RCU does not exist, we never have to check for
797  * CPUs being in quiescent states.
798  */
799 static void rcu_preempt_note_context_switch(void)
800 {
801 }
802 
803 /*
804  * Because preemptible RCU does not exist, there are never any preempted
805  * RCU readers.
806  */
807 static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
808 {
809         return 0;
810 }
811 
812 /*
813  * Because there is no preemptible RCU, there can be no readers blocked.
814  */
815 static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
816 {
817         return false;
818 }
819 
820 /*
821  * Because preemptible RCU does not exist, we never have to check for
822  * tasks blocked within RCU read-side critical sections.
823  */
824 static void rcu_print_detail_task_stall(struct rcu_state *rsp)
825 {
826 }
827 
828 /*
829  * Because preemptible RCU does not exist, we never have to check for
830  * tasks blocked within RCU read-side critical sections.
831  */
832 static int rcu_print_task_stall(struct rcu_node *rnp)
833 {
834         return 0;
835 }
836 
837 /*
838  * Because there is no preemptible RCU, there can be no readers blocked,
839  * so there is no need to check for blocked tasks.  So check only for
840  * bogus qsmask values.
841  */
842 static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
843 {
844         WARN_ON_ONCE(rnp->qsmask);
845 }
846 
847 /*
848  * Because preemptible RCU does not exist, it never has any callbacks
849  * to check.
850  */
851 static void rcu_preempt_check_callbacks(void)
852 {
853 }
854 
855 /*
856  * Wait for an rcu-preempt grace period, but make it happen quickly.
857  * But because preemptible RCU does not exist, map to rcu-sched.
858  */
859 void synchronize_rcu_expedited(void)
860 {
861         synchronize_sched_expedited();
862 }
863 EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
864 
865 /*
866  * Because preemptible RCU does not exist, rcu_barrier() is just
867  * another name for rcu_barrier_sched().
868  */
869 void rcu_barrier(void)
870 {
871         rcu_barrier_sched();
872 }
873 EXPORT_SYMBOL_GPL(rcu_barrier);
874 
875 /*
876  * Because preemptible RCU does not exist, it need not be initialized.
877  */
878 static void __init __rcu_init_preempt(void)
879 {
880 }
881 
882 /*
883  * Because preemptible RCU does not exist, tasks cannot possibly exit
884  * while in preemptible RCU read-side critical sections.
885  */
886 void exit_rcu(void)
887 {
888 }
889 
890 #endif /* #else #ifdef CONFIG_PREEMPT_RCU */
891 
892 #ifdef CONFIG_RCU_BOOST
893 
894 #include "../locking/rtmutex_common.h"
895 
896 #ifdef CONFIG_RCU_TRACE
897 
898 static void rcu_initiate_boost_trace(struct rcu_node *rnp)
899 {
900         if (!rcu_preempt_has_tasks(rnp))
901                 rnp->n_balk_blkd_tasks++;
902         else if (rnp->exp_tasks == NULL && rnp->gp_tasks == NULL)
903                 rnp->n_balk_exp_gp_tasks++;
904         else if (rnp->gp_tasks != NULL && rnp->boost_tasks != NULL)
905                 rnp->n_balk_boost_tasks++;
906         else if (rnp->gp_tasks != NULL && rnp->qsmask != 0)
907                 rnp->n_balk_notblocked++;
908         else if (rnp->gp_tasks != NULL &&
909                  ULONG_CMP_LT(jiffies, rnp->boost_time))
910                 rnp->n_balk_notyet++;
911         else
912                 rnp->n_balk_nos++;
913 }
914 
915 #else /* #ifdef CONFIG_RCU_TRACE */
916 
917 static void rcu_initiate_boost_trace(struct rcu_node *rnp)
918 {
919 }
920 
921 #endif /* #else #ifdef CONFIG_RCU_TRACE */
922 
923 static void rcu_wake_cond(struct task_struct *t, int status)
924 {
925         /*
926          * If the thread is yielding, only wake it when this
927          * is invoked from idle
928          */
929         if (status != RCU_KTHREAD_YIELDING || is_idle_task(current))
930                 wake_up_process(t);
931 }
932 
933 /*
934  * Carry out RCU priority boosting on the task indicated by ->exp_tasks
935  * or ->boost_tasks, advancing the pointer to the next task in the
936  * ->blkd_tasks list.
937  *
938  * Note that irqs must be enabled: boosting the task can block.
939  * Returns 1 if there are more tasks needing to be boosted.
940  */
941 static int rcu_boost(struct rcu_node *rnp)
942 {
943         unsigned long flags;
944         struct task_struct *t;
945         struct list_head *tb;
946 
947         if (READ_ONCE(rnp->exp_tasks) == NULL &&
948             READ_ONCE(rnp->boost_tasks) == NULL)
949                 return 0;  /* Nothing left to boost. */
950 
951         raw_spin_lock_irqsave(&rnp->lock, flags);
952         smp_mb__after_unlock_lock();
953 
954         /*
955          * Recheck under the lock: all tasks in need of boosting
956          * might exit their RCU read-side critical sections on their own.
957          */
958         if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL) {
959                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
960                 return 0;
961         }
962 
963         /*
964          * Preferentially boost tasks blocking expedited grace periods.
965          * This cannot starve the normal grace periods because a second
966          * expedited grace period must boost all blocked tasks, including
967          * those blocking the pre-existing normal grace period.
968          */
969         if (rnp->exp_tasks != NULL) {
970                 tb = rnp->exp_tasks;
971                 rnp->n_exp_boosts++;
972         } else {
973                 tb = rnp->boost_tasks;
974                 rnp->n_normal_boosts++;
975         }
976         rnp->n_tasks_boosted++;
977 
978         /*
979          * We boost task t by manufacturing an rt_mutex that appears to
980          * be held by task t.  We leave a pointer to that rt_mutex where
981          * task t can find it, and task t will release the mutex when it
982          * exits its outermost RCU read-side critical section.  Then
983          * simply acquiring this artificial rt_mutex will boost task
984          * t's priority.  (Thanks to tglx for suggesting this approach!)
985          *
986          * Note that task t must acquire rnp->lock to remove itself from
987          * the ->blkd_tasks list, which it will do from exit() if from
988          * nowhere else.  We therefore are guaranteed that task t will
989          * stay around at least until we drop rnp->lock.  Note that
990          * rnp->lock also resolves races between our priority boosting
991          * and task t's exiting its outermost RCU read-side critical
992          * section.
993          */
994         t = container_of(tb, struct task_struct, rcu_node_entry);
995         rt_mutex_init_proxy_locked(&rnp->boost_mtx, t);
996         raw_spin_unlock_irqrestore(&rnp->lock, flags);
997         /* Lock only for side effect: boosts task t's priority. */
998         rt_mutex_lock(&rnp->boost_mtx);
999         rt_mutex_unlock(&rnp->boost_mtx);  /* Then keep lockdep happy. */
1000 
1001         return READ_ONCE(rnp->exp_tasks) != NULL ||
1002                READ_ONCE(rnp->boost_tasks) != NULL;
1003 }
1004 
1005 /*
1006  * Priority-boosting kthread, one per leaf rcu_node.
1007  */
1008 static int rcu_boost_kthread(void *arg)
1009 {
1010         struct rcu_node *rnp = (struct rcu_node *)arg;
1011         int spincnt = 0;
1012         int more2boost;
1013 
1014         trace_rcu_utilization(TPS("Start boost kthread@init"));
1015         for (;;) {
1016                 rnp->boost_kthread_status = RCU_KTHREAD_WAITING;
1017                 trace_rcu_utilization(TPS("End boost kthread@rcu_wait"));
1018                 rcu_wait(rnp->boost_tasks || rnp->exp_tasks);
1019                 trace_rcu_utilization(TPS("Start boost kthread@rcu_wait"));
1020                 rnp->boost_kthread_status = RCU_KTHREAD_RUNNING;
1021                 more2boost = rcu_boost(rnp);
1022                 if (more2boost)
1023                         spincnt++;
1024                 else
1025                         spincnt = 0;
1026                 if (spincnt > 10) {
1027                         rnp->boost_kthread_status = RCU_KTHREAD_YIELDING;
1028                         trace_rcu_utilization(TPS("End boost kthread@rcu_yield"));
1029                         schedule_timeout_interruptible(2);
1030                         trace_rcu_utilization(TPS("Start boost kthread@rcu_yield"));
1031                         spincnt = 0;
1032                 }
1033         }
1034         /* NOTREACHED */
1035         trace_rcu_utilization(TPS("End boost kthread@notreached"));
1036         return 0;
1037 }
1038 
1039 /*
1040  * Check to see if it is time to start boosting RCU readers that are
1041  * blocking the current grace period, and, if so, tell the per-rcu_node
1042  * kthread to start boosting them.  If there is an expedited grace
1043  * period in progress, it is always time to boost.
1044  *
1045  * The caller must hold rnp->lock, which this function releases.
1046  * The ->boost_kthread_task is immortal, so we don't need to worry
1047  * about it going away.
1048  */
1049 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
1050         __releases(rnp->lock)
1051 {
1052         struct task_struct *t;
1053 
1054         if (!rcu_preempt_blocked_readers_cgp(rnp) && rnp->exp_tasks == NULL) {
1055                 rnp->n_balk_exp_gp_tasks++;
1056                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1057                 return;
1058         }
1059         if (rnp->exp_tasks != NULL ||
1060             (rnp->gp_tasks != NULL &&
1061              rnp->boost_tasks == NULL &&
1062              rnp->qsmask == 0 &&
1063              ULONG_CMP_GE(jiffies, rnp->boost_time))) {
1064                 if (rnp->exp_tasks == NULL)
1065                         rnp->boost_tasks = rnp->gp_tasks;
1066                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1067                 t = rnp->boost_kthread_task;
1068                 if (t)
1069                         rcu_wake_cond(t, rnp->boost_kthread_status);
1070         } else {
1071                 rcu_initiate_boost_trace(rnp);
1072                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1073         }
1074 }
1075 
1076 /*
1077  * Wake up the per-CPU kthread to invoke RCU callbacks.
1078  */
1079 static void invoke_rcu_callbacks_kthread(void)
1080 {
1081         unsigned long flags;
1082 
1083         local_irq_save(flags);
1084         __this_cpu_write(rcu_cpu_has_work, 1);
1085         if (__this_cpu_read(rcu_cpu_kthread_task) != NULL &&
1086             current != __this_cpu_read(rcu_cpu_kthread_task)) {
1087                 rcu_wake_cond(__this_cpu_read(rcu_cpu_kthread_task),
1088                               __this_cpu_read(rcu_cpu_kthread_status));
1089         }
1090         local_irq_restore(flags);
1091 }
1092 
1093 /*
1094  * Is the current CPU running the RCU-callbacks kthread?
1095  * Caller must have preemption disabled.
1096  */
1097 static bool rcu_is_callbacks_kthread(void)
1098 {
1099         return __this_cpu_read(rcu_cpu_kthread_task) == current;
1100 }
1101 
1102 #define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)
1103 
1104 /*
1105  * Do priority-boost accounting for the start of a new grace period.
1106  */
1107 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1108 {
1109         rnp->boost_time = jiffies + RCU_BOOST_DELAY_JIFFIES;
1110 }
1111 
1112 /*
1113  * Create an RCU-boost kthread for the specified node if one does not
1114  * already exist.  We only create this kthread for preemptible RCU.
1115  * Returns zero if all is well, a negated errno otherwise.
1116  */
1117 static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
1118                                        struct rcu_node *rnp)
1119 {
1120         int rnp_index = rnp - &rsp->node[0];
1121         unsigned long flags;
1122         struct sched_param sp;
1123         struct task_struct *t;
1124 
1125         if (rcu_state_p != rsp)
1126                 return 0;
1127 
1128         if (!rcu_scheduler_fully_active || rcu_rnp_online_cpus(rnp) == 0)
1129                 return 0;
1130 
1131         rsp->boost = 1;
1132         if (rnp->boost_kthread_task != NULL)
1133                 return 0;
1134         t = kthread_create(rcu_boost_kthread, (void *)rnp,
1135                            "rcub/%d", rnp_index);
1136         if (IS_ERR(t))
1137                 return PTR_ERR(t);
1138         raw_spin_lock_irqsave(&rnp->lock, flags);
1139         smp_mb__after_unlock_lock();
1140         rnp->boost_kthread_task = t;
1141         raw_spin_unlock_irqrestore(&rnp->lock, flags);
1142         sp.sched_priority = kthread_prio;
1143         sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
1144         wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */
1145         return 0;
1146 }
1147 
1148 static void rcu_kthread_do_work(void)
1149 {
1150         rcu_do_batch(&rcu_sched_state, this_cpu_ptr(&rcu_sched_data));
1151         rcu_do_batch(&rcu_bh_state, this_cpu_ptr(&rcu_bh_data));
1152         rcu_preempt_do_callbacks();
1153 }
1154 
1155 static void rcu_cpu_kthread_setup(unsigned int cpu)
1156 {
1157         struct sched_param sp;
1158 
1159         sp.sched_priority = kthread_prio;
1160         sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1161 }
1162 
1163 static void rcu_cpu_kthread_park(unsigned int cpu)
1164 {
1165         per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
1166 }
1167 
1168 static int rcu_cpu_kthread_should_run(unsigned int cpu)
1169 {
1170         return __this_cpu_read(rcu_cpu_has_work);
1171 }
1172 
1173 /*
1174  * Per-CPU kernel thread that invokes RCU callbacks.  This replaces the
1175  * RCU softirq used in flavors and configurations of RCU that do not
1176  * support RCU priority boosting.
1177  */
1178 static void rcu_cpu_kthread(unsigned int cpu)
1179 {
1180         unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status);
1181         char work, *workp = this_cpu_ptr(&rcu_cpu_has_work);
1182         int spincnt;
1183 
1184         for (spincnt = 0; spincnt < 10; spincnt++) {
1185                 trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait"));
1186                 local_bh_disable();
1187                 *statusp = RCU_KTHREAD_RUNNING;
1188                 this_cpu_inc(rcu_cpu_kthread_loops);
1189                 local_irq_disable();
1190                 work = *workp;
1191                 *workp = 0;
1192                 local_irq_enable();
1193                 if (work)
1194                         rcu_kthread_do_work();
1195                 local_bh_enable();
1196                 if (*workp == 0) {
1197                         trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
1198                         *statusp = RCU_KTHREAD_WAITING;
1199                         return;
1200                 }
1201         }
1202         *statusp = RCU_KTHREAD_YIELDING;
1203         trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
1204         schedule_timeout_interruptible(2);
1205         trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
1206         *statusp = RCU_KTHREAD_WAITING;
1207 }
1208 
1209 /*
1210  * Set the per-rcu_node kthread's affinity to cover all CPUs that are
1211  * served by the rcu_node in question.  The CPU hotplug lock is still
1212  * held, so the value of rnp->qsmaskinit will be stable.
1213  *
1214  * We don't include outgoingcpu in the affinity set, use -1 if there is
1215  * no outgoing CPU.  If there are no CPUs left in the affinity set,
1216  * this function allows the kthread to execute on any CPU.
1217  */
1218 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1219 {
1220         struct task_struct *t = rnp->boost_kthread_task;
1221         unsigned long mask = rcu_rnp_online_cpus(rnp);
1222         cpumask_var_t cm;
1223         int cpu;
1224 
1225         if (!t)
1226                 return;
1227         if (!zalloc_cpumask_var(&cm, GFP_KERNEL))
1228                 return;
1229         for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask >>= 1)
1230                 if ((mask & 0x1) && cpu != outgoingcpu)
1231                         cpumask_set_cpu(cpu, cm);
1232         if (cpumask_weight(cm) == 0)
1233                 cpumask_setall(cm);
1234         set_cpus_allowed_ptr(t, cm);
1235         free_cpumask_var(cm);
1236 }
1237 
1238 static struct smp_hotplug_thread rcu_cpu_thread_spec = {
1239         .store                  = &rcu_cpu_kthread_task,
1240         .thread_should_run      = rcu_cpu_kthread_should_run,
1241         .thread_fn              = rcu_cpu_kthread,
1242         .thread_comm            = "rcuc/%u",
1243         .setup                  = rcu_cpu_kthread_setup,
1244         .park                   = rcu_cpu_kthread_park,
1245 };
1246 
1247 /*
1248  * Spawn boost kthreads -- called as soon as the scheduler is running.
1249  */
1250 static void __init rcu_spawn_boost_kthreads(void)
1251 {
1252         struct rcu_node *rnp;
1253         int cpu;
1254 
1255         for_each_possible_cpu(cpu)
1256                 per_cpu(rcu_cpu_has_work, cpu) = 0;
1257         BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec));
1258         rcu_for_each_leaf_node(rcu_state_p, rnp)
1259                 (void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp);
1260 }
1261 
1262 static void rcu_prepare_kthreads(int cpu)
1263 {
1264         struct rcu_data *rdp = per_cpu_ptr(rcu_state_p->rda, cpu);
1265         struct rcu_node *rnp = rdp->mynode;
1266 
1267         /* Fire up the incoming CPU's kthread and leaf rcu_node kthread. */
1268         if (rcu_scheduler_fully_active)
1269                 (void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp);
1270 }
1271 
1272 #else /* #ifdef CONFIG_RCU_BOOST */
1273 
1274 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
1275         __releases(rnp->lock)
1276 {
1277         raw_spin_unlock_irqrestore(&rnp->lock, flags);
1278 }
1279 
1280 static void invoke_rcu_callbacks_kthread(void)
1281 {
1282         WARN_ON_ONCE(1);
1283 }
1284 
1285 static bool rcu_is_callbacks_kthread(void)
1286 {
1287         return false;
1288 }
1289 
1290 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1291 {
1292 }
1293 
1294 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1295 {
1296 }
1297 
1298 static void __init rcu_spawn_boost_kthreads(void)
1299 {
1300 }
1301 
1302 static void rcu_prepare_kthreads(int cpu)
1303 {
1304 }
1305 
1306 #endif /* #else #ifdef CONFIG_RCU_BOOST */
1307 
1308 #if !defined(CONFIG_RCU_FAST_NO_HZ)
1309 
1310 /*
1311  * Check to see if any future RCU-related work will need to be done
1312  * by the current CPU, even if none need be done immediately, returning
1313  * 1 if so.  This function is part of the RCU implementation; it is -not-
1314  * an exported member of the RCU API.
1315  *
1316  * Because we not have RCU_FAST_NO_HZ, just check whether this CPU needs
1317  * any flavor of RCU.
1318  */
1319 int rcu_needs_cpu(u64 basemono, u64 *nextevt)
1320 {
1321         *nextevt = KTIME_MAX;
1322         return IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL)
1323                ? 0 : rcu_cpu_has_callbacks(NULL);
1324 }
1325 
1326 /*
1327  * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
1328  * after it.
1329  */
1330 static void rcu_cleanup_after_idle(void)
1331 {
1332 }
1333 
1334 /*
1335  * Do the idle-entry grace-period work, which, because CONFIG_RCU_FAST_NO_HZ=n,
1336  * is nothing.
1337  */
1338 static void rcu_prepare_for_idle(void)
1339 {
1340 }
1341 
1342 /*
1343  * Don't bother keeping a running count of the number of RCU callbacks
1344  * posted because CONFIG_RCU_FAST_NO_HZ=n.
1345  */
1346 static void rcu_idle_count_callbacks_posted(void)
1347 {
1348 }
1349 
1350 #else /* #if !defined(CONFIG_RCU_FAST_NO_HZ) */
1351 
1352 /*
1353  * This code is invoked when a CPU goes idle, at which point we want
1354  * to have the CPU do everything required for RCU so that it can enter
1355  * the energy-efficient dyntick-idle mode.  This is handled by a
1356  * state machine implemented by rcu_prepare_for_idle() below.
1357  *
1358  * The following three proprocessor symbols control this state machine:
1359  *
1360  * RCU_IDLE_GP_DELAY gives the number of jiffies that a CPU is permitted
1361  *      to sleep in dyntick-idle mode with RCU callbacks pending.  This
1362  *      is sized to be roughly one RCU grace period.  Those energy-efficiency
1363  *      benchmarkers who might otherwise be tempted to set this to a large
1364  *      number, be warned: Setting RCU_IDLE_GP_DELAY too high can hang your
1365  *      system.  And if you are -that- concerned about energy efficiency,
1366  *      just power the system down and be done with it!
1367  * RCU_IDLE_LAZY_GP_DELAY gives the number of jiffies that a CPU is
1368  *      permitted to sleep in dyntick-idle mode with only lazy RCU
1369  *      callbacks pending.  Setting this too high can OOM your system.
1370  *
1371  * The values below work well in practice.  If future workloads require
1372  * adjustment, they can be converted into kernel config parameters, though
1373  * making the state machine smarter might be a better option.
1374  */
1375 #define RCU_IDLE_GP_DELAY 4             /* Roughly one grace period. */
1376 #define RCU_IDLE_LAZY_GP_DELAY (6 * HZ) /* Roughly six seconds. */
1377 
1378 static int rcu_idle_gp_delay = RCU_IDLE_GP_DELAY;
1379 module_param(rcu_idle_gp_delay, int, 0644);
1380 static int rcu_idle_lazy_gp_delay = RCU_IDLE_LAZY_GP_DELAY;
1381 module_param(rcu_idle_lazy_gp_delay, int, 0644);
1382 
1383 /*
1384  * Try to advance callbacks for all flavors of RCU on the current CPU, but
1385  * only if it has been awhile since the last time we did so.  Afterwards,
1386  * if there are any callbacks ready for immediate invocation, return true.
1387  */
1388 static bool __maybe_unused rcu_try_advance_all_cbs(void)
1389 {
1390         bool cbs_ready = false;
1391         struct rcu_data *rdp;
1392         struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1393         struct rcu_node *rnp;
1394         struct rcu_state *rsp;
1395 
1396         /* Exit early if we advanced recently. */
1397         if (jiffies == rdtp->last_advance_all)
1398                 return false;
1399         rdtp->last_advance_all = jiffies;
1400 
1401         for_each_rcu_flavor(rsp) {
1402                 rdp = this_cpu_ptr(rsp->rda);
1403                 rnp = rdp->mynode;
1404 
1405                 /*
1406                  * Don't bother checking unless a grace period has
1407                  * completed since we last checked and there are
1408                  * callbacks not yet ready to invoke.
1409                  */
1410                 if ((rdp->completed != rnp->completed ||
1411                      unlikely(READ_ONCE(rdp->gpwrap))) &&
1412                     rdp->nxttail[RCU_DONE_TAIL] != rdp->nxttail[RCU_NEXT_TAIL])
1413                         note_gp_changes(rsp, rdp);
1414 
1415                 if (cpu_has_callbacks_ready_to_invoke(rdp))
1416                         cbs_ready = true;
1417         }
1418         return cbs_ready;
1419 }
1420 
1421 /*
1422  * Allow the CPU to enter dyntick-idle mode unless it has callbacks ready
1423  * to invoke.  If the CPU has callbacks, try to advance them.  Tell the
1424  * caller to set the timeout based on whether or not there are non-lazy
1425  * callbacks.
1426  *
1427  * The caller must have disabled interrupts.
1428  */
1429 int rcu_needs_cpu(u64 basemono, u64 *nextevt)
1430 {
1431         struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1432         unsigned long dj;
1433 
1434         if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL)) {
1435                 *nextevt = KTIME_MAX;
1436                 return 0;
1437         }
1438 
1439         /* Snapshot to detect later posting of non-lazy callback. */
1440         rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
1441 
1442         /* If no callbacks, RCU doesn't need the CPU. */
1443         if (!rcu_cpu_has_callbacks(&rdtp->all_lazy)) {
1444                 *nextevt = KTIME_MAX;
1445                 return 0;
1446         }
1447 
1448         /* Attempt to advance callbacks. */
1449         if (rcu_try_advance_all_cbs()) {
1450                 /* Some ready to invoke, so initiate later invocation. */
1451                 invoke_rcu_core();
1452                 return 1;
1453         }
1454         rdtp->last_accelerate = jiffies;
1455 
1456         /* Request timer delay depending on laziness, and round. */
1457         if (!rdtp->all_lazy) {
1458                 dj = round_up(rcu_idle_gp_delay + jiffies,
1459                                rcu_idle_gp_delay) - jiffies;
1460         } else {
1461                 dj = round_jiffies(rcu_idle_lazy_gp_delay + jiffies) - jiffies;
1462         }
1463         *nextevt = basemono + dj * TICK_NSEC;
1464         return 0;
1465 }
1466 
1467 /*
1468  * Prepare a CPU for idle from an RCU perspective.  The first major task
1469  * is to sense whether nohz mode has been enabled or disabled via sysfs.
1470  * The second major task is to check to see if a non-lazy callback has
1471  * arrived at a CPU that previously had only lazy callbacks.  The third
1472  * major task is to accelerate (that is, assign grace-period numbers to)
1473  * any recently arrived callbacks.
1474  *
1475  * The caller must have disabled interrupts.
1476  */
1477 static void rcu_prepare_for_idle(void)
1478 {
1479         bool needwake;
1480         struct rcu_data *rdp;
1481         struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1482         struct rcu_node *rnp;
1483         struct rcu_state *rsp;
1484         int tne;
1485 
1486         if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL))
1487                 return;
1488 
1489         /* Handle nohz enablement switches conservatively. */
1490         tne = READ_ONCE(tick_nohz_active);
1491         if (tne != rdtp->tick_nohz_enabled_snap) {
1492                 if (rcu_cpu_has_callbacks(NULL))
1493                         invoke_rcu_core(); /* force nohz to see update. */
1494                 rdtp->tick_nohz_enabled_snap = tne;
1495                 return;
1496         }
1497         if (!tne)
1498                 return;
1499 
1500         /* If this is a no-CBs CPU, no callbacks, just return. */
1501         if (rcu_is_nocb_cpu(smp_processor_id()))
1502                 return;
1503 
1504         /*
1505          * If a non-lazy callback arrived at a CPU having only lazy
1506          * callbacks, invoke RCU core for the side-effect of recalculating
1507          * idle duration on re-entry to idle.
1508          */
1509         if (rdtp->all_lazy &&
1510             rdtp->nonlazy_posted != rdtp->nonlazy_posted_snap) {
1511                 rdtp->all_lazy = false;
1512                 rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
1513                 invoke_rcu_core();
1514                 return;
1515         }
1516 
1517         /*
1518          * If we have not yet accelerated this jiffy, accelerate all
1519          * callbacks on this CPU.
1520          */
1521         if (rdtp->last_accelerate == jiffies)
1522                 return;
1523         rdtp->last_accelerate = jiffies;
1524         for_each_rcu_flavor(rsp) {
1525                 rdp = this_cpu_ptr(rsp->rda);
1526                 if (!*rdp->nxttail[RCU_DONE_TAIL])
1527                         continue;
1528                 rnp = rdp->mynode;
1529                 raw_spin_lock(&rnp->lock); /* irqs already disabled. */
1530                 smp_mb__after_unlock_lock();
1531                 needwake = rcu_accelerate_cbs(rsp, rnp, rdp);
1532                 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
1533                 if (needwake)
1534                         rcu_gp_kthread_wake(rsp);
1535         }
1536 }
1537 
1538 /*
1539  * Clean up for exit from idle.  Attempt to advance callbacks based on
1540  * any grace periods that elapsed while the CPU was idle, and if any
1541  * callbacks are now ready to invoke, initiate invocation.
1542  */
1543 static void rcu_cleanup_after_idle(void)
1544 {
1545         if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL) ||
1546             rcu_is_nocb_cpu(smp_processor_id()))
1547                 return;
1548         if (rcu_try_advance_all_cbs())
1549                 invoke_rcu_core();
1550 }
1551 
1552 /*
1553  * Keep a running count of the number of non-lazy callbacks posted
1554  * on this CPU.  This running counter (which is never decremented) allows
1555  * rcu_prepare_for_idle() to detect when something out of the idle loop
1556  * posts a callback, even if an equal number of callbacks are invoked.
1557  * Of course, callbacks should only be posted from within a trace event
1558  * designed to be called from idle or from within RCU_NONIDLE().
1559  */
1560 static void rcu_idle_count_callbacks_posted(void)
1561 {
1562         __this_cpu_add(rcu_dynticks.nonlazy_posted, 1);
1563 }
1564 
1565 /*
1566  * Data for flushing lazy RCU callbacks at OOM time.
1567  */
1568 static atomic_t oom_callback_count;
1569 static DECLARE_WAIT_QUEUE_HEAD(oom_callback_wq);
1570 
1571 /*
1572  * RCU OOM callback -- decrement the outstanding count and deliver the
1573  * wake-up if we are the last one.
1574  */
1575 static void rcu_oom_callback(struct rcu_head *rhp)
1576 {
1577         if (atomic_dec_and_test(&oom_callback_count))
1578                 wake_up(&oom_callback_wq);
1579 }
1580 
1581 /*
1582  * Post an rcu_oom_notify callback on the current CPU if it has at
1583  * least one lazy callback.  This will unnecessarily post callbacks
1584  * to CPUs that already have a non-lazy callback at the end of their
1585  * callback list, but this is an infrequent operation, so accept some
1586  * extra overhead to keep things simple.
1587  */
1588 static void rcu_oom_notify_cpu(void *unused)
1589 {
1590         struct rcu_state *rsp;
1591         struct rcu_data *rdp;
1592 
1593         for_each_rcu_flavor(rsp) {
1594                 rdp = raw_cpu_ptr(rsp->rda);
1595                 if (rdp->qlen_lazy != 0) {
1596                         atomic_inc(&oom_callback_count);
1597                         rsp->call(&rdp->oom_head, rcu_oom_callback);
1598                 }
1599         }
1600 }
1601 
1602 /*
1603  * If low on memory, ensure that each CPU has a non-lazy callback.
1604  * This will wake up CPUs that have only lazy callbacks, in turn
1605  * ensuring that they free up the corresponding memory in a timely manner.
1606  * Because an uncertain amount of memory will be freed in some uncertain
1607  * timeframe, we do not claim to have freed anything.
1608  */
1609 static int rcu_oom_notify(struct notifier_block *self,
1610                           unsigned long notused, void *nfreed)
1611 {
1612         int cpu;
1613 
1614         /* Wait for callbacks from earlier instance to complete. */
1615         wait_event(oom_callback_wq, atomic_read(&oom_callback_count) == 0);
1616         smp_mb(); /* Ensure callback reuse happens after callback invocation. */
1617 
1618         /*
1619          * Prevent premature wakeup: ensure that all increments happen
1620          * before there is a chance of the counter reaching zero.
1621          */
1622         atomic_set(&oom_callback_count, 1);
1623 
1624         for_each_online_cpu(cpu) {
1625                 smp_call_function_single(cpu, rcu_oom_notify_cpu, NULL, 1);
1626                 cond_resched_rcu_qs();
1627         }
1628 
1629         /* Unconditionally decrement: no need to wake ourselves up. */
1630         atomic_dec(&oom_callback_count);
1631 
1632         return NOTIFY_OK;
1633 }
1634 
1635 static struct notifier_block rcu_oom_nb = {
1636         .notifier_call = rcu_oom_notify
1637 };
1638 
1639 static int __init rcu_register_oom_notifier(void)
1640 {
1641         register_oom_notifier(&rcu_oom_nb);
1642         return 0;
1643 }
1644 early_initcall(rcu_register_oom_notifier);
1645 
1646 #endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */
1647 
1648 #ifdef CONFIG_RCU_FAST_NO_HZ
1649 
1650 static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
1651 {
1652         struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
1653         unsigned long nlpd = rdtp->nonlazy_posted - rdtp->nonlazy_posted_snap;
1654 
1655         sprintf(cp, "last_accelerate: %04lx/%04lx, nonlazy_posted: %ld, %c%c",
1656                 rdtp->last_accelerate & 0xffff, jiffies & 0xffff,
1657                 ulong2long(nlpd),
1658                 rdtp->all_lazy ? 'L' : '.',
1659                 rdtp->tick_nohz_enabled_snap ? '.' : 'D');
1660 }
1661 
1662 #else /* #ifdef CONFIG_RCU_FAST_NO_HZ */
1663 
1664 static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
1665 {
1666         *cp = '\0';
1667 }
1668 
1669 #endif /* #else #ifdef CONFIG_RCU_FAST_NO_HZ */
1670 
1671 /* Initiate the stall-info list. */
1672 static void print_cpu_stall_info_begin(void)
1673 {
1674         pr_cont("\n");
1675 }
1676 
1677 /*
1678  * Print out diagnostic information for the specified stalled CPU.
1679  *
1680  * If the specified CPU is aware of the current RCU grace period
1681  * (flavor specified by rsp), then print the number of scheduling
1682  * clock interrupts the CPU has taken during the time that it has
1683  * been aware.  Otherwise, print the number of RCU grace periods
1684  * that this CPU is ignorant of, for example, "1" if the CPU was
1685  * aware of the previous grace period.
1686  *
1687  * Also print out idle and (if CONFIG_RCU_FAST_NO_HZ) idle-entry info.
1688  */
1689 static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
1690 {
1691         char fast_no_hz[72];
1692         struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
1693         struct rcu_dynticks *rdtp = rdp->dynticks;
1694         char *ticks_title;
1695         unsigned long ticks_value;
1696 
1697         if (rsp->gpnum == rdp->gpnum) {
1698                 ticks_title = "ticks this GP";
1699                 ticks_value = rdp->ticks_this_gp;
1700         } else {
1701                 ticks_title = "GPs behind";
1702                 ticks_value = rsp->gpnum - rdp->gpnum;
1703         }
1704         print_cpu_stall_fast_no_hz(fast_no_hz, cpu);
1705         pr_err("\t%d: (%lu %s) idle=%03x/%llx/%d softirq=%u/%u fqs=%ld %s\n",
1706                cpu, ticks_value, ticks_title,
1707                atomic_read(&rdtp->dynticks) & 0xfff,
1708                rdtp->dynticks_nesting, rdtp->dynticks_nmi_nesting,
1709                rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
1710                READ_ONCE(rsp->n_force_qs) - rsp->n_force_qs_gpstart,
1711                fast_no_hz);
1712 }
1713 
1714 /* Terminate the stall-info list. */
1715 static void print_cpu_stall_info_end(void)
1716 {
1717         pr_err("\t");
1718 }
1719 
1720 /* Zero ->ticks_this_gp for all flavors of RCU. */
1721 static void zero_cpu_stall_ticks(struct rcu_data *rdp)
1722 {
1723         rdp->ticks_this_gp = 0;
1724         rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id());
1725 }
1726 
1727 /* Increment ->ticks_this_gp for all flavors of RCU. */
1728 static void increment_cpu_stall_ticks(void)
1729 {
1730         struct rcu_state *rsp;
1731 
1732         for_each_rcu_flavor(rsp)
1733                 raw_cpu_inc(rsp->rda->ticks_this_gp);
1734 }
1735 
1736 #ifdef CONFIG_RCU_NOCB_CPU
1737 
1738 /*
1739  * Offload callback processing from the boot-time-specified set of CPUs
1740  * specified by rcu_nocb_mask.  For each CPU in the set, there is a
1741  * kthread created that pulls the callbacks from the corresponding CPU,
1742  * waits for a grace period to elapse, and invokes the callbacks.
1743  * The no-CBs CPUs do a wake_up() on their kthread when they insert
1744  * a callback into any empty list, unless the rcu_nocb_poll boot parameter
1745  * has been specified, in which case each kthread actively polls its
1746  * CPU.  (Which isn't so great for energy efficiency, but which does
1747  * reduce RCU's overhead on that CPU.)
1748  *
1749  * This is intended to be used in conjunction with Frederic Weisbecker's
1750  * adaptive-idle work, which would seriously reduce OS jitter on CPUs
1751  * running CPU-bound user-mode computations.
1752  *
1753  * Offloading of callback processing could also in theory be used as
1754  * an energy-efficiency measure because CPUs with no RCU callbacks
1755  * queued are more aggressive about entering dyntick-idle mode.
1756  */
1757 
1758 
1759 /* Parse the boot-time rcu_nocb_mask CPU list from the kernel parameters. */
1760 static int __init rcu_nocb_setup(char *str)
1761 {
1762         alloc_bootmem_cpumask_var(&rcu_nocb_mask);
1763         have_rcu_nocb_mask = true;
1764         cpulist_parse(str, rcu_nocb_mask);
1765         return 1;
1766 }
1767 __setup("rcu_nocbs=", rcu_nocb_setup);
1768 
1769 static int __init parse_rcu_nocb_poll(char *arg)
1770 {
1771         rcu_nocb_poll = 1;
1772         return 0;
1773 }
1774 early_param("rcu_nocb_poll", parse_rcu_nocb_poll);
1775 
1776 /*
1777  * Wake up any no-CBs CPUs' kthreads that were waiting on the just-ended
1778  * grace period.
1779  */
1780 static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)
1781 {
1782         wake_up_all(&rnp->nocb_gp_wq[rnp->completed & 0x1]);
1783 }
1784 
1785 /*
1786  * Set the root rcu_node structure's ->need_future_gp field
1787  * based on the sum of those of all rcu_node structures.  This does
1788  * double-count the root rcu_node structure's requests, but this
1789  * is necessary to handle the possibility of a rcu_nocb_kthread()
1790  * having awakened during the time that the rcu_node structures
1791  * were being updated for the end of the previous grace period.
1792  */
1793 static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)
1794 {
1795         rnp->need_future_gp[(rnp->completed + 1) & 0x1] += nrq;
1796 }
1797 
1798 static void rcu_init_one_nocb(struct rcu_node *rnp)
1799 {
1800         init_waitqueue_head(&rnp->nocb_gp_wq[0]);
1801         init_waitqueue_head(&rnp->nocb_gp_wq[1]);
1802 }
1803 
1804 #ifndef CONFIG_RCU_NOCB_CPU_ALL
1805 /* Is the specified CPU a no-CBs CPU? */
1806 bool rcu_is_nocb_cpu(int cpu)
1807 {
1808         if (have_rcu_nocb_mask)
1809                 return cpumask_test_cpu(cpu, rcu_nocb_mask);
1810         return false;
1811 }
1812 #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
1813 
1814 /*
1815  * Kick the leader kthread for this NOCB group.
1816  */
1817 static void wake_nocb_leader(struct rcu_data *rdp, bool force)
1818 {
1819         struct rcu_data *rdp_leader = rdp->nocb_leader;
1820 
1821         if (!READ_ONCE(rdp_leader->nocb_kthread))
1822                 return;
1823         if (READ_ONCE(rdp_leader->nocb_leader_sleep) || force) {
1824                 /* Prior smp_mb__after_atomic() orders against prior enqueue. */
1825                 WRITE_ONCE(rdp_leader->nocb_leader_sleep, false);
1826                 wake_up(&rdp_leader->nocb_wq);
1827         }
1828 }
1829 
1830 /*
1831  * Does the specified CPU need an RCU callback for the specified flavor
1832  * of rcu_barrier()?
1833  */
1834 static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)
1835 {
1836         struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
1837         unsigned long ret;
1838 #ifdef CONFIG_PROVE_RCU
1839         struct rcu_head *rhp;
1840 #endif /* #ifdef CONFIG_PROVE_RCU */
1841 
1842         /*
1843          * Check count of all no-CBs callbacks awaiting invocation.
1844          * There needs to be a barrier before this function is called,
1845          * but associated with a prior determination that no more
1846          * callbacks would be posted.  In the worst case, the first
1847          * barrier in _rcu_barrier() suffices (but the caller cannot
1848          * necessarily rely on this, not a substitute for the caller
1849          * getting the concurrency design right!).  There must also be
1850          * a barrier between the following load an posting of a callback
1851          * (if a callback is in fact needed).  This is associated with an
1852          * atomic_inc() in the caller.
1853          */
1854         ret = atomic_long_read(&rdp->nocb_q_count);
1855 
1856 #ifdef CONFIG_PROVE_RCU
1857         rhp = READ_ONCE(rdp->nocb_head);
1858         if (!rhp)
1859                 rhp = READ_ONCE(rdp->nocb_gp_head);
1860         if (!rhp)
1861                 rhp = READ_ONCE(rdp->nocb_follower_head);
1862 
1863         /* Having no rcuo kthread but CBs after scheduler starts is bad! */
1864         if (!READ_ONCE(rdp->nocb_kthread) && rhp &&
1865             rcu_scheduler_fully_active) {
1866                 /* RCU callback enqueued before CPU first came online??? */
1867                 pr_err("RCU: Never-onlined no-CBs CPU %d has CB %p\n",
1868                        cpu, rhp->func);
1869                 WARN_ON_ONCE(1);
1870         }
1871 #endif /* #ifdef CONFIG_PROVE_RCU */
1872 
1873         return !!ret;
1874 }
1875 
1876 /*
1877  * Enqueue the specified string of rcu_head structures onto the specified
1878  * CPU's no-CBs lists.  The CPU is specified by rdp, the head of the
1879  * string by rhp, and the tail of the string by rhtp.  The non-lazy/lazy
1880  * counts are supplied by rhcount and rhcount_lazy.
1881  *
1882  * If warranted, also wake up the kthread servicing this CPUs queues.
1883  */
1884 static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,
1885                                     struct rcu_head *rhp,
1886                                     struct rcu_head **rhtp,
1887                                     int rhcount, int rhcount_lazy,
1888                                     unsigned long flags)
1889 {
1890         int len;
1891         struct rcu_head **old_rhpp;
1892         struct task_struct *t;
1893 
1894         /* Enqueue the callback on the nocb list and update counts. */
1895         atomic_long_add(rhcount, &rdp->nocb_q_count);
1896         /* rcu_barrier() relies on ->nocb_q_count add before xchg. */
1897         old_rhpp = xchg(&rdp->nocb_tail, rhtp);
1898         WRITE_ONCE(*old_rhpp, rhp);
1899         atomic_long_add(rhcount_lazy, &rdp->nocb_q_count_lazy);
1900         smp_mb__after_atomic(); /* Store *old_rhpp before _wake test. */
1901 
1902         /* If we are not being polled and there is a kthread, awaken it ... */
1903         t = READ_ONCE(rdp->nocb_kthread);
1904         if (rcu_nocb_poll || !t) {
1905                 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1906                                     TPS("WakeNotPoll"));
1907                 return;
1908         }
1909         len = atomic_long_read(&rdp->nocb_q_count);
1910         if (old_rhpp == &rdp->nocb_head) {
1911                 if (!irqs_disabled_flags(flags)) {
1912                         /* ... if queue was empty ... */
1913                         wake_nocb_leader(rdp, false);
1914                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1915                                             TPS("WakeEmpty"));
1916                 } else {
1917                         rdp->nocb_defer_wakeup = RCU_NOGP_WAKE;
1918                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1919                                             TPS("WakeEmptyIsDeferred"));
1920                 }
1921                 rdp->qlen_last_fqs_check = 0;
1922         } else if (len > rdp->qlen_last_fqs_check + qhimark) {
1923                 /* ... or if many callbacks queued. */
1924                 if (!irqs_disabled_flags(flags)) {
1925                         wake_nocb_leader(rdp, true);
1926                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1927                                             TPS("WakeOvf"));
1928                 } else {
1929                         rdp->nocb_defer_wakeup = RCU_NOGP_WAKE_FORCE;
1930                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1931                                             TPS("WakeOvfIsDeferred"));
1932                 }
1933                 rdp->qlen_last_fqs_check = LONG_MAX / 2;
1934         } else {
1935                 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WakeNot"));
1936         }
1937         return;
1938 }
1939 
1940 /*
1941  * This is a helper for __call_rcu(), which invokes this when the normal
1942  * callback queue is inoperable.  If this is not a no-CBs CPU, this
1943  * function returns failure back to __call_rcu(), which can complain
1944  * appropriately.
1945  *
1946  * Otherwise, this function queues the callback where the corresponding
1947  * "rcuo" kthread can find it.
1948  */
1949 static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
1950                             bool lazy, unsigned long flags)
1951 {
1952 
1953         if (!rcu_is_nocb_cpu(rdp->cpu))
1954                 return false;
1955         __call_rcu_nocb_enqueue(rdp, rhp, &rhp->next, 1, lazy, flags);
1956         if (__is_kfree_rcu_offset((unsigned long)rhp->func))
1957                 trace_rcu_kfree_callback(rdp->rsp->name, rhp,
1958                                          (unsigned long)rhp->func,
1959                                          -atomic_long_read(&rdp->nocb_q_count_lazy),
1960                                          -atomic_long_read(&rdp->nocb_q_count));
1961         else
1962                 trace_rcu_callback(rdp->rsp->name, rhp,
1963                                    -atomic_long_read(&rdp->nocb_q_count_lazy),
1964                                    -atomic_long_read(&rdp->nocb_q_count));
1965 
1966         /*
1967          * If called from an extended quiescent state with interrupts
1968          * disabled, invoke the RCU core in order to allow the idle-entry
1969          * deferred-wakeup check to function.
1970          */
1971         if (irqs_disabled_flags(flags) &&
1972             !rcu_is_watching() &&
1973             cpu_online(smp_processor_id()))
1974                 invoke_rcu_core();
1975 
1976         return true;
1977 }
1978 
1979 /*
1980  * Adopt orphaned callbacks on a no-CBs CPU, or return 0 if this is
1981  * not a no-CBs CPU.
1982  */
1983 static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
1984                                                      struct rcu_data *rdp,
1985                                                      unsigned long flags)
1986 {
1987         long ql = rsp->qlen;
1988         long qll = rsp->qlen_lazy;
1989 
1990         /* If this is not a no-CBs CPU, tell the caller to do it the old way. */
1991         if (!rcu_is_nocb_cpu(smp_processor_id()))
1992                 return false;
1993         rsp->qlen = 0;
1994         rsp->qlen_lazy = 0;
1995 
1996         /* First, enqueue the donelist, if any.  This preserves CB ordering. */
1997         if (rsp->orphan_donelist != NULL) {
1998                 __call_rcu_nocb_enqueue(rdp, rsp->orphan_donelist,
1999                                         rsp->orphan_donetail, ql, qll, flags);
2000                 ql = qll = 0;
2001                 rsp->orphan_donelist = NULL;
2002                 rsp->orphan_donetail = &rsp->orphan_donelist;
2003         }
2004         if (rsp->orphan_nxtlist != NULL) {
2005                 __call_rcu_nocb_enqueue(rdp, rsp->orphan_nxtlist,
2006                                         rsp->orphan_nxttail, ql, qll, flags);
2007                 ql = qll = 0;
2008                 rsp->orphan_nxtlist = NULL;
2009                 rsp->orphan_nxttail = &rsp->orphan_nxtlist;
2010         }
2011         return true;
2012 }
2013 
2014 /*
2015  * If necessary, kick off a new grace period, and either way wait
2016  * for a subsequent grace period to complete.
2017  */
2018 static void rcu_nocb_wait_gp(struct rcu_data *rdp)
2019 {
2020         unsigned long c;
2021         bool d;
2022         unsigned long flags;
2023         bool needwake;
2024         struct rcu_node *rnp = rdp->mynode;
2025 
2026         raw_spin_lock_irqsave(&rnp->lock, flags);
2027         smp_mb__after_unlock_lock();
2028         needwake = rcu_start_future_gp(rnp, rdp, &c);
2029         raw_spin_unlock_irqrestore(&rnp->lock, flags);
2030         if (needwake)
2031                 rcu_gp_kthread_wake(rdp->rsp);
2032 
2033         /*
2034          * Wait for the grace period.  Do so interruptibly to avoid messing
2035          * up the load average.
2036          */
2037         trace_rcu_future_gp(rnp, rdp, c, TPS("StartWait"));
2038         for (;;) {
2039                 wait_event_interruptible(
2040                         rnp->nocb_gp_wq[c & 0x1],
2041                         (d = ULONG_CMP_GE(READ_ONCE(rnp->completed), c)));
2042                 if (likely(d))
2043                         break;
2044                 WARN_ON(signal_pending(current));
2045                 trace_rcu_future_gp(rnp, rdp, c, TPS("ResumeWait"));
2046         }
2047         trace_rcu_future_gp(rnp, rdp, c, TPS("EndWait"));
2048         smp_mb(); /* Ensure that CB invocation happens after GP end. */
2049 }
2050 
2051 /*
2052  * Leaders come here to wait for additional callbacks to show up.
2053  * This function does not return until callbacks appear.
2054  */
2055 static void nocb_leader_wait(struct rcu_data *my_rdp)
2056 {
2057         bool firsttime = true;
2058         bool gotcbs;
2059         struct rcu_data *rdp;
2060         struct rcu_head **tail;
2061 
2062 wait_again:
2063 
2064         /* Wait for callbacks to appear. */
2065         if (!rcu_nocb_poll) {
2066                 trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep");
2067                 wait_event_interruptible(my_rdp->nocb_wq,
2068                                 !READ_ONCE(my_rdp->nocb_leader_sleep));
2069                 /* Memory barrier handled by smp_mb() calls below and repoll. */
2070         } else if (firsttime) {
2071                 firsttime = false; /* Don't drown trace log with "Poll"! */
2072                 trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Poll");
2073         }
2074 
2075         /*
2076          * Each pass through the following loop checks a follower for CBs.
2077          * We are our own first follower.  Any CBs found are moved to
2078          * nocb_gp_head, where they await a grace period.
2079          */
2080         gotcbs = false;
2081         for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {
2082                 rdp->nocb_gp_head = READ_ONCE(rdp->nocb_head);
2083                 if (!rdp->nocb_gp_head)
2084                         continue;  /* No CBs here, try next follower. */
2085 
2086                 /* Move callbacks to wait-for-GP list, which is empty. */
2087                 WRITE_ONCE(rdp->nocb_head, NULL);
2088                 rdp->nocb_gp_tail = xchg(&rdp->nocb_tail, &rdp->nocb_head);
2089                 gotcbs = true;
2090         }
2091 
2092         /*
2093          * If there were no callbacks, sleep a bit, rescan after a
2094          * memory barrier, and go retry.
2095          */
2096         if (unlikely(!gotcbs)) {
2097                 if (!rcu_nocb_poll)
2098                         trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu,
2099                                             "WokeEmpty");
2100                 WARN_ON(signal_pending(current));
2101                 schedule_timeout_interruptible(1);
2102 
2103                 /* Rescan in case we were a victim of memory ordering. */
2104                 my_rdp->nocb_leader_sleep = true;
2105                 smp_mb();  /* Ensure _sleep true before scan. */
2106                 for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower)
2107                         if (READ_ONCE(rdp->nocb_head)) {
2108                                 /* Found CB, so short-circuit next wait. */
2109                                 my_rdp->nocb_leader_sleep = false;
2110                                 break;
2111                         }
2112                 goto wait_again;
2113         }
2114 
2115         /* Wait for one grace period. */
2116         rcu_nocb_wait_gp(my_rdp);
2117 
2118         /*
2119          * We left ->nocb_leader_sleep unset to reduce cache thrashing.
2120          * We set it now, but recheck for new callbacks while
2121          * traversing our follower list.
2122          */
2123         my_rdp->nocb_leader_sleep = true;
2124         smp_mb(); /* Ensure _sleep true before scan of ->nocb_head. */
2125 
2126         /* Each pass through the following loop wakes a follower, if needed. */
2127         for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {
2128                 if (READ_ONCE(rdp->nocb_head))
2129                         my_rdp->nocb_leader_sleep = false;/* No need to sleep.*/
2130                 if (!rdp->nocb_gp_head)
2131                         continue; /* No CBs, so no need to wake follower. */
2132 
2133                 /* Append callbacks to follower's "done" list. */
2134                 tail = xchg(&rdp->nocb_follower_tail, rdp->nocb_gp_tail);
2135                 *tail = rdp->nocb_gp_head;
2136                 smp_mb__after_atomic(); /* Store *tail before wakeup. */
2137                 if (rdp != my_rdp && tail == &rdp->nocb_follower_head) {
2138                         /*
2139                          * List was empty, wake up the follower.
2140                          * Memory barriers supplied by atomic_long_add().
2141                          */
2142                         wake_up(&rdp->nocb_wq);
2143                 }
2144         }
2145 
2146         /* If we (the leader) don't have CBs, go wait some more. */
2147         if (!my_rdp->nocb_follower_head)
2148                 goto wait_again;
2149 }
2150 
2151 /*
2152  * Followers come here to wait for additional callbacks to show up.
2153  * This function does not return until callbacks appear.
2154  */
2155 static void nocb_follower_wait(struct rcu_data *rdp)
2156 {
2157         bool firsttime = true;
2158 
2159         for (;;) {
2160                 if (!rcu_nocb_poll) {
2161                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2162                                             "FollowerSleep");
2163                         wait_event_interruptible(rdp->nocb_wq,
2164                                                  READ_ONCE(rdp->nocb_follower_head));
2165                 } else if (firsttime) {
2166                         /* Don't drown trace log with "Poll"! */
2167                         firsttime = false;
2168                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "Poll");
2169                 }
2170                 if (smp_load_acquire(&rdp->nocb_follower_head)) {
2171                         /* ^^^ Ensure CB invocation follows _head test. */
2172                         return;
2173                 }
2174                 if (!rcu_nocb_poll)
2175                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2176                                             "WokeEmpty");
2177                 WARN_ON(signal_pending(current));
2178                 schedule_timeout_interruptible(1);
2179         }
2180 }
2181 
2182 /*
2183  * Per-rcu_data kthread, but only for no-CBs CPUs.  Each kthread invokes
2184  * callbacks queued by the corresponding no-CBs CPU, however, there is
2185  * an optional leader-follower relationship so that the grace-period
2186  * kthreads don't have to do quite so many wakeups.
2187  */
2188 static int rcu_nocb_kthread(void *arg)
2189 {
2190         int c, cl;
2191         struct rcu_head *list;
2192         struct rcu_head *next;
2193         struct rcu_head **tail;
2194         struct rcu_data *rdp = arg;
2195 
2196         /* Each pass through this loop invokes one batch of callbacks */
2197         for (;;) {
2198                 /* Wait for callbacks. */
2199                 if (rdp->nocb_leader == rdp)
2200                         nocb_leader_wait(rdp);
2201                 else
2202                         nocb_follower_wait(rdp);
2203 
2204                 /* Pull the ready-to-invoke callbacks onto local list. */
2205                 list = READ_ONCE(rdp->nocb_follower_head);
2206                 BUG_ON(!list);
2207                 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "WokeNonEmpty");
2208                 WRITE_ONCE(rdp->nocb_follower_head, NULL);
2209                 tail = xchg(&rdp->nocb_follower_tail, &rdp->nocb_follower_head);
2210 
2211                 /* Each pass through the following loop invokes a callback. */
2212                 trace_rcu_batch_start(rdp->rsp->name,
2213                                       atomic_long_read(&rdp->nocb_q_count_lazy),
2214                                       atomic_long_read(&rdp->nocb_q_count), -1);
2215                 c = cl = 0;
2216                 while (list) {
2217                         next = list->next;
2218                         /* Wait for enqueuing to complete, if needed. */
2219                         while (next == NULL && &list->next != tail) {
2220                                 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2221                                                     TPS("WaitQueue"));
2222                                 schedule_timeout_interruptible(1);
2223                                 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2224                                                     TPS("WokeQueue"));
2225                                 next = list->next;
2226                         }
2227                         debug_rcu_head_unqueue(list);
2228                         local_bh_disable();
2229                         if (__rcu_reclaim(rdp->rsp->name, list))
2230                                 cl++;
2231                         c++;
2232                         local_bh_enable();
2233                         list = next;
2234                 }
2235                 trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1);
2236                 smp_mb__before_atomic();  /* _add after CB invocation. */
2237                 atomic_long_add(-c, &rdp->nocb_q_count);
2238                 atomic_long_add(-cl, &rdp->nocb_q_count_lazy);
2239                 rdp->n_nocbs_invoked += c;
2240         }
2241         return 0;
2242 }
2243 
2244 /* Is a deferred wakeup of rcu_nocb_kthread() required? */
2245 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
2246 {
2247         return READ_ONCE(rdp->nocb_defer_wakeup);
2248 }
2249 
2250 /* Do a deferred wakeup of rcu_nocb_kthread(). */
2251 static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
2252 {
2253         int ndw;
2254 
2255         if (!rcu_nocb_need_deferred_wakeup(rdp))
2256                 return;
2257         ndw = READ_ONCE(rdp->nocb_defer_wakeup);
2258         WRITE_ONCE(rdp->nocb_defer_wakeup, RCU_NOGP_WAKE_NOT);
2259         wake_nocb_leader(rdp, ndw == RCU_NOGP_WAKE_FORCE);
2260         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("DeferredWake"));
2261 }
2262 
2263 void __init rcu_init_nohz(void)
2264 {
2265         int cpu;
2266         bool need_rcu_nocb_mask = true;
2267         struct rcu_state *rsp;
2268 
2269 #ifdef CONFIG_RCU_NOCB_CPU_NONE
2270         need_rcu_nocb_mask = false;
2271 #endif /* #ifndef CONFIG_RCU_NOCB_CPU_NONE */
2272 
2273 #if defined(CONFIG_NO_HZ_FULL)
2274         if (tick_nohz_full_running && cpumask_weight(tick_nohz_full_mask))
2275                 need_rcu_nocb_mask = true;
2276 #endif /* #if defined(CONFIG_NO_HZ_FULL) */
2277 
2278         if (!have_rcu_nocb_mask && need_rcu_nocb_mask) {
2279                 if (!zalloc_cpumask_var(&rcu_nocb_mask, GFP_KERNEL)) {
2280                         pr_info("rcu_nocb_mask allocation failed, callback offloading disabled.\n");
2281                         return;
2282                 }
2283                 have_rcu_nocb_mask = true;
2284         }
2285         if (!have_rcu_nocb_mask)
2286                 return;
2287 
2288 #ifdef CONFIG_RCU_NOCB_CPU_ZERO
2289         pr_info("\tOffload RCU callbacks from CPU 0\n");
2290         cpumask_set_cpu(0, rcu_nocb_mask);
2291 #endif /* #ifdef CONFIG_RCU_NOCB_CPU_ZERO */
2292 #ifdef CONFIG_RCU_NOCB_CPU_ALL
2293         pr_info("\tOffload RCU callbacks from all CPUs\n");
2294         cpumask_copy(rcu_nocb_mask, cpu_possible_mask);
2295 #endif /* #ifdef CONFIG_RCU_NOCB_CPU_ALL */
2296 #if defined(CONFIG_NO_HZ_FULL)
2297         if (tick_nohz_full_running)
2298                 cpumask_or(rcu_nocb_mask, rcu_nocb_mask, tick_nohz_full_mask);
2299 #endif /* #if defined(CONFIG_NO_HZ_FULL) */
2300 
2301         if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) {
2302                 pr_info("\tNote: kernel parameter 'rcu_nocbs=' contains nonexistent CPUs.\n");
2303                 cpumask_and(rcu_nocb_mask, cpu_possible_mask,
2304                             rcu_nocb_mask);
2305         }
2306         pr_info("\tOffload RCU callbacks from CPUs: %*pbl.\n",
2307                 cpumask_pr_args(rcu_nocb_mask));
2308         if (rcu_nocb_poll)
2309                 pr_info("\tPoll for callbacks from no-CBs CPUs.\n");
2310 
2311         for_each_rcu_flavor(rsp) {
2312                 for_each_cpu(cpu, rcu_nocb_mask)
2313                         init_nocb_callback_list(per_cpu_ptr(rsp->rda, cpu));
2314                 rcu_organize_nocb_kthreads(rsp);
2315         }
2316 }
2317 
2318 /* Initialize per-rcu_data variables for no-CBs CPUs. */
2319 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
2320 {
2321         rdp->nocb_tail = &rdp->nocb_head;
2322         init_waitqueue_head(&rdp->nocb_wq);
2323         rdp->nocb_follower_tail = &rdp->nocb_follower_head;
2324 }
2325 
2326 /*
2327  * If the specified CPU is a no-CBs CPU that does not already have its
2328  * rcuo kthread for the specified RCU flavor, spawn it.  If the CPUs are
2329  * brought online out of order, this can require re-organizing the
2330  * leader-follower relationships.
2331  */
2332 static void rcu_spawn_one_nocb_kthread(struct rcu_state *rsp, int cpu)
2333 {
2334         struct rcu_data *rdp;
2335         struct rcu_data *rdp_last;
2336         struct rcu_data *rdp_old_leader;
2337         struct rcu_data *rdp_spawn = per_cpu_ptr(rsp->rda, cpu);
2338         struct task_struct *t;
2339 
2340         /*
2341          * If this isn't a no-CBs CPU or if it already has an rcuo kthread,
2342          * then nothing to do.
2343          */
2344         if (!rcu_is_nocb_cpu(cpu) || rdp_spawn->nocb_kthread)
2345                 return;
2346 
2347         /* If we didn't spawn the leader first, reorganize! */
2348         rdp_old_leader = rdp_spawn->nocb_leader;
2349         if (rdp_old_leader != rdp_spawn && !rdp_old_leader->nocb_kthread) {
2350                 rdp_last = NULL;
2351                 rdp = rdp_old_leader;
2352                 do {
2353                         rdp->nocb_leader = rdp_spawn;
2354                         if (rdp_last && rdp != rdp_spawn)
2355                                 rdp_last->nocb_next_follower = rdp;
2356                         if (rdp == rdp_spawn) {
2357                                 rdp = rdp->nocb_next_follower;
2358                         } else {
2359                                 rdp_last = rdp;
2360                                 rdp = rdp->nocb_next_follower;
2361                                 rdp_last->nocb_next_follower = NULL;
2362                         }
2363                 } while (rdp);
2364                 rdp_spawn->nocb_next_follower = rdp_old_leader;
2365         }
2366 
2367         /* Spawn the kthread for this CPU and RCU flavor. */
2368         t = kthread_run(rcu_nocb_kthread, rdp_spawn,
2369                         "rcuo%c/%d", rsp->abbr, cpu);
2370         BUG_ON(IS_ERR(t));
2371         WRITE_ONCE(rdp_spawn->nocb_kthread, t);
2372 }
2373 
2374 /*
2375  * If the specified CPU is a no-CBs CPU that does not already have its
2376  * rcuo kthreads, spawn them.
2377  */
2378 static void rcu_spawn_all_nocb_kthreads(int cpu)
2379 {
2380         struct rcu_state *rsp;
2381 
2382         if (rcu_scheduler_fully_active)
2383                 for_each_rcu_flavor(rsp)
2384                         rcu_spawn_one_nocb_kthread(rsp, cpu);
2385 }
2386 
2387 /*
2388  * Once the scheduler is running, spawn rcuo kthreads for all online
2389  * no-CBs CPUs.  This assumes that the early_initcall()s happen before
2390  * non-boot CPUs come online -- if this changes, we will need to add
2391  * some mutual exclusion.
2392  */
2393 static void __init rcu_spawn_nocb_kthreads(void)
2394 {
2395         int cpu;
2396 
2397         for_each_online_cpu(cpu)
2398                 rcu_spawn_all_nocb_kthreads(cpu);
2399 }
2400 
2401 /* How many follower CPU IDs per leader?  Default of -1 for sqrt(nr_cpu_ids). */
2402 static int rcu_nocb_leader_stride = -1;
2403 module_param(rcu_nocb_leader_stride, int, 0444);
2404 
2405 /*
2406  * Initialize leader-follower relationships for all no-CBs CPU.
2407  */
2408 static void __init rcu_organize_nocb_kthreads(struct rcu_state *rsp)
2409 {
2410         int cpu;
2411         int ls = rcu_nocb_leader_stride;
2412         int nl = 0;  /* Next leader. */
2413         struct rcu_data *rdp;
2414         struct rcu_data *rdp_leader = NULL;  /* Suppress misguided gcc warn. */
2415         struct rcu_data *rdp_prev = NULL;
2416 
2417         if (!have_rcu_nocb_mask)
2418                 return;
2419         if (ls == -1) {
2420                 ls = int_sqrt(nr_cpu_ids);
2421                 rcu_nocb_leader_stride = ls;
2422         }
2423 
2424         /*
2425          * Each pass through this loop sets up one rcu_data structure and
2426          * spawns one rcu_nocb_kthread().
2427          */
2428         for_each_cpu(cpu, rcu_nocb_mask) {
2429                 rdp = per_cpu_ptr(rsp->rda, cpu);
2430                 if (rdp->cpu >= nl) {
2431                         /* New leader, set up for followers & next leader. */
2432                         nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls;
2433                         rdp->nocb_leader = rdp;
2434                         rdp_leader = rdp;
2435                 } else {
2436                         /* Another follower, link to previous leader. */
2437                         rdp->nocb_leader = rdp_leader;
2438                         rdp_prev->nocb_next_follower = rdp;
2439                 }
2440                 rdp_prev = rdp;
2441         }
2442 }
2443 
2444 /* Prevent __call_rcu() from enqueuing callbacks on no-CBs CPUs */
2445 static bool init_nocb_callback_list(struct rcu_data *rdp)
2446 {
2447         if (!rcu_is_nocb_cpu(rdp->cpu))
2448                 return false;
2449 
2450         /* If there are early-boot callbacks, move them to nocb lists. */
2451         if (rdp->nxtlist) {
2452                 rdp->nocb_head = rdp->nxtlist;
2453                 rdp->nocb_tail = rdp->nxttail[RCU_NEXT_TAIL];
2454                 atomic_long_set(&rdp->nocb_q_count, rdp->qlen);
2455                 atomic_long_set(&rdp->nocb_q_count_lazy, rdp->qlen_lazy);
2456                 rdp->nxtlist = NULL;
2457                 rdp->qlen = 0;
2458                 rdp->qlen_lazy = 0;
2459         }
2460         rdp->nxttail[RCU_NEXT_TAIL] = NULL;
2461         return true;
2462 }
2463 
2464 #else /* #ifdef CONFIG_RCU_NOCB_CPU */
2465 
2466 static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)
2467 {
2468         WARN_ON_ONCE(1); /* Should be dead code. */
2469         return false;
2470 }
2471 
2472 static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)
2473 {
2474 }
2475 
2476 static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)
2477 {
2478 }
2479 
2480 static void rcu_init_one_nocb(struct rcu_node *rnp)
2481 {
2482 }
2483 
2484 static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
2485                             bool lazy, unsigned long flags)
2486 {
2487         return false;
2488 }
2489 
2490 static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
2491                                                      struct rcu_data *rdp,
2492                                                      unsigned long flags)
2493 {
2494         return false;
2495 }
2496 
2497 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
2498 {
2499 }
2500 
2501 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
2502 {
2503         return false;
2504 }
2505 
2506 static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
2507 {
2508 }
2509 
2510 static void rcu_spawn_all_nocb_kthreads(int cpu)
2511 {
2512 }
2513 
2514 static void __init rcu_spawn_nocb_kthreads(void)
2515 {
2516 }
2517 
2518 static bool init_nocb_callback_list(struct rcu_data *rdp)
2519 {
2520         return false;
2521 }
2522 
2523 #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
2524 
2525 /*
2526  * An adaptive-ticks CPU can potentially execute in kernel mode for an
2527  * arbitrarily long period of time with the scheduling-clock tick turned
2528  * off.  RCU will be paying attention to this CPU because it is in the
2529  * kernel, but the CPU cannot be guaranteed to be executing the RCU state
2530  * machine because the scheduling-clock tick has been disabled.  Therefore,
2531  * if an adaptive-ticks CPU is failing to respond to the current grace
2532  * period and has not be idle from an RCU perspective, kick it.
2533  */
2534 static void __maybe_unused rcu_kick_nohz_cpu(int cpu)
2535 {
2536 #ifdef CONFIG_NO_HZ_FULL
2537         if (tick_nohz_full_cpu(cpu))
2538                 smp_send_reschedule(cpu);
2539 #endif /* #ifdef CONFIG_NO_HZ_FULL */
2540 }
2541 
2542 
2543 #ifdef CONFIG_NO_HZ_FULL_SYSIDLE
2544 
2545 static int full_sysidle_state;          /* Current system-idle state. */
2546 #define RCU_SYSIDLE_NOT         0       /* Some CPU is not idle. */
2547 #define RCU_SYSIDLE_SHORT       1       /* All CPUs idle for brief period. */
2548 #define RCU_SYSIDLE_LONG        2       /* All CPUs idle for long enough. */
2549 #define RCU_SYSIDLE_FULL        3       /* All CPUs idle, ready for sysidle. */
2550 #define RCU_SYSIDLE_FULL_NOTED  4       /* Actually entered sysidle state. */
2551 
2552 /*
2553  * Invoked to note exit from irq or task transition to idle.  Note that
2554  * usermode execution does -not- count as idle here!  After all, we want
2555  * to detect full-system idle states, not RCU quiescent states and grace
2556  * periods.  The caller must have disabled interrupts.
2557  */
2558 static void rcu_sysidle_enter(int irq)
2559 {
2560         unsigned long j;
2561         struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
2562 
2563         /* If there are no nohz_full= CPUs, no need to track this. */
2564         if (!tick_nohz_full_enabled())
2565                 return;
2566 
2567         /* Adjust nesting, check for fully idle. */
2568         if (irq) {
2569                 rdtp->dynticks_idle_nesting--;
2570                 WARN_ON_ONCE(rdtp->dynticks_idle_nesting < 0);
2571                 if (rdtp->dynticks_idle_nesting != 0)
2572                         return;  /* Still not fully idle. */
2573         } else {
2574                 if ((rdtp->dynticks_idle_nesting & DYNTICK_TASK_NEST_MASK) ==
2575                     DYNTICK_TASK_NEST_VALUE) {
2576                         rdtp->dynticks_idle_nesting = 0;
2577                 } else {
2578                         rdtp->dynticks_idle_nesting -= DYNTICK_TASK_NEST_VALUE;
2579                         WARN_ON_ONCE(rdtp->dynticks_idle_nesting < 0);
2580                         return;  /* Still not fully idle. */
2581                 }
2582         }
2583 
2584         /* Record start of fully idle period. */
2585         j = jiffies;
2586         WRITE_ONCE(rdtp->dynticks_idle_jiffies, j);
2587         smp_mb__before_atomic();
2588         atomic_inc(&rdtp->dynticks_idle);
2589         smp_mb__after_atomic();
2590         WARN_ON_ONCE(atomic_read(&rdtp->dynticks_idle) & 0x1);
2591 }
2592 
2593 /*
2594  * Unconditionally force exit from full system-idle state.  This is
2595  * invoked when a normal CPU exits idle, but must be called separately
2596  * for the timekeeping CPU (tick_do_timer_cpu).  The reason for this
2597  * is that the timekeeping CPU is permitted to take scheduling-clock
2598  * interrupts while the system is in system-idle state, and of course
2599  * rcu_sysidle_exit() has no way of distinguishing a scheduling-clock
2600  * interrupt from any other type of interrupt.
2601  */
2602 void rcu_sysidle_force_exit(void)
2603 {
2604         int oldstate = READ_ONCE(full_sysidle_state);
2605         int newoldstate;
2606 
2607         /*
2608          * Each pass through the following loop attempts to exit full
2609          * system-idle state.  If contention proves to be a problem,
2610          * a trylock-based contention tree could be used here.
2611          */
2612         while (oldstate > RCU_SYSIDLE_SHORT) {
2613                 newoldstate = cmpxchg(&full_sysidle_state,
2614                                       oldstate, RCU_SYSIDLE_NOT);
2615                 if (oldstate == newoldstate &&
2616                     oldstate == RCU_SYSIDLE_FULL_NOTED) {
2617                         rcu_kick_nohz_cpu(tick_do_timer_cpu);
2618                         return; /* We cleared it, done! */
2619                 }
2620                 oldstate = newoldstate;
2621         }
2622         smp_mb(); /* Order initial oldstate fetch vs. later non-idle work. */
2623 }
2624 
2625 /*
2626  * Invoked to note entry to irq or task transition from idle.  Note that
2627  * usermode execution does -not- count as idle here!  The caller must
2628  * have disabled interrupts.
2629  */
2630 static void rcu_sysidle_exit(int irq)
2631 {
2632         struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
2633 
2634         /* If there are no nohz_full= CPUs, no need to track this. */
2635         if (!tick_nohz_full_enabled())
2636                 return;
2637 
2638         /* Adjust nesting, check for already non-idle. */
2639         if (irq) {
2640                 rdtp->dynticks_idle_nesting++;
2641                 WARN_ON_ONCE(rdtp->dynticks_idle_nesting <= 0);
2642                 if (rdtp->dynticks_idle_nesting != 1)
2643                         return; /* Already non-idle. */
2644         } else {
2645                 /*
2646                  * Allow for irq misnesting.  Yes, it really is possible
2647                  * to enter an irq handler then never leave it, and maybe
2648                  * also vice versa.  Handle both possibilities.
2649                  */
2650                 if (rdtp->dynticks_idle_nesting & DYNTICK_TASK_NEST_MASK) {
2651                         rdtp->dynticks_idle_nesting += DYNTICK_TASK_NEST_VALUE;
2652                         WARN_ON_ONCE(rdtp->dynticks_idle_nesting <= 0);
2653                         return; /* Already non-idle. */
2654                 } else {
2655                         rdtp->dynticks_idle_nesting = DYNTICK_TASK_EXIT_IDLE;
2656                 }
2657         }
2658 
2659         /* Record end of idle period. */
2660         smp_mb__before_atomic();
2661         atomic_inc(&rdtp->dynticks_idle);
2662         smp_mb__after_atomic();
2663         WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks_idle) & 0x1));
2664 
2665         /*
2666          * If we are the timekeeping CPU, we are permitted to be non-idle
2667          * during a system-idle state.  This must be the case, because
2668          * the timekeeping CPU has to take scheduling-clock interrupts
2669          * during the time that the system is transitioning to full
2670          * system-idle state.  This means that the timekeeping CPU must
2671          * invoke rcu_sysidle_force_exit() directly if it does anything
2672          * more than take a scheduling-clock interrupt.
2673          */
2674         if (smp_processor_id() == tick_do_timer_cpu)
2675                 return;
2676 
2677         /* Update system-idle state: We are clearly no longer fully idle! */
2678         rcu_sysidle_force_exit();
2679 }
2680 
2681 /*
2682  * Check to see if the current CPU is idle.  Note that usermode execution
2683  * does not count as idle.  The caller must have disabled interrupts,
2684  * and must be running on tick_do_timer_cpu.
2685  */
2686 static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
2687                                   unsigned long *maxj)
2688 {
2689         int cur;
2690         unsigned long j;
2691         struct rcu_dynticks *rdtp = rdp->dynticks;
2692 
2693         /* If there are no nohz_full= CPUs, don't check system-wide idleness. */
2694         if (!tick_nohz_full_enabled())
2695                 return;
2696 
2697         /*
2698          * If some other CPU has already reported non-idle, if this is
2699          * not the flavor of RCU that tracks sysidle state, or if this
2700          * is an offline or the timekeeping CPU, nothing to do.
2701          */
2702         if (!*isidle || rdp->rsp != rcu_state_p ||
2703             cpu_is_offline(rdp->cpu) || rdp->cpu == tick_do_timer_cpu)
2704                 return;
2705         /* Verify affinity of current kthread. */
2706         WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu);
2707 
2708         /* Pick up current idle and NMI-nesting counter and check. */
2709         cur = atomic_read(&rdtp->dynticks_idle);
2710         if (cur & 0x1) {
2711                 *isidle = false; /* We are not idle! */
2712                 return;
2713         }
2714         smp_mb(); /* Read counters before timestamps. */
2715 
2716         /* Pick up timestamps. */
2717         j = READ_ONCE(rdtp->dynticks_idle_jiffies);
2718         /* If this CPU entered idle more recently, update maxj timestamp. */
2719         if (ULONG_CMP_LT(*maxj, j))
2720                 *maxj = j;
2721 }
2722 
2723 /*
2724  * Is this the flavor of RCU that is handling full-system idle?
2725  */
2726 static bool is_sysidle_rcu_state(struct rcu_state *rsp)
2727 {
2728         return rsp == rcu_state_p;
2729 }
2730 
2731 /*
2732  * Return a delay in jiffies based on the number of CPUs, rcu_node
2733  * leaf fanout, and jiffies tick rate.  The idea is to allow larger
2734  * systems more time to transition to full-idle state in order to
2735  * avoid the cache thrashing that otherwise occur on the state variable.
2736  * Really small systems (less than a couple of tens of CPUs) should
2737  * instead use a single global atomically incremented counter, and later
2738  * versions of this will automatically reconfigure themselves accordingly.
2739  */
2740 static unsigned long rcu_sysidle_delay(void)
2741 {
2742         if (nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL)
2743                 return 0;
2744         return DIV_ROUND_UP(nr_cpu_ids * HZ, rcu_fanout_leaf * 1000);
2745 }
2746 
2747 /*
2748  * Advance the full-system-idle state.  This is invoked when all of
2749  * the non-timekeeping CPUs are idle.
2750  */
2751 static void rcu_sysidle(unsigned long j)
2752 {
2753         /* Check the current state. */
2754         switch (READ_ONCE(full_sysidle_state)) {
2755         case RCU_SYSIDLE_NOT:
2756 
2757                 /* First time all are idle, so note a short idle period. */
2758                 WRITE_ONCE(full_sysidle_state, RCU_SYSIDLE_SHORT);
2759                 break;
2760 
2761         case RCU_SYSIDLE_SHORT:
2762 
2763                 /*
2764                  * Idle for a bit, time to advance to next state?
2765                  * cmpxchg failure means race with non-idle, let them win.
2766                  */
2767                 if (ULONG_CMP_GE(jiffies, j + rcu_sysidle_delay()))
2768                         (void)cmpxchg(&full_sysidle_state,
2769                                       RCU_SYSIDLE_SHORT, RCU_SYSIDLE_LONG);
2770                 break;
2771 
2772         case RCU_SYSIDLE_LONG:
2773 
2774                 /*
2775                  * Do an additional check pass before advancing to full.
2776                  * cmpxchg failure means race with non-idle, let them win.
2777                  */
2778                 if (ULONG_CMP_GE(jiffies, j + rcu_sysidle_delay()))
2779                         (void)cmpxchg(&full_sysidle_state,
2780                                       RCU_SYSIDLE_LONG, RCU_SYSIDLE_FULL);
2781                 break;
2782 
2783         default:
2784                 break;
2785         }
2786 }
2787 
2788 /*
2789  * Found a non-idle non-timekeeping CPU, so kick the system-idle state
2790  * back to the beginning.
2791  */
2792 static void rcu_sysidle_cancel(void)
2793 {
2794         smp_mb();
2795         if (full_sysidle_state > RCU_SYSIDLE_SHORT)
2796                 WRITE_ONCE(full_sysidle_state, RCU_SYSIDLE_NOT);
2797 }
2798 
2799 /*
2800  * Update the sysidle state based on the results of a force-quiescent-state
2801  * scan of the CPUs' dyntick-idle state.
2802  */
2803 static void rcu_sysidle_report(struct rcu_state *rsp, int isidle,
2804                                unsigned long maxj, bool gpkt)
2805 {
2806         if (rsp != rcu_state_p)
2807                 return;  /* Wrong flavor, ignore. */
2808         if (gpkt && nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL)
2809                 return;  /* Running state machine from timekeeping CPU. */
2810         if (isidle)
2811                 rcu_sysidle(maxj);    /* More idle! */
2812         else
2813                 rcu_sysidle_cancel(); /* Idle is over. */
2814 }
2815 
2816 /*
2817  * Wrapper for rcu_sysidle_report() when called from the grace-period
2818  * kthread's context.
2819  */
2820 static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
2821                                   unsigned long maxj)
2822 {
2823         /* If there are no nohz_full= CPUs, no need to track this. */
2824         if (!tick_nohz_full_enabled())
2825                 return;
2826 
2827         rcu_sysidle_report(rsp, isidle, maxj, true);
2828 }
2829 
2830 /* Callback and function for forcing an RCU grace period. */
2831 struct rcu_sysidle_head {
2832         struct rcu_head rh;
2833         int inuse;
2834 };
2835 
2836 static void rcu_sysidle_cb(struct rcu_head *rhp)
2837 {
2838         struct rcu_sysidle_head *rshp;
2839 
2840         /*
2841          * The following memory barrier is needed to replace the
2842          * memory barriers that would normally be in the memory
2843          * allocator.
2844          */
2845         smp_mb();  /* grace period precedes setting inuse. */
2846 
2847         rshp = container_of(rhp, struct rcu_sysidle_head, rh);
2848         WRITE_ONCE(rshp->inuse, 0);
2849 }
2850 
2851 /*
2852  * Check to see if the system is fully idle, other than the timekeeping CPU.
2853  * The caller must have disabled interrupts.  This is not intended to be
2854  * called unless tick_nohz_full_enabled().
2855  */
2856 bool rcu_sys_is_idle(void)
2857 {
2858         static struct rcu_sysidle_head rsh;
2859         int rss = READ_ONCE(full_sysidle_state);
2860 
2861         if (WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu))
2862                 return false;
2863 
2864         /* Handle small-system case by doing a full scan of CPUs. */
2865         if (nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL) {
2866                 int oldrss = rss - 1;
2867 
2868                 /*
2869                  * One pass to advance to each state up to _FULL.
2870                  * Give up if any pass fails to advance the state.
2871                  */
2872                 while (rss < RCU_SYSIDLE_FULL && oldrss < rss) {
2873                         int cpu;
2874                         bool isidle = true;
2875                         unsigned long maxj = jiffies - ULONG_MAX / 4;
2876                         struct rcu_data *rdp;
2877 
2878                         /* Scan all the CPUs looking for nonidle CPUs. */
2879                         for_each_possible_cpu(cpu) {
2880                                 rdp = per_cpu_ptr(rcu_state_p->rda, cpu);
2881                                 rcu_sysidle_check_cpu(rdp, &isidle, &maxj);
2882                                 if (!isidle)
2883                                         break;
2884                         }
2885                         rcu_sysidle_report(rcu_state_p, isidle, maxj, false);
2886                         oldrss = rss;
2887                         rss = READ_ONCE(full_sysidle_state);
2888                 }
2889         }
2890 
2891         /* If this is the first observation of an idle period, record it. */
2892         if (rss == RCU_SYSIDLE_FULL) {
2893                 rss = cmpxchg(&full_sysidle_state,
2894                               RCU_SYSIDLE_FULL, RCU_SYSIDLE_FULL_NOTED);
2895                 return rss == RCU_SYSIDLE_FULL;
2896         }
2897 
2898         smp_mb(); /* ensure rss load happens before later caller actions. */
2899 
2900         /* If already fully idle, tell the caller (in case of races). */
2901         if (rss == RCU_SYSIDLE_FULL_NOTED)
2902                 return true;
2903 
2904         /*
2905          * If we aren't there yet, and a grace period is not in flight,
2906          * initiate a grace period.  Either way, tell the caller that
2907          * we are not there yet.  We use an xchg() rather than an assignment
2908          * to make up for the memory barriers that would otherwise be
2909          * provided by the memory allocator.
2910          */
2911         if (nr_cpu_ids > CONFIG_NO_HZ_FULL_SYSIDLE_SMALL &&
2912             !rcu_gp_in_progress(rcu_state_p) &&
2913             !rsh.inuse && xchg(&rsh.inuse, 1) == 0)
2914                 call_rcu(&rsh.rh, rcu_sysidle_cb);
2915         return false;
2916 }
2917 
2918 /*
2919  * Initialize dynticks sysidle state for CPUs coming online.
2920  */
2921 static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp)
2922 {
2923         rdtp->dynticks_idle_nesting = DYNTICK_TASK_NEST_VALUE;
2924 }
2925 
2926 #else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
2927 
2928 static void rcu_sysidle_enter(int irq)
2929 {
2930 }
2931 
2932 static void rcu_sysidle_exit(int irq)
2933 {
2934 }
2935 
2936 static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
2937                                   unsigned long *maxj)
2938 {
2939 }
2940 
2941 static bool is_sysidle_rcu_state(struct rcu_state *rsp)
2942 {
2943         return false;
2944 }
2945 
2946 static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
2947                                   unsigned long maxj)
2948 {
2949 }
2950 
2951 static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp)
2952 {
2953 }
2954 
2955 #endif /* #else #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
2956 
2957 /*
2958  * Is this CPU a NO_HZ_FULL CPU that should ignore RCU so that the
2959  * grace-period kthread will do force_quiescent_state() processing?
2960  * The idea is to avoid waking up RCU core processing on such a
2961  * CPU unless the grace period has extended for too long.
2962  *
2963  * This code relies on the fact that all NO_HZ_FULL CPUs are also
2964  * CONFIG_RCU_NOCB_CPU CPUs.
2965  */
2966 static bool rcu_nohz_full_cpu(struct rcu_state *rsp)
2967 {
2968 #ifdef CONFIG_NO_HZ_FULL
2969         if (tick_nohz_full_cpu(smp_processor_id()) &&
2970             (!rcu_gp_in_progress(rsp) ||
2971              ULONG_CMP_LT(jiffies, READ_ONCE(rsp->gp_start) + HZ)))
2972                 return true;
2973 #endif /* #ifdef CONFIG_NO_HZ_FULL */
2974         return false;
2975 }
2976 
2977 /*
2978  * Bind the grace-period kthread for the sysidle flavor of RCU to the
2979  * timekeeping CPU.
2980  */
2981 static void rcu_bind_gp_kthread(void)
2982 {
2983         int __maybe_unused cpu;
2984 
2985         if (!tick_nohz_full_enabled())
2986                 return;
2987 #ifdef CONFIG_NO_HZ_FULL_SYSIDLE
2988         cpu = tick_do_timer_cpu;
2989         if (cpu >= 0 && cpu < nr_cpu_ids)
2990                 set_cpus_allowed_ptr(current, cpumask_of(cpu));
2991 #else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
2992         housekeeping_affine(current);
2993 #endif /* #else #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
2994 }
2995 
2996 /* Record the current task on dyntick-idle entry. */
2997 static void rcu_dynticks_task_enter(void)
2998 {
2999 #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
3000         WRITE_ONCE(current->rcu_tasks_idle_cpu, smp_processor_id());
3001 #endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */
3002 }
3003 
3004 /* Record no current task on dyntick-idle exit. */
3005 static void rcu_dynticks_task_exit(void)
3006 {
3007 #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
3008         WRITE_ONCE(current->rcu_tasks_idle_cpu, -1);
3009 #endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */
3010 }
3011 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp