~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/rcu/tree_plugin.h

Version: ~ [ linux-5.16 ] ~ [ linux-5.15.13 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.90 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.170 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.224 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.261 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.296 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.298 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Read-Copy Update mechanism for mutual exclusion (tree-based version)
  3  * Internal non-public definitions that provide either classic
  4  * or preemptible semantics.
  5  *
  6  * This program is free software; you can redistribute it and/or modify
  7  * it under the terms of the GNU General Public License as published by
  8  * the Free Software Foundation; either version 2 of the License, or
  9  * (at your option) any later version.
 10  *
 11  * This program is distributed in the hope that it will be useful,
 12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 14  * GNU General Public License for more details.
 15  *
 16  * You should have received a copy of the GNU General Public License
 17  * along with this program; if not, you can access it online at
 18  * http://www.gnu.org/licenses/gpl-2.0.html.
 19  *
 20  * Copyright Red Hat, 2009
 21  * Copyright IBM Corporation, 2009
 22  *
 23  * Author: Ingo Molnar <mingo@elte.hu>
 24  *         Paul E. McKenney <paulmck@linux.vnet.ibm.com>
 25  */
 26 
 27 #include <linux/delay.h>
 28 #include <linux/gfp.h>
 29 #include <linux/oom.h>
 30 #include <linux/smpboot.h>
 31 #include "../time/tick-internal.h"
 32 
 33 #ifdef CONFIG_RCU_BOOST
 34 
 35 #include "../locking/rtmutex_common.h"
 36 
 37 /*
 38  * Control variables for per-CPU and per-rcu_node kthreads.  These
 39  * handle all flavors of RCU.
 40  */
 41 static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);
 42 DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
 43 DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
 44 DEFINE_PER_CPU(char, rcu_cpu_has_work);
 45 
 46 #else /* #ifdef CONFIG_RCU_BOOST */
 47 
 48 /*
 49  * Some architectures do not define rt_mutexes, but if !CONFIG_RCU_BOOST,
 50  * all uses are in dead code.  Provide a definition to keep the compiler
 51  * happy, but add WARN_ON_ONCE() to complain if used in the wrong place.
 52  * This probably needs to be excluded from -rt builds.
 53  */
 54 #define rt_mutex_owner(a) ({ WARN_ON_ONCE(1); NULL; })
 55 
 56 #endif /* #else #ifdef CONFIG_RCU_BOOST */
 57 
 58 #ifdef CONFIG_RCU_NOCB_CPU
 59 static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */
 60 static bool have_rcu_nocb_mask;     /* Was rcu_nocb_mask allocated? */
 61 static bool __read_mostly rcu_nocb_poll;    /* Offload kthread are to poll. */
 62 #endif /* #ifdef CONFIG_RCU_NOCB_CPU */
 63 
 64 /*
 65  * Check the RCU kernel configuration parameters and print informative
 66  * messages about anything out of the ordinary.
 67  */
 68 static void __init rcu_bootup_announce_oddness(void)
 69 {
 70         if (IS_ENABLED(CONFIG_RCU_TRACE))
 71                 pr_info("\tRCU debugfs-based tracing is enabled.\n");
 72         if ((IS_ENABLED(CONFIG_64BIT) && RCU_FANOUT != 64) ||
 73             (!IS_ENABLED(CONFIG_64BIT) && RCU_FANOUT != 32))
 74                 pr_info("\tCONFIG_RCU_FANOUT set to non-default value of %d\n",
 75                        RCU_FANOUT);
 76         if (rcu_fanout_exact)
 77                 pr_info("\tHierarchical RCU autobalancing is disabled.\n");
 78         if (IS_ENABLED(CONFIG_RCU_FAST_NO_HZ))
 79                 pr_info("\tRCU dyntick-idle grace-period acceleration is enabled.\n");
 80         if (IS_ENABLED(CONFIG_PROVE_RCU))
 81                 pr_info("\tRCU lockdep checking is enabled.\n");
 82         if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_RUNNABLE))
 83                 pr_info("\tRCU torture testing starts during boot.\n");
 84         if (RCU_NUM_LVLS >= 4)
 85                 pr_info("\tFour(or more)-level hierarchy is enabled.\n");
 86         if (RCU_FANOUT_LEAF != 16)
 87                 pr_info("\tBuild-time adjustment of leaf fanout to %d.\n",
 88                         RCU_FANOUT_LEAF);
 89         if (rcu_fanout_leaf != RCU_FANOUT_LEAF)
 90                 pr_info("\tBoot-time adjustment of leaf fanout to %d.\n", rcu_fanout_leaf);
 91         if (nr_cpu_ids != NR_CPUS)
 92                 pr_info("\tRCU restricting CPUs from NR_CPUS=%d to nr_cpu_ids=%d.\n", NR_CPUS, nr_cpu_ids);
 93         if (IS_ENABLED(CONFIG_RCU_BOOST))
 94                 pr_info("\tRCU kthread priority: %d.\n", kthread_prio);
 95 }
 96 
 97 #ifdef CONFIG_PREEMPT_RCU
 98 
 99 RCU_STATE_INITIALIZER(rcu_preempt, 'p', call_rcu);
100 static struct rcu_state *const rcu_state_p = &rcu_preempt_state;
101 static struct rcu_data __percpu *const rcu_data_p = &rcu_preempt_data;
102 
103 static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
104                                bool wake);
105 
106 /*
107  * Tell them what RCU they are running.
108  */
109 static void __init rcu_bootup_announce(void)
110 {
111         pr_info("Preemptible hierarchical RCU implementation.\n");
112         rcu_bootup_announce_oddness();
113 }
114 
115 /* Flags for rcu_preempt_ctxt_queue() decision table. */
116 #define RCU_GP_TASKS    0x8
117 #define RCU_EXP_TASKS   0x4
118 #define RCU_GP_BLKD     0x2
119 #define RCU_EXP_BLKD    0x1
120 
121 /*
122  * Queues a task preempted within an RCU-preempt read-side critical
123  * section into the appropriate location within the ->blkd_tasks list,
124  * depending on the states of any ongoing normal and expedited grace
125  * periods.  The ->gp_tasks pointer indicates which element the normal
126  * grace period is waiting on (NULL if none), and the ->exp_tasks pointer
127  * indicates which element the expedited grace period is waiting on (again,
128  * NULL if none).  If a grace period is waiting on a given element in the
129  * ->blkd_tasks list, it also waits on all subsequent elements.  Thus,
130  * adding a task to the tail of the list blocks any grace period that is
131  * already waiting on one of the elements.  In contrast, adding a task
132  * to the head of the list won't block any grace period that is already
133  * waiting on one of the elements.
134  *
135  * This queuing is imprecise, and can sometimes make an ongoing grace
136  * period wait for a task that is not strictly speaking blocking it.
137  * Given the choice, we needlessly block a normal grace period rather than
138  * blocking an expedited grace period.
139  *
140  * Note that an endless sequence of expedited grace periods still cannot
141  * indefinitely postpone a normal grace period.  Eventually, all of the
142  * fixed number of preempted tasks blocking the normal grace period that are
143  * not also blocking the expedited grace period will resume and complete
144  * their RCU read-side critical sections.  At that point, the ->gp_tasks
145  * pointer will equal the ->exp_tasks pointer, at which point the end of
146  * the corresponding expedited grace period will also be the end of the
147  * normal grace period.
148  */
149 static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp)
150         __releases(rnp->lock) /* But leaves rrupts disabled. */
151 {
152         int blkd_state = (rnp->gp_tasks ? RCU_GP_TASKS : 0) +
153                          (rnp->exp_tasks ? RCU_EXP_TASKS : 0) +
154                          (rnp->qsmask & rdp->grpmask ? RCU_GP_BLKD : 0) +
155                          (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0);
156         struct task_struct *t = current;
157 
158         /*
159          * Decide where to queue the newly blocked task.  In theory,
160          * this could be an if-statement.  In practice, when I tried
161          * that, it was quite messy.
162          */
163         switch (blkd_state) {
164         case 0:
165         case                RCU_EXP_TASKS:
166         case                RCU_EXP_TASKS + RCU_GP_BLKD:
167         case RCU_GP_TASKS:
168         case RCU_GP_TASKS + RCU_EXP_TASKS:
169 
170                 /*
171                  * Blocking neither GP, or first task blocking the normal
172                  * GP but not blocking the already-waiting expedited GP.
173                  * Queue at the head of the list to avoid unnecessarily
174                  * blocking the already-waiting GPs.
175                  */
176                 list_add(&t->rcu_node_entry, &rnp->blkd_tasks);
177                 break;
178 
179         case                                              RCU_EXP_BLKD:
180         case                                RCU_GP_BLKD:
181         case                                RCU_GP_BLKD + RCU_EXP_BLKD:
182         case RCU_GP_TASKS +                               RCU_EXP_BLKD:
183         case RCU_GP_TASKS +                 RCU_GP_BLKD + RCU_EXP_BLKD:
184         case RCU_GP_TASKS + RCU_EXP_TASKS + RCU_GP_BLKD + RCU_EXP_BLKD:
185 
186                 /*
187                  * First task arriving that blocks either GP, or first task
188                  * arriving that blocks the expedited GP (with the normal
189                  * GP already waiting), or a task arriving that blocks
190                  * both GPs with both GPs already waiting.  Queue at the
191                  * tail of the list to avoid any GP waiting on any of the
192                  * already queued tasks that are not blocking it.
193                  */
194                 list_add_tail(&t->rcu_node_entry, &rnp->blkd_tasks);
195                 break;
196 
197         case                RCU_EXP_TASKS +               RCU_EXP_BLKD:
198         case                RCU_EXP_TASKS + RCU_GP_BLKD + RCU_EXP_BLKD:
199         case RCU_GP_TASKS + RCU_EXP_TASKS +               RCU_EXP_BLKD:
200 
201                 /*
202                  * Second or subsequent task blocking the expedited GP.
203                  * The task either does not block the normal GP, or is the
204                  * first task blocking the normal GP.  Queue just after
205                  * the first task blocking the expedited GP.
206                  */
207                 list_add(&t->rcu_node_entry, rnp->exp_tasks);
208                 break;
209 
210         case RCU_GP_TASKS +                 RCU_GP_BLKD:
211         case RCU_GP_TASKS + RCU_EXP_TASKS + RCU_GP_BLKD:
212 
213                 /*
214                  * Second or subsequent task blocking the normal GP.
215                  * The task does not block the expedited GP. Queue just
216                  * after the first task blocking the normal GP.
217                  */
218                 list_add(&t->rcu_node_entry, rnp->gp_tasks);
219                 break;
220 
221         default:
222 
223                 /* Yet another exercise in excessive paranoia. */
224                 WARN_ON_ONCE(1);
225                 break;
226         }
227 
228         /*
229          * We have now queued the task.  If it was the first one to
230          * block either grace period, update the ->gp_tasks and/or
231          * ->exp_tasks pointers, respectively, to reference the newly
232          * blocked tasks.
233          */
234         if (!rnp->gp_tasks && (blkd_state & RCU_GP_BLKD))
235                 rnp->gp_tasks = &t->rcu_node_entry;
236         if (!rnp->exp_tasks && (blkd_state & RCU_EXP_BLKD))
237                 rnp->exp_tasks = &t->rcu_node_entry;
238         raw_spin_unlock(&rnp->lock); /* rrupts remain disabled. */
239 
240         /*
241          * Report the quiescent state for the expedited GP.  This expedited
242          * GP should not be able to end until we report, so there should be
243          * no need to check for a subsequent expedited GP.  (Though we are
244          * still in a quiescent state in any case.)
245          */
246         if (blkd_state & RCU_EXP_BLKD &&
247             t->rcu_read_unlock_special.b.exp_need_qs) {
248                 t->rcu_read_unlock_special.b.exp_need_qs = false;
249                 rcu_report_exp_rdp(rdp->rsp, rdp, true);
250         } else {
251                 WARN_ON_ONCE(t->rcu_read_unlock_special.b.exp_need_qs);
252         }
253 }
254 
255 /*
256  * Record a preemptible-RCU quiescent state for the specified CPU.  Note
257  * that this just means that the task currently running on the CPU is
258  * not in a quiescent state.  There might be any number of tasks blocked
259  * while in an RCU read-side critical section.
260  *
261  * As with the other rcu_*_qs() functions, callers to this function
262  * must disable preemption.
263  */
264 static void rcu_preempt_qs(void)
265 {
266         if (__this_cpu_read(rcu_data_p->cpu_no_qs.s)) {
267                 trace_rcu_grace_period(TPS("rcu_preempt"),
268                                        __this_cpu_read(rcu_data_p->gpnum),
269                                        TPS("cpuqs"));
270                 __this_cpu_write(rcu_data_p->cpu_no_qs.b.norm, false);
271                 barrier(); /* Coordinate with rcu_preempt_check_callbacks(). */
272                 current->rcu_read_unlock_special.b.need_qs = false;
273         }
274 }
275 
276 /*
277  * We have entered the scheduler, and the current task might soon be
278  * context-switched away from.  If this task is in an RCU read-side
279  * critical section, we will no longer be able to rely on the CPU to
280  * record that fact, so we enqueue the task on the blkd_tasks list.
281  * The task will dequeue itself when it exits the outermost enclosing
282  * RCU read-side critical section.  Therefore, the current grace period
283  * cannot be permitted to complete until the blkd_tasks list entries
284  * predating the current grace period drain, in other words, until
285  * rnp->gp_tasks becomes NULL.
286  *
287  * Caller must disable interrupts.
288  */
289 static void rcu_preempt_note_context_switch(void)
290 {
291         struct task_struct *t = current;
292         struct rcu_data *rdp;
293         struct rcu_node *rnp;
294 
295         if (t->rcu_read_lock_nesting > 0 &&
296             !t->rcu_read_unlock_special.b.blocked) {
297 
298                 /* Possibly blocking in an RCU read-side critical section. */
299                 rdp = this_cpu_ptr(rcu_state_p->rda);
300                 rnp = rdp->mynode;
301                 raw_spin_lock_rcu_node(rnp);
302                 t->rcu_read_unlock_special.b.blocked = true;
303                 t->rcu_blocked_node = rnp;
304 
305                 /*
306                  * Verify the CPU's sanity, trace the preemption, and
307                  * then queue the task as required based on the states
308                  * of any ongoing and expedited grace periods.
309                  */
310                 WARN_ON_ONCE((rdp->grpmask & rcu_rnp_online_cpus(rnp)) == 0);
311                 WARN_ON_ONCE(!list_empty(&t->rcu_node_entry));
312                 trace_rcu_preempt_task(rdp->rsp->name,
313                                        t->pid,
314                                        (rnp->qsmask & rdp->grpmask)
315                                        ? rnp->gpnum
316                                        : rnp->gpnum + 1);
317                 rcu_preempt_ctxt_queue(rnp, rdp);
318         } else if (t->rcu_read_lock_nesting < 0 &&
319                    t->rcu_read_unlock_special.s) {
320 
321                 /*
322                  * Complete exit from RCU read-side critical section on
323                  * behalf of preempted instance of __rcu_read_unlock().
324                  */
325                 rcu_read_unlock_special(t);
326         }
327 
328         /*
329          * Either we were not in an RCU read-side critical section to
330          * begin with, or we have now recorded that critical section
331          * globally.  Either way, we can now note a quiescent state
332          * for this CPU.  Again, if we were in an RCU read-side critical
333          * section, and if that critical section was blocking the current
334          * grace period, then the fact that the task has been enqueued
335          * means that we continue to block the current grace period.
336          */
337         rcu_preempt_qs();
338 }
339 
340 /*
341  * Check for preempted RCU readers blocking the current grace period
342  * for the specified rcu_node structure.  If the caller needs a reliable
343  * answer, it must hold the rcu_node's ->lock.
344  */
345 static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
346 {
347         return rnp->gp_tasks != NULL;
348 }
349 
350 /*
351  * Advance a ->blkd_tasks-list pointer to the next entry, instead
352  * returning NULL if at the end of the list.
353  */
354 static struct list_head *rcu_next_node_entry(struct task_struct *t,
355                                              struct rcu_node *rnp)
356 {
357         struct list_head *np;
358 
359         np = t->rcu_node_entry.next;
360         if (np == &rnp->blkd_tasks)
361                 np = NULL;
362         return np;
363 }
364 
365 /*
366  * Return true if the specified rcu_node structure has tasks that were
367  * preempted within an RCU read-side critical section.
368  */
369 static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
370 {
371         return !list_empty(&rnp->blkd_tasks);
372 }
373 
374 /*
375  * Handle special cases during rcu_read_unlock(), such as needing to
376  * notify RCU core processing or task having blocked during the RCU
377  * read-side critical section.
378  */
379 void rcu_read_unlock_special(struct task_struct *t)
380 {
381         bool empty_exp;
382         bool empty_norm;
383         bool empty_exp_now;
384         unsigned long flags;
385         struct list_head *np;
386         bool drop_boost_mutex = false;
387         struct rcu_data *rdp;
388         struct rcu_node *rnp;
389         union rcu_special special;
390 
391         /* NMI handlers cannot block and cannot safely manipulate state. */
392         if (in_nmi())
393                 return;
394 
395         local_irq_save(flags);
396 
397         /*
398          * If RCU core is waiting for this CPU to exit its critical section,
399          * report the fact that it has exited.  Because irqs are disabled,
400          * t->rcu_read_unlock_special cannot change.
401          */
402         special = t->rcu_read_unlock_special;
403         if (special.b.need_qs) {
404                 rcu_preempt_qs();
405                 t->rcu_read_unlock_special.b.need_qs = false;
406                 if (!t->rcu_read_unlock_special.s) {
407                         local_irq_restore(flags);
408                         return;
409                 }
410         }
411 
412         /*
413          * Respond to a request for an expedited grace period, but only if
414          * we were not preempted, meaning that we were running on the same
415          * CPU throughout.  If we were preempted, the exp_need_qs flag
416          * would have been cleared at the time of the first preemption,
417          * and the quiescent state would be reported when we were dequeued.
418          */
419         if (special.b.exp_need_qs) {
420                 WARN_ON_ONCE(special.b.blocked);
421                 t->rcu_read_unlock_special.b.exp_need_qs = false;
422                 rdp = this_cpu_ptr(rcu_state_p->rda);
423                 rcu_report_exp_rdp(rcu_state_p, rdp, true);
424                 if (!t->rcu_read_unlock_special.s) {
425                         local_irq_restore(flags);
426                         return;
427                 }
428         }
429 
430         /* Hardware IRQ handlers cannot block, complain if they get here. */
431         if (in_irq() || in_serving_softirq()) {
432                 lockdep_rcu_suspicious(__FILE__, __LINE__,
433                                        "rcu_read_unlock() from irq or softirq with blocking in critical section!!!\n");
434                 pr_alert("->rcu_read_unlock_special: %#x (b: %d, enq: %d nq: %d)\n",
435                          t->rcu_read_unlock_special.s,
436                          t->rcu_read_unlock_special.b.blocked,
437                          t->rcu_read_unlock_special.b.exp_need_qs,
438                          t->rcu_read_unlock_special.b.need_qs);
439                 local_irq_restore(flags);
440                 return;
441         }
442 
443         /* Clean up if blocked during RCU read-side critical section. */
444         if (special.b.blocked) {
445                 t->rcu_read_unlock_special.b.blocked = false;
446 
447                 /*
448                  * Remove this task from the list it blocked on.  The task
449                  * now remains queued on the rcu_node corresponding to the
450                  * CPU it first blocked on, so there is no longer any need
451                  * to loop.  Retain a WARN_ON_ONCE() out of sheer paranoia.
452                  */
453                 rnp = t->rcu_blocked_node;
454                 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
455                 WARN_ON_ONCE(rnp != t->rcu_blocked_node);
456                 empty_norm = !rcu_preempt_blocked_readers_cgp(rnp);
457                 empty_exp = sync_rcu_preempt_exp_done(rnp);
458                 smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */
459                 np = rcu_next_node_entry(t, rnp);
460                 list_del_init(&t->rcu_node_entry);
461                 t->rcu_blocked_node = NULL;
462                 trace_rcu_unlock_preempted_task(TPS("rcu_preempt"),
463                                                 rnp->gpnum, t->pid);
464                 if (&t->rcu_node_entry == rnp->gp_tasks)
465                         rnp->gp_tasks = np;
466                 if (&t->rcu_node_entry == rnp->exp_tasks)
467                         rnp->exp_tasks = np;
468                 if (IS_ENABLED(CONFIG_RCU_BOOST)) {
469                         if (&t->rcu_node_entry == rnp->boost_tasks)
470                                 rnp->boost_tasks = np;
471                         /* Snapshot ->boost_mtx ownership w/rnp->lock held. */
472                         drop_boost_mutex = rt_mutex_owner(&rnp->boost_mtx) == t;
473                 }
474 
475                 /*
476                  * If this was the last task on the current list, and if
477                  * we aren't waiting on any CPUs, report the quiescent state.
478                  * Note that rcu_report_unblock_qs_rnp() releases rnp->lock,
479                  * so we must take a snapshot of the expedited state.
480                  */
481                 empty_exp_now = sync_rcu_preempt_exp_done(rnp);
482                 if (!empty_norm && !rcu_preempt_blocked_readers_cgp(rnp)) {
483                         trace_rcu_quiescent_state_report(TPS("preempt_rcu"),
484                                                          rnp->gpnum,
485                                                          0, rnp->qsmask,
486                                                          rnp->level,
487                                                          rnp->grplo,
488                                                          rnp->grphi,
489                                                          !!rnp->gp_tasks);
490                         rcu_report_unblock_qs_rnp(rcu_state_p, rnp, flags);
491                 } else {
492                         raw_spin_unlock_irqrestore(&rnp->lock, flags);
493                 }
494 
495                 /* Unboost if we were boosted. */
496                 if (IS_ENABLED(CONFIG_RCU_BOOST) && drop_boost_mutex)
497                         rt_mutex_unlock(&rnp->boost_mtx);
498 
499                 /*
500                  * If this was the last task on the expedited lists,
501                  * then we need to report up the rcu_node hierarchy.
502                  */
503                 if (!empty_exp && empty_exp_now)
504                         rcu_report_exp_rnp(rcu_state_p, rnp, true);
505         } else {
506                 local_irq_restore(flags);
507         }
508 }
509 
510 /*
511  * Dump detailed information for all tasks blocking the current RCU
512  * grace period on the specified rcu_node structure.
513  */
514 static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
515 {
516         unsigned long flags;
517         struct task_struct *t;
518 
519         raw_spin_lock_irqsave_rcu_node(rnp, flags);
520         if (!rcu_preempt_blocked_readers_cgp(rnp)) {
521                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
522                 return;
523         }
524         t = list_entry(rnp->gp_tasks->prev,
525                        struct task_struct, rcu_node_entry);
526         list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry)
527                 sched_show_task(t);
528         raw_spin_unlock_irqrestore(&rnp->lock, flags);
529 }
530 
531 /*
532  * Dump detailed information for all tasks blocking the current RCU
533  * grace period.
534  */
535 static void rcu_print_detail_task_stall(struct rcu_state *rsp)
536 {
537         struct rcu_node *rnp = rcu_get_root(rsp);
538 
539         rcu_print_detail_task_stall_rnp(rnp);
540         rcu_for_each_leaf_node(rsp, rnp)
541                 rcu_print_detail_task_stall_rnp(rnp);
542 }
543 
544 static void rcu_print_task_stall_begin(struct rcu_node *rnp)
545 {
546         pr_err("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):",
547                rnp->level, rnp->grplo, rnp->grphi);
548 }
549 
550 static void rcu_print_task_stall_end(void)
551 {
552         pr_cont("\n");
553 }
554 
555 /*
556  * Scan the current list of tasks blocked within RCU read-side critical
557  * sections, printing out the tid of each.
558  */
559 static int rcu_print_task_stall(struct rcu_node *rnp)
560 {
561         struct task_struct *t;
562         int ndetected = 0;
563 
564         if (!rcu_preempt_blocked_readers_cgp(rnp))
565                 return 0;
566         rcu_print_task_stall_begin(rnp);
567         t = list_entry(rnp->gp_tasks->prev,
568                        struct task_struct, rcu_node_entry);
569         list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
570                 pr_cont(" P%d", t->pid);
571                 ndetected++;
572         }
573         rcu_print_task_stall_end();
574         return ndetected;
575 }
576 
577 /*
578  * Scan the current list of tasks blocked within RCU read-side critical
579  * sections, printing out the tid of each that is blocking the current
580  * expedited grace period.
581  */
582 static int rcu_print_task_exp_stall(struct rcu_node *rnp)
583 {
584         struct task_struct *t;
585         int ndetected = 0;
586 
587         if (!rnp->exp_tasks)
588                 return 0;
589         t = list_entry(rnp->exp_tasks->prev,
590                        struct task_struct, rcu_node_entry);
591         list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
592                 pr_cont(" P%d", t->pid);
593                 ndetected++;
594         }
595         return ndetected;
596 }
597 
598 /*
599  * Check that the list of blocked tasks for the newly completed grace
600  * period is in fact empty.  It is a serious bug to complete a grace
601  * period that still has RCU readers blocked!  This function must be
602  * invoked -before- updating this rnp's ->gpnum, and the rnp's ->lock
603  * must be held by the caller.
604  *
605  * Also, if there are blocked tasks on the list, they automatically
606  * block the newly created grace period, so set up ->gp_tasks accordingly.
607  */
608 static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
609 {
610         WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp));
611         if (rcu_preempt_has_tasks(rnp))
612                 rnp->gp_tasks = rnp->blkd_tasks.next;
613         WARN_ON_ONCE(rnp->qsmask);
614 }
615 
616 /*
617  * Check for a quiescent state from the current CPU.  When a task blocks,
618  * the task is recorded in the corresponding CPU's rcu_node structure,
619  * which is checked elsewhere.
620  *
621  * Caller must disable hard irqs.
622  */
623 static void rcu_preempt_check_callbacks(void)
624 {
625         struct task_struct *t = current;
626 
627         if (t->rcu_read_lock_nesting == 0) {
628                 rcu_preempt_qs();
629                 return;
630         }
631         if (t->rcu_read_lock_nesting > 0 &&
632             __this_cpu_read(rcu_data_p->core_needs_qs) &&
633             __this_cpu_read(rcu_data_p->cpu_no_qs.b.norm))
634                 t->rcu_read_unlock_special.b.need_qs = true;
635 }
636 
637 #ifdef CONFIG_RCU_BOOST
638 
639 static void rcu_preempt_do_callbacks(void)
640 {
641         rcu_do_batch(rcu_state_p, this_cpu_ptr(rcu_data_p));
642 }
643 
644 #endif /* #ifdef CONFIG_RCU_BOOST */
645 
646 /*
647  * Queue a preemptible-RCU callback for invocation after a grace period.
648  */
649 void call_rcu(struct rcu_head *head, rcu_callback_t func)
650 {
651         __call_rcu(head, func, rcu_state_p, -1, 0);
652 }
653 EXPORT_SYMBOL_GPL(call_rcu);
654 
655 /**
656  * synchronize_rcu - wait until a grace period has elapsed.
657  *
658  * Control will return to the caller some time after a full grace
659  * period has elapsed, in other words after all currently executing RCU
660  * read-side critical sections have completed.  Note, however, that
661  * upon return from synchronize_rcu(), the caller might well be executing
662  * concurrently with new RCU read-side critical sections that began while
663  * synchronize_rcu() was waiting.  RCU read-side critical sections are
664  * delimited by rcu_read_lock() and rcu_read_unlock(), and may be nested.
665  *
666  * See the description of synchronize_sched() for more detailed information
667  * on memory ordering guarantees.
668  */
669 void synchronize_rcu(void)
670 {
671         RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
672                          lock_is_held(&rcu_lock_map) ||
673                          lock_is_held(&rcu_sched_lock_map),
674                          "Illegal synchronize_rcu() in RCU read-side critical section");
675         if (!rcu_scheduler_active)
676                 return;
677         if (rcu_gp_is_expedited())
678                 synchronize_rcu_expedited();
679         else
680                 wait_rcu_gp(call_rcu);
681 }
682 EXPORT_SYMBOL_GPL(synchronize_rcu);
683 
684 /*
685  * Remote handler for smp_call_function_single().  If there is an
686  * RCU read-side critical section in effect, request that the
687  * next rcu_read_unlock() record the quiescent state up the
688  * ->expmask fields in the rcu_node tree.  Otherwise, immediately
689  * report the quiescent state.
690  */
691 static void sync_rcu_exp_handler(void *info)
692 {
693         struct rcu_data *rdp;
694         struct rcu_state *rsp = info;
695         struct task_struct *t = current;
696 
697         /*
698          * Within an RCU read-side critical section, request that the next
699          * rcu_read_unlock() report.  Unless this RCU read-side critical
700          * section has already blocked, in which case it is already set
701          * up for the expedited grace period to wait on it.
702          */
703         if (t->rcu_read_lock_nesting > 0 &&
704             !t->rcu_read_unlock_special.b.blocked) {
705                 t->rcu_read_unlock_special.b.exp_need_qs = true;
706                 return;
707         }
708 
709         /*
710          * We are either exiting an RCU read-side critical section (negative
711          * values of t->rcu_read_lock_nesting) or are not in one at all
712          * (zero value of t->rcu_read_lock_nesting).  Or we are in an RCU
713          * read-side critical section that blocked before this expedited
714          * grace period started.  Either way, we can immediately report
715          * the quiescent state.
716          */
717         rdp = this_cpu_ptr(rsp->rda);
718         rcu_report_exp_rdp(rsp, rdp, true);
719 }
720 
721 /**
722  * synchronize_rcu_expedited - Brute-force RCU grace period
723  *
724  * Wait for an RCU-preempt grace period, but expedite it.  The basic
725  * idea is to invoke synchronize_sched_expedited() to push all the tasks to
726  * the ->blkd_tasks lists and wait for this list to drain.  This consumes
727  * significant time on all CPUs and is unfriendly to real-time workloads,
728  * so is thus not recommended for any sort of common-case code.
729  * In fact, if you are using synchronize_rcu_expedited() in a loop,
730  * please restructure your code to batch your updates, and then Use a
731  * single synchronize_rcu() instead.
732  */
733 void synchronize_rcu_expedited(void)
734 {
735         struct rcu_node *rnp;
736         struct rcu_node *rnp_unlock;
737         struct rcu_state *rsp = rcu_state_p;
738         unsigned long s;
739 
740         /* If expedited grace periods are prohibited, fall back to normal. */
741         if (rcu_gp_is_normal()) {
742                 wait_rcu_gp(call_rcu);
743                 return;
744         }
745 
746         s = rcu_exp_gp_seq_snap(rsp);
747 
748         rnp_unlock = exp_funnel_lock(rsp, s);
749         if (rnp_unlock == NULL)
750                 return;  /* Someone else did our work for us. */
751 
752         rcu_exp_gp_seq_start(rsp);
753 
754         /* Initialize the rcu_node tree in preparation for the wait. */
755         sync_rcu_exp_select_cpus(rsp, sync_rcu_exp_handler);
756 
757         /* Wait for snapshotted ->blkd_tasks lists to drain. */
758         rnp = rcu_get_root(rsp);
759         synchronize_sched_expedited_wait(rsp);
760 
761         /* Clean up and exit. */
762         rcu_exp_gp_seq_end(rsp);
763         mutex_unlock(&rnp_unlock->exp_funnel_mutex);
764 }
765 EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
766 
767 /**
768  * rcu_barrier - Wait until all in-flight call_rcu() callbacks complete.
769  *
770  * Note that this primitive does not necessarily wait for an RCU grace period
771  * to complete.  For example, if there are no RCU callbacks queued anywhere
772  * in the system, then rcu_barrier() is within its rights to return
773  * immediately, without waiting for anything, much less an RCU grace period.
774  */
775 void rcu_barrier(void)
776 {
777         _rcu_barrier(rcu_state_p);
778 }
779 EXPORT_SYMBOL_GPL(rcu_barrier);
780 
781 /*
782  * Initialize preemptible RCU's state structures.
783  */
784 static void __init __rcu_init_preempt(void)
785 {
786         rcu_init_one(rcu_state_p);
787 }
788 
789 /*
790  * Check for a task exiting while in a preemptible-RCU read-side
791  * critical section, clean up if so.  No need to issue warnings,
792  * as debug_check_no_locks_held() already does this if lockdep
793  * is enabled.
794  */
795 void exit_rcu(void)
796 {
797         struct task_struct *t = current;
798 
799         if (likely(list_empty(&current->rcu_node_entry)))
800                 return;
801         t->rcu_read_lock_nesting = 1;
802         barrier();
803         t->rcu_read_unlock_special.b.blocked = true;
804         __rcu_read_unlock();
805 }
806 
807 #else /* #ifdef CONFIG_PREEMPT_RCU */
808 
809 static struct rcu_state *const rcu_state_p = &rcu_sched_state;
810 static struct rcu_data __percpu *const rcu_data_p = &rcu_sched_data;
811 
812 /*
813  * Tell them what RCU they are running.
814  */
815 static void __init rcu_bootup_announce(void)
816 {
817         pr_info("Hierarchical RCU implementation.\n");
818         rcu_bootup_announce_oddness();
819 }
820 
821 /*
822  * Because preemptible RCU does not exist, we never have to check for
823  * CPUs being in quiescent states.
824  */
825 static void rcu_preempt_note_context_switch(void)
826 {
827 }
828 
829 /*
830  * Because preemptible RCU does not exist, there are never any preempted
831  * RCU readers.
832  */
833 static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
834 {
835         return 0;
836 }
837 
838 /*
839  * Because there is no preemptible RCU, there can be no readers blocked.
840  */
841 static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
842 {
843         return false;
844 }
845 
846 /*
847  * Because preemptible RCU does not exist, we never have to check for
848  * tasks blocked within RCU read-side critical sections.
849  */
850 static void rcu_print_detail_task_stall(struct rcu_state *rsp)
851 {
852 }
853 
854 /*
855  * Because preemptible RCU does not exist, we never have to check for
856  * tasks blocked within RCU read-side critical sections.
857  */
858 static int rcu_print_task_stall(struct rcu_node *rnp)
859 {
860         return 0;
861 }
862 
863 /*
864  * Because preemptible RCU does not exist, we never have to check for
865  * tasks blocked within RCU read-side critical sections that are
866  * blocking the current expedited grace period.
867  */
868 static int rcu_print_task_exp_stall(struct rcu_node *rnp)
869 {
870         return 0;
871 }
872 
873 /*
874  * Because there is no preemptible RCU, there can be no readers blocked,
875  * so there is no need to check for blocked tasks.  So check only for
876  * bogus qsmask values.
877  */
878 static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
879 {
880         WARN_ON_ONCE(rnp->qsmask);
881 }
882 
883 /*
884  * Because preemptible RCU does not exist, it never has any callbacks
885  * to check.
886  */
887 static void rcu_preempt_check_callbacks(void)
888 {
889 }
890 
891 /*
892  * Wait for an rcu-preempt grace period, but make it happen quickly.
893  * But because preemptible RCU does not exist, map to rcu-sched.
894  */
895 void synchronize_rcu_expedited(void)
896 {
897         synchronize_sched_expedited();
898 }
899 EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
900 
901 /*
902  * Because preemptible RCU does not exist, rcu_barrier() is just
903  * another name for rcu_barrier_sched().
904  */
905 void rcu_barrier(void)
906 {
907         rcu_barrier_sched();
908 }
909 EXPORT_SYMBOL_GPL(rcu_barrier);
910 
911 /*
912  * Because preemptible RCU does not exist, it need not be initialized.
913  */
914 static void __init __rcu_init_preempt(void)
915 {
916 }
917 
918 /*
919  * Because preemptible RCU does not exist, tasks cannot possibly exit
920  * while in preemptible RCU read-side critical sections.
921  */
922 void exit_rcu(void)
923 {
924 }
925 
926 #endif /* #else #ifdef CONFIG_PREEMPT_RCU */
927 
928 #ifdef CONFIG_RCU_BOOST
929 
930 #include "../locking/rtmutex_common.h"
931 
932 #ifdef CONFIG_RCU_TRACE
933 
934 static void rcu_initiate_boost_trace(struct rcu_node *rnp)
935 {
936         if (!rcu_preempt_has_tasks(rnp))
937                 rnp->n_balk_blkd_tasks++;
938         else if (rnp->exp_tasks == NULL && rnp->gp_tasks == NULL)
939                 rnp->n_balk_exp_gp_tasks++;
940         else if (rnp->gp_tasks != NULL && rnp->boost_tasks != NULL)
941                 rnp->n_balk_boost_tasks++;
942         else if (rnp->gp_tasks != NULL && rnp->qsmask != 0)
943                 rnp->n_balk_notblocked++;
944         else if (rnp->gp_tasks != NULL &&
945                  ULONG_CMP_LT(jiffies, rnp->boost_time))
946                 rnp->n_balk_notyet++;
947         else
948                 rnp->n_balk_nos++;
949 }
950 
951 #else /* #ifdef CONFIG_RCU_TRACE */
952 
953 static void rcu_initiate_boost_trace(struct rcu_node *rnp)
954 {
955 }
956 
957 #endif /* #else #ifdef CONFIG_RCU_TRACE */
958 
959 static void rcu_wake_cond(struct task_struct *t, int status)
960 {
961         /*
962          * If the thread is yielding, only wake it when this
963          * is invoked from idle
964          */
965         if (status != RCU_KTHREAD_YIELDING || is_idle_task(current))
966                 wake_up_process(t);
967 }
968 
969 /*
970  * Carry out RCU priority boosting on the task indicated by ->exp_tasks
971  * or ->boost_tasks, advancing the pointer to the next task in the
972  * ->blkd_tasks list.
973  *
974  * Note that irqs must be enabled: boosting the task can block.
975  * Returns 1 if there are more tasks needing to be boosted.
976  */
977 static int rcu_boost(struct rcu_node *rnp)
978 {
979         unsigned long flags;
980         struct task_struct *t;
981         struct list_head *tb;
982 
983         if (READ_ONCE(rnp->exp_tasks) == NULL &&
984             READ_ONCE(rnp->boost_tasks) == NULL)
985                 return 0;  /* Nothing left to boost. */
986 
987         raw_spin_lock_irqsave_rcu_node(rnp, flags);
988 
989         /*
990          * Recheck under the lock: all tasks in need of boosting
991          * might exit their RCU read-side critical sections on their own.
992          */
993         if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL) {
994                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
995                 return 0;
996         }
997 
998         /*
999          * Preferentially boost tasks blocking expedited grace periods.
1000          * This cannot starve the normal grace periods because a second
1001          * expedited grace period must boost all blocked tasks, including
1002          * those blocking the pre-existing normal grace period.
1003          */
1004         if (rnp->exp_tasks != NULL) {
1005                 tb = rnp->exp_tasks;
1006                 rnp->n_exp_boosts++;
1007         } else {
1008                 tb = rnp->boost_tasks;
1009                 rnp->n_normal_boosts++;
1010         }
1011         rnp->n_tasks_boosted++;
1012 
1013         /*
1014          * We boost task t by manufacturing an rt_mutex that appears to
1015          * be held by task t.  We leave a pointer to that rt_mutex where
1016          * task t can find it, and task t will release the mutex when it
1017          * exits its outermost RCU read-side critical section.  Then
1018          * simply acquiring this artificial rt_mutex will boost task
1019          * t's priority.  (Thanks to tglx for suggesting this approach!)
1020          *
1021          * Note that task t must acquire rnp->lock to remove itself from
1022          * the ->blkd_tasks list, which it will do from exit() if from
1023          * nowhere else.  We therefore are guaranteed that task t will
1024          * stay around at least until we drop rnp->lock.  Note that
1025          * rnp->lock also resolves races between our priority boosting
1026          * and task t's exiting its outermost RCU read-side critical
1027          * section.
1028          */
1029         t = container_of(tb, struct task_struct, rcu_node_entry);
1030         rt_mutex_init_proxy_locked(&rnp->boost_mtx, t);
1031         raw_spin_unlock_irqrestore(&rnp->lock, flags);
1032         /* Lock only for side effect: boosts task t's priority. */
1033         rt_mutex_lock(&rnp->boost_mtx);
1034         rt_mutex_unlock(&rnp->boost_mtx);  /* Then keep lockdep happy. */
1035 
1036         return READ_ONCE(rnp->exp_tasks) != NULL ||
1037                READ_ONCE(rnp->boost_tasks) != NULL;
1038 }
1039 
1040 /*
1041  * Priority-boosting kthread, one per leaf rcu_node.
1042  */
1043 static int rcu_boost_kthread(void *arg)
1044 {
1045         struct rcu_node *rnp = (struct rcu_node *)arg;
1046         int spincnt = 0;
1047         int more2boost;
1048 
1049         trace_rcu_utilization(TPS("Start boost kthread@init"));
1050         for (;;) {
1051                 rnp->boost_kthread_status = RCU_KTHREAD_WAITING;
1052                 trace_rcu_utilization(TPS("End boost kthread@rcu_wait"));
1053                 rcu_wait(rnp->boost_tasks || rnp->exp_tasks);
1054                 trace_rcu_utilization(TPS("Start boost kthread@rcu_wait"));
1055                 rnp->boost_kthread_status = RCU_KTHREAD_RUNNING;
1056                 more2boost = rcu_boost(rnp);
1057                 if (more2boost)
1058                         spincnt++;
1059                 else
1060                         spincnt = 0;
1061                 if (spincnt > 10) {
1062                         rnp->boost_kthread_status = RCU_KTHREAD_YIELDING;
1063                         trace_rcu_utilization(TPS("End boost kthread@rcu_yield"));
1064                         schedule_timeout_interruptible(2);
1065                         trace_rcu_utilization(TPS("Start boost kthread@rcu_yield"));
1066                         spincnt = 0;
1067                 }
1068         }
1069         /* NOTREACHED */
1070         trace_rcu_utilization(TPS("End boost kthread@notreached"));
1071         return 0;
1072 }
1073 
1074 /*
1075  * Check to see if it is time to start boosting RCU readers that are
1076  * blocking the current grace period, and, if so, tell the per-rcu_node
1077  * kthread to start boosting them.  If there is an expedited grace
1078  * period in progress, it is always time to boost.
1079  *
1080  * The caller must hold rnp->lock, which this function releases.
1081  * The ->boost_kthread_task is immortal, so we don't need to worry
1082  * about it going away.
1083  */
1084 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
1085         __releases(rnp->lock)
1086 {
1087         struct task_struct *t;
1088 
1089         if (!rcu_preempt_blocked_readers_cgp(rnp) && rnp->exp_tasks == NULL) {
1090                 rnp->n_balk_exp_gp_tasks++;
1091                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1092                 return;
1093         }
1094         if (rnp->exp_tasks != NULL ||
1095             (rnp->gp_tasks != NULL &&
1096              rnp->boost_tasks == NULL &&
1097              rnp->qsmask == 0 &&
1098              ULONG_CMP_GE(jiffies, rnp->boost_time))) {
1099                 if (rnp->exp_tasks == NULL)
1100                         rnp->boost_tasks = rnp->gp_tasks;
1101                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1102                 t = rnp->boost_kthread_task;
1103                 if (t)
1104                         rcu_wake_cond(t, rnp->boost_kthread_status);
1105         } else {
1106                 rcu_initiate_boost_trace(rnp);
1107                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1108         }
1109 }
1110 
1111 /*
1112  * Wake up the per-CPU kthread to invoke RCU callbacks.
1113  */
1114 static void invoke_rcu_callbacks_kthread(void)
1115 {
1116         unsigned long flags;
1117 
1118         local_irq_save(flags);
1119         __this_cpu_write(rcu_cpu_has_work, 1);
1120         if (__this_cpu_read(rcu_cpu_kthread_task) != NULL &&
1121             current != __this_cpu_read(rcu_cpu_kthread_task)) {
1122                 rcu_wake_cond(__this_cpu_read(rcu_cpu_kthread_task),
1123                               __this_cpu_read(rcu_cpu_kthread_status));
1124         }
1125         local_irq_restore(flags);
1126 }
1127 
1128 /*
1129  * Is the current CPU running the RCU-callbacks kthread?
1130  * Caller must have preemption disabled.
1131  */
1132 static bool rcu_is_callbacks_kthread(void)
1133 {
1134         return __this_cpu_read(rcu_cpu_kthread_task) == current;
1135 }
1136 
1137 #define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)
1138 
1139 /*
1140  * Do priority-boost accounting for the start of a new grace period.
1141  */
1142 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1143 {
1144         rnp->boost_time = jiffies + RCU_BOOST_DELAY_JIFFIES;
1145 }
1146 
1147 /*
1148  * Create an RCU-boost kthread for the specified node if one does not
1149  * already exist.  We only create this kthread for preemptible RCU.
1150  * Returns zero if all is well, a negated errno otherwise.
1151  */
1152 static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
1153                                        struct rcu_node *rnp)
1154 {
1155         int rnp_index = rnp - &rsp->node[0];
1156         unsigned long flags;
1157         struct sched_param sp;
1158         struct task_struct *t;
1159 
1160         if (rcu_state_p != rsp)
1161                 return 0;
1162 
1163         if (!rcu_scheduler_fully_active || rcu_rnp_online_cpus(rnp) == 0)
1164                 return 0;
1165 
1166         rsp->boost = 1;
1167         if (rnp->boost_kthread_task != NULL)
1168                 return 0;
1169         t = kthread_create(rcu_boost_kthread, (void *)rnp,
1170                            "rcub/%d", rnp_index);
1171         if (IS_ERR(t))
1172                 return PTR_ERR(t);
1173         raw_spin_lock_irqsave_rcu_node(rnp, flags);
1174         rnp->boost_kthread_task = t;
1175         raw_spin_unlock_irqrestore(&rnp->lock, flags);
1176         sp.sched_priority = kthread_prio;
1177         sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
1178         wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */
1179         return 0;
1180 }
1181 
1182 static void rcu_kthread_do_work(void)
1183 {
1184         rcu_do_batch(&rcu_sched_state, this_cpu_ptr(&rcu_sched_data));
1185         rcu_do_batch(&rcu_bh_state, this_cpu_ptr(&rcu_bh_data));
1186         rcu_preempt_do_callbacks();
1187 }
1188 
1189 static void rcu_cpu_kthread_setup(unsigned int cpu)
1190 {
1191         struct sched_param sp;
1192 
1193         sp.sched_priority = kthread_prio;
1194         sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1195 }
1196 
1197 static void rcu_cpu_kthread_park(unsigned int cpu)
1198 {
1199         per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
1200 }
1201 
1202 static int rcu_cpu_kthread_should_run(unsigned int cpu)
1203 {
1204         return __this_cpu_read(rcu_cpu_has_work);
1205 }
1206 
1207 /*
1208  * Per-CPU kernel thread that invokes RCU callbacks.  This replaces the
1209  * RCU softirq used in flavors and configurations of RCU that do not
1210  * support RCU priority boosting.
1211  */
1212 static void rcu_cpu_kthread(unsigned int cpu)
1213 {
1214         unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status);
1215         char work, *workp = this_cpu_ptr(&rcu_cpu_has_work);
1216         int spincnt;
1217 
1218         for (spincnt = 0; spincnt < 10; spincnt++) {
1219                 trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait"));
1220                 local_bh_disable();
1221                 *statusp = RCU_KTHREAD_RUNNING;
1222                 this_cpu_inc(rcu_cpu_kthread_loops);
1223                 local_irq_disable();
1224                 work = *workp;
1225                 *workp = 0;
1226                 local_irq_enable();
1227                 if (work)
1228                         rcu_kthread_do_work();
1229                 local_bh_enable();
1230                 if (*workp == 0) {
1231                         trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
1232                         *statusp = RCU_KTHREAD_WAITING;
1233                         return;
1234                 }
1235         }
1236         *statusp = RCU_KTHREAD_YIELDING;
1237         trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
1238         schedule_timeout_interruptible(2);
1239         trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
1240         *statusp = RCU_KTHREAD_WAITING;
1241 }
1242 
1243 /*
1244  * Set the per-rcu_node kthread's affinity to cover all CPUs that are
1245  * served by the rcu_node in question.  The CPU hotplug lock is still
1246  * held, so the value of rnp->qsmaskinit will be stable.
1247  *
1248  * We don't include outgoingcpu in the affinity set, use -1 if there is
1249  * no outgoing CPU.  If there are no CPUs left in the affinity set,
1250  * this function allows the kthread to execute on any CPU.
1251  */
1252 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1253 {
1254         struct task_struct *t = rnp->boost_kthread_task;
1255         unsigned long mask = rcu_rnp_online_cpus(rnp);
1256         cpumask_var_t cm;
1257         int cpu;
1258 
1259         if (!t)
1260                 return;
1261         if (!zalloc_cpumask_var(&cm, GFP_KERNEL))
1262                 return;
1263         for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask >>= 1)
1264                 if ((mask & 0x1) && cpu != outgoingcpu)
1265                         cpumask_set_cpu(cpu, cm);
1266         if (cpumask_weight(cm) == 0)
1267                 cpumask_setall(cm);
1268         set_cpus_allowed_ptr(t, cm);
1269         free_cpumask_var(cm);
1270 }
1271 
1272 static struct smp_hotplug_thread rcu_cpu_thread_spec = {
1273         .store                  = &rcu_cpu_kthread_task,
1274         .thread_should_run      = rcu_cpu_kthread_should_run,
1275         .thread_fn              = rcu_cpu_kthread,
1276         .thread_comm            = "rcuc/%u",
1277         .setup                  = rcu_cpu_kthread_setup,
1278         .park                   = rcu_cpu_kthread_park,
1279 };
1280 
1281 /*
1282  * Spawn boost kthreads -- called as soon as the scheduler is running.
1283  */
1284 static void __init rcu_spawn_boost_kthreads(void)
1285 {
1286         struct rcu_node *rnp;
1287         int cpu;
1288 
1289         for_each_possible_cpu(cpu)
1290                 per_cpu(rcu_cpu_has_work, cpu) = 0;
1291         BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec));
1292         rcu_for_each_leaf_node(rcu_state_p, rnp)
1293                 (void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp);
1294 }
1295 
1296 static void rcu_prepare_kthreads(int cpu)
1297 {
1298         struct rcu_data *rdp = per_cpu_ptr(rcu_state_p->rda, cpu);
1299         struct rcu_node *rnp = rdp->mynode;
1300 
1301         /* Fire up the incoming CPU's kthread and leaf rcu_node kthread. */
1302         if (rcu_scheduler_fully_active)
1303                 (void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp);
1304 }
1305 
1306 #else /* #ifdef CONFIG_RCU_BOOST */
1307 
1308 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
1309         __releases(rnp->lock)
1310 {
1311         raw_spin_unlock_irqrestore(&rnp->lock, flags);
1312 }
1313 
1314 static void invoke_rcu_callbacks_kthread(void)
1315 {
1316         WARN_ON_ONCE(1);
1317 }
1318 
1319 static bool rcu_is_callbacks_kthread(void)
1320 {
1321         return false;
1322 }
1323 
1324 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1325 {
1326 }
1327 
1328 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1329 {
1330 }
1331 
1332 static void __init rcu_spawn_boost_kthreads(void)
1333 {
1334 }
1335 
1336 static void rcu_prepare_kthreads(int cpu)
1337 {
1338 }
1339 
1340 #endif /* #else #ifdef CONFIG_RCU_BOOST */
1341 
1342 #if !defined(CONFIG_RCU_FAST_NO_HZ)
1343 
1344 /*
1345  * Check to see if any future RCU-related work will need to be done
1346  * by the current CPU, even if none need be done immediately, returning
1347  * 1 if so.  This function is part of the RCU implementation; it is -not-
1348  * an exported member of the RCU API.
1349  *
1350  * Because we not have RCU_FAST_NO_HZ, just check whether this CPU needs
1351  * any flavor of RCU.
1352  */
1353 int rcu_needs_cpu(u64 basemono, u64 *nextevt)
1354 {
1355         *nextevt = KTIME_MAX;
1356         return IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL)
1357                ? 0 : rcu_cpu_has_callbacks(NULL);
1358 }
1359 
1360 /*
1361  * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
1362  * after it.
1363  */
1364 static void rcu_cleanup_after_idle(void)
1365 {
1366 }
1367 
1368 /*
1369  * Do the idle-entry grace-period work, which, because CONFIG_RCU_FAST_NO_HZ=n,
1370  * is nothing.
1371  */
1372 static void rcu_prepare_for_idle(void)
1373 {
1374 }
1375 
1376 /*
1377  * Don't bother keeping a running count of the number of RCU callbacks
1378  * posted because CONFIG_RCU_FAST_NO_HZ=n.
1379  */
1380 static void rcu_idle_count_callbacks_posted(void)
1381 {
1382 }
1383 
1384 #else /* #if !defined(CONFIG_RCU_FAST_NO_HZ) */
1385 
1386 /*
1387  * This code is invoked when a CPU goes idle, at which point we want
1388  * to have the CPU do everything required for RCU so that it can enter
1389  * the energy-efficient dyntick-idle mode.  This is handled by a
1390  * state machine implemented by rcu_prepare_for_idle() below.
1391  *
1392  * The following three proprocessor symbols control this state machine:
1393  *
1394  * RCU_IDLE_GP_DELAY gives the number of jiffies that a CPU is permitted
1395  *      to sleep in dyntick-idle mode with RCU callbacks pending.  This
1396  *      is sized to be roughly one RCU grace period.  Those energy-efficiency
1397  *      benchmarkers who might otherwise be tempted to set this to a large
1398  *      number, be warned: Setting RCU_IDLE_GP_DELAY too high can hang your
1399  *      system.  And if you are -that- concerned about energy efficiency,
1400  *      just power the system down and be done with it!
1401  * RCU_IDLE_LAZY_GP_DELAY gives the number of jiffies that a CPU is
1402  *      permitted to sleep in dyntick-idle mode with only lazy RCU
1403  *      callbacks pending.  Setting this too high can OOM your system.
1404  *
1405  * The values below work well in practice.  If future workloads require
1406  * adjustment, they can be converted into kernel config parameters, though
1407  * making the state machine smarter might be a better option.
1408  */
1409 #define RCU_IDLE_GP_DELAY 4             /* Roughly one grace period. */
1410 #define RCU_IDLE_LAZY_GP_DELAY (6 * HZ) /* Roughly six seconds. */
1411 
1412 static int rcu_idle_gp_delay = RCU_IDLE_GP_DELAY;
1413 module_param(rcu_idle_gp_delay, int, 0644);
1414 static int rcu_idle_lazy_gp_delay = RCU_IDLE_LAZY_GP_DELAY;
1415 module_param(rcu_idle_lazy_gp_delay, int, 0644);
1416 
1417 /*
1418  * Try to advance callbacks for all flavors of RCU on the current CPU, but
1419  * only if it has been awhile since the last time we did so.  Afterwards,
1420  * if there are any callbacks ready for immediate invocation, return true.
1421  */
1422 static bool __maybe_unused rcu_try_advance_all_cbs(void)
1423 {
1424         bool cbs_ready = false;
1425         struct rcu_data *rdp;
1426         struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1427         struct rcu_node *rnp;
1428         struct rcu_state *rsp;
1429 
1430         /* Exit early if we advanced recently. */
1431         if (jiffies == rdtp->last_advance_all)
1432                 return false;
1433         rdtp->last_advance_all = jiffies;
1434 
1435         for_each_rcu_flavor(rsp) {
1436                 rdp = this_cpu_ptr(rsp->rda);
1437                 rnp = rdp->mynode;
1438 
1439                 /*
1440                  * Don't bother checking unless a grace period has
1441                  * completed since we last checked and there are
1442                  * callbacks not yet ready to invoke.
1443                  */
1444                 if ((rdp->completed != rnp->completed ||
1445                      unlikely(READ_ONCE(rdp->gpwrap))) &&
1446                     rdp->nxttail[RCU_DONE_TAIL] != rdp->nxttail[RCU_NEXT_TAIL])
1447                         note_gp_changes(rsp, rdp);
1448 
1449                 if (cpu_has_callbacks_ready_to_invoke(rdp))
1450                         cbs_ready = true;
1451         }
1452         return cbs_ready;
1453 }
1454 
1455 /*
1456  * Allow the CPU to enter dyntick-idle mode unless it has callbacks ready
1457  * to invoke.  If the CPU has callbacks, try to advance them.  Tell the
1458  * caller to set the timeout based on whether or not there are non-lazy
1459  * callbacks.
1460  *
1461  * The caller must have disabled interrupts.
1462  */
1463 int rcu_needs_cpu(u64 basemono, u64 *nextevt)
1464 {
1465         struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1466         unsigned long dj;
1467 
1468         if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL)) {
1469                 *nextevt = KTIME_MAX;
1470                 return 0;
1471         }
1472 
1473         /* Snapshot to detect later posting of non-lazy callback. */
1474         rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
1475 
1476         /* If no callbacks, RCU doesn't need the CPU. */
1477         if (!rcu_cpu_has_callbacks(&rdtp->all_lazy)) {
1478                 *nextevt = KTIME_MAX;
1479                 return 0;
1480         }
1481 
1482         /* Attempt to advance callbacks. */
1483         if (rcu_try_advance_all_cbs()) {
1484                 /* Some ready to invoke, so initiate later invocation. */
1485                 invoke_rcu_core();
1486                 return 1;
1487         }
1488         rdtp->last_accelerate = jiffies;
1489 
1490         /* Request timer delay depending on laziness, and round. */
1491         if (!rdtp->all_lazy) {
1492                 dj = round_up(rcu_idle_gp_delay + jiffies,
1493                                rcu_idle_gp_delay) - jiffies;
1494         } else {
1495                 dj = round_jiffies(rcu_idle_lazy_gp_delay + jiffies) - jiffies;
1496         }
1497         *nextevt = basemono + dj * TICK_NSEC;
1498         return 0;
1499 }
1500 
1501 /*
1502  * Prepare a CPU for idle from an RCU perspective.  The first major task
1503  * is to sense whether nohz mode has been enabled or disabled via sysfs.
1504  * The second major task is to check to see if a non-lazy callback has
1505  * arrived at a CPU that previously had only lazy callbacks.  The third
1506  * major task is to accelerate (that is, assign grace-period numbers to)
1507  * any recently arrived callbacks.
1508  *
1509  * The caller must have disabled interrupts.
1510  */
1511 static void rcu_prepare_for_idle(void)
1512 {
1513         bool needwake;
1514         struct rcu_data *rdp;
1515         struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1516         struct rcu_node *rnp;
1517         struct rcu_state *rsp;
1518         int tne;
1519 
1520         if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL) ||
1521             rcu_is_nocb_cpu(smp_processor_id()))
1522                 return;
1523 
1524         /* Handle nohz enablement switches conservatively. */
1525         tne = READ_ONCE(tick_nohz_active);
1526         if (tne != rdtp->tick_nohz_enabled_snap) {
1527                 if (rcu_cpu_has_callbacks(NULL))
1528                         invoke_rcu_core(); /* force nohz to see update. */
1529                 rdtp->tick_nohz_enabled_snap = tne;
1530                 return;
1531         }
1532         if (!tne)
1533                 return;
1534 
1535         /*
1536          * If a non-lazy callback arrived at a CPU having only lazy
1537          * callbacks, invoke RCU core for the side-effect of recalculating
1538          * idle duration on re-entry to idle.
1539          */
1540         if (rdtp->all_lazy &&
1541             rdtp->nonlazy_posted != rdtp->nonlazy_posted_snap) {
1542                 rdtp->all_lazy = false;
1543                 rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
1544                 invoke_rcu_core();
1545                 return;
1546         }
1547 
1548         /*
1549          * If we have not yet accelerated this jiffy, accelerate all
1550          * callbacks on this CPU.
1551          */
1552         if (rdtp->last_accelerate == jiffies)
1553                 return;
1554         rdtp->last_accelerate = jiffies;
1555         for_each_rcu_flavor(rsp) {
1556                 rdp = this_cpu_ptr(rsp->rda);
1557                 if (!*rdp->nxttail[RCU_DONE_TAIL])
1558                         continue;
1559                 rnp = rdp->mynode;
1560                 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
1561                 needwake = rcu_accelerate_cbs(rsp, rnp, rdp);
1562                 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
1563                 if (needwake)
1564                         rcu_gp_kthread_wake(rsp);
1565         }
1566 }
1567 
1568 /*
1569  * Clean up for exit from idle.  Attempt to advance callbacks based on
1570  * any grace periods that elapsed while the CPU was idle, and if any
1571  * callbacks are now ready to invoke, initiate invocation.
1572  */
1573 static void rcu_cleanup_after_idle(void)
1574 {
1575         if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL) ||
1576             rcu_is_nocb_cpu(smp_processor_id()))
1577                 return;
1578         if (rcu_try_advance_all_cbs())
1579                 invoke_rcu_core();
1580 }
1581 
1582 /*
1583  * Keep a running count of the number of non-lazy callbacks posted
1584  * on this CPU.  This running counter (which is never decremented) allows
1585  * rcu_prepare_for_idle() to detect when something out of the idle loop
1586  * posts a callback, even if an equal number of callbacks are invoked.
1587  * Of course, callbacks should only be posted from within a trace event
1588  * designed to be called from idle or from within RCU_NONIDLE().
1589  */
1590 static void rcu_idle_count_callbacks_posted(void)
1591 {
1592         __this_cpu_add(rcu_dynticks.nonlazy_posted, 1);
1593 }
1594 
1595 /*
1596  * Data for flushing lazy RCU callbacks at OOM time.
1597  */
1598 static atomic_t oom_callback_count;
1599 static DECLARE_WAIT_QUEUE_HEAD(oom_callback_wq);
1600 
1601 /*
1602  * RCU OOM callback -- decrement the outstanding count and deliver the
1603  * wake-up if we are the last one.
1604  */
1605 static void rcu_oom_callback(struct rcu_head *rhp)
1606 {
1607         if (atomic_dec_and_test(&oom_callback_count))
1608                 wake_up(&oom_callback_wq);
1609 }
1610 
1611 /*
1612  * Post an rcu_oom_notify callback on the current CPU if it has at
1613  * least one lazy callback.  This will unnecessarily post callbacks
1614  * to CPUs that already have a non-lazy callback at the end of their
1615  * callback list, but this is an infrequent operation, so accept some
1616  * extra overhead to keep things simple.
1617  */
1618 static void rcu_oom_notify_cpu(void *unused)
1619 {
1620         struct rcu_state *rsp;
1621         struct rcu_data *rdp;
1622 
1623         for_each_rcu_flavor(rsp) {
1624                 rdp = raw_cpu_ptr(rsp->rda);
1625                 if (rdp->qlen_lazy != 0) {
1626                         atomic_inc(&oom_callback_count);
1627                         rsp->call(&rdp->oom_head, rcu_oom_callback);
1628                 }
1629         }
1630 }
1631 
1632 /*
1633  * If low on memory, ensure that each CPU has a non-lazy callback.
1634  * This will wake up CPUs that have only lazy callbacks, in turn
1635  * ensuring that they free up the corresponding memory in a timely manner.
1636  * Because an uncertain amount of memory will be freed in some uncertain
1637  * timeframe, we do not claim to have freed anything.
1638  */
1639 static int rcu_oom_notify(struct notifier_block *self,
1640                           unsigned long notused, void *nfreed)
1641 {
1642         int cpu;
1643 
1644         /* Wait for callbacks from earlier instance to complete. */
1645         wait_event(oom_callback_wq, atomic_read(&oom_callback_count) == 0);
1646         smp_mb(); /* Ensure callback reuse happens after callback invocation. */
1647 
1648         /*
1649          * Prevent premature wakeup: ensure that all increments happen
1650          * before there is a chance of the counter reaching zero.
1651          */
1652         atomic_set(&oom_callback_count, 1);
1653 
1654         for_each_online_cpu(cpu) {
1655                 smp_call_function_single(cpu, rcu_oom_notify_cpu, NULL, 1);
1656                 cond_resched_rcu_qs();
1657         }
1658 
1659         /* Unconditionally decrement: no need to wake ourselves up. */
1660         atomic_dec(&oom_callback_count);
1661 
1662         return NOTIFY_OK;
1663 }
1664 
1665 static struct notifier_block rcu_oom_nb = {
1666         .notifier_call = rcu_oom_notify
1667 };
1668 
1669 static int __init rcu_register_oom_notifier(void)
1670 {
1671         register_oom_notifier(&rcu_oom_nb);
1672         return 0;
1673 }
1674 early_initcall(rcu_register_oom_notifier);
1675 
1676 #endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */
1677 
1678 #ifdef CONFIG_RCU_FAST_NO_HZ
1679 
1680 static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
1681 {
1682         struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
1683         unsigned long nlpd = rdtp->nonlazy_posted - rdtp->nonlazy_posted_snap;
1684 
1685         sprintf(cp, "last_accelerate: %04lx/%04lx, nonlazy_posted: %ld, %c%c",
1686                 rdtp->last_accelerate & 0xffff, jiffies & 0xffff,
1687                 ulong2long(nlpd),
1688                 rdtp->all_lazy ? 'L' : '.',
1689                 rdtp->tick_nohz_enabled_snap ? '.' : 'D');
1690 }
1691 
1692 #else /* #ifdef CONFIG_RCU_FAST_NO_HZ */
1693 
1694 static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
1695 {
1696         *cp = '\0';
1697 }
1698 
1699 #endif /* #else #ifdef CONFIG_RCU_FAST_NO_HZ */
1700 
1701 /* Initiate the stall-info list. */
1702 static void print_cpu_stall_info_begin(void)
1703 {
1704         pr_cont("\n");
1705 }
1706 
1707 /*
1708  * Print out diagnostic information for the specified stalled CPU.
1709  *
1710  * If the specified CPU is aware of the current RCU grace period
1711  * (flavor specified by rsp), then print the number of scheduling
1712  * clock interrupts the CPU has taken during the time that it has
1713  * been aware.  Otherwise, print the number of RCU grace periods
1714  * that this CPU is ignorant of, for example, "1" if the CPU was
1715  * aware of the previous grace period.
1716  *
1717  * Also print out idle and (if CONFIG_RCU_FAST_NO_HZ) idle-entry info.
1718  */
1719 static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
1720 {
1721         char fast_no_hz[72];
1722         struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
1723         struct rcu_dynticks *rdtp = rdp->dynticks;
1724         char *ticks_title;
1725         unsigned long ticks_value;
1726 
1727         if (rsp->gpnum == rdp->gpnum) {
1728                 ticks_title = "ticks this GP";
1729                 ticks_value = rdp->ticks_this_gp;
1730         } else {
1731                 ticks_title = "GPs behind";
1732                 ticks_value = rsp->gpnum - rdp->gpnum;
1733         }
1734         print_cpu_stall_fast_no_hz(fast_no_hz, cpu);
1735         pr_err("\t%d-%c%c%c: (%lu %s) idle=%03x/%llx/%d softirq=%u/%u fqs=%ld %s\n",
1736                cpu,
1737                "O."[!!cpu_online(cpu)],
1738                "o."[!!(rdp->grpmask & rdp->mynode->qsmaskinit)],
1739                "N."[!!(rdp->grpmask & rdp->mynode->qsmaskinitnext)],
1740                ticks_value, ticks_title,
1741                atomic_read(&rdtp->dynticks) & 0xfff,
1742                rdtp->dynticks_nesting, rdtp->dynticks_nmi_nesting,
1743                rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
1744                READ_ONCE(rsp->n_force_qs) - rsp->n_force_qs_gpstart,
1745                fast_no_hz);
1746 }
1747 
1748 /* Terminate the stall-info list. */
1749 static void print_cpu_stall_info_end(void)
1750 {
1751         pr_err("\t");
1752 }
1753 
1754 /* Zero ->ticks_this_gp for all flavors of RCU. */
1755 static void zero_cpu_stall_ticks(struct rcu_data *rdp)
1756 {
1757         rdp->ticks_this_gp = 0;
1758         rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id());
1759 }
1760 
1761 /* Increment ->ticks_this_gp for all flavors of RCU. */
1762 static void increment_cpu_stall_ticks(void)
1763 {
1764         struct rcu_state *rsp;
1765 
1766         for_each_rcu_flavor(rsp)
1767                 raw_cpu_inc(rsp->rda->ticks_this_gp);
1768 }
1769 
1770 #ifdef CONFIG_RCU_NOCB_CPU
1771 
1772 /*
1773  * Offload callback processing from the boot-time-specified set of CPUs
1774  * specified by rcu_nocb_mask.  For each CPU in the set, there is a
1775  * kthread created that pulls the callbacks from the corresponding CPU,
1776  * waits for a grace period to elapse, and invokes the callbacks.
1777  * The no-CBs CPUs do a wake_up() on their kthread when they insert
1778  * a callback into any empty list, unless the rcu_nocb_poll boot parameter
1779  * has been specified, in which case each kthread actively polls its
1780  * CPU.  (Which isn't so great for energy efficiency, but which does
1781  * reduce RCU's overhead on that CPU.)
1782  *
1783  * This is intended to be used in conjunction with Frederic Weisbecker's
1784  * adaptive-idle work, which would seriously reduce OS jitter on CPUs
1785  * running CPU-bound user-mode computations.
1786  *
1787  * Offloading of callback processing could also in theory be used as
1788  * an energy-efficiency measure because CPUs with no RCU callbacks
1789  * queued are more aggressive about entering dyntick-idle mode.
1790  */
1791 
1792 
1793 /* Parse the boot-time rcu_nocb_mask CPU list from the kernel parameters. */
1794 static int __init rcu_nocb_setup(char *str)
1795 {
1796         alloc_bootmem_cpumask_var(&rcu_nocb_mask);
1797         have_rcu_nocb_mask = true;
1798         cpulist_parse(str, rcu_nocb_mask);
1799         return 1;
1800 }
1801 __setup("rcu_nocbs=", rcu_nocb_setup);
1802 
1803 static int __init parse_rcu_nocb_poll(char *arg)
1804 {
1805         rcu_nocb_poll = 1;
1806         return 0;
1807 }
1808 early_param("rcu_nocb_poll", parse_rcu_nocb_poll);
1809 
1810 /*
1811  * Wake up any no-CBs CPUs' kthreads that were waiting on the just-ended
1812  * grace period.
1813  */
1814 static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)
1815 {
1816         wake_up_all(&rnp->nocb_gp_wq[rnp->completed & 0x1]);
1817 }
1818 
1819 /*
1820  * Set the root rcu_node structure's ->need_future_gp field
1821  * based on the sum of those of all rcu_node structures.  This does
1822  * double-count the root rcu_node structure's requests, but this
1823  * is necessary to handle the possibility of a rcu_nocb_kthread()
1824  * having awakened during the time that the rcu_node structures
1825  * were being updated for the end of the previous grace period.
1826  */
1827 static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)
1828 {
1829         rnp->need_future_gp[(rnp->completed + 1) & 0x1] += nrq;
1830 }
1831 
1832 static void rcu_init_one_nocb(struct rcu_node *rnp)
1833 {
1834         init_waitqueue_head(&rnp->nocb_gp_wq[0]);
1835         init_waitqueue_head(&rnp->nocb_gp_wq[1]);
1836 }
1837 
1838 #ifndef CONFIG_RCU_NOCB_CPU_ALL
1839 /* Is the specified CPU a no-CBs CPU? */
1840 bool rcu_is_nocb_cpu(int cpu)
1841 {
1842         if (have_rcu_nocb_mask)
1843                 return cpumask_test_cpu(cpu, rcu_nocb_mask);
1844         return false;
1845 }
1846 #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
1847 
1848 /*
1849  * Kick the leader kthread for this NOCB group.
1850  */
1851 static void wake_nocb_leader(struct rcu_data *rdp, bool force)
1852 {
1853         struct rcu_data *rdp_leader = rdp->nocb_leader;
1854 
1855         if (!READ_ONCE(rdp_leader->nocb_kthread))
1856                 return;
1857         if (READ_ONCE(rdp_leader->nocb_leader_sleep) || force) {
1858                 /* Prior smp_mb__after_atomic() orders against prior enqueue. */
1859                 WRITE_ONCE(rdp_leader->nocb_leader_sleep, false);
1860                 wake_up(&rdp_leader->nocb_wq);
1861         }
1862 }
1863 
1864 /*
1865  * Does the specified CPU need an RCU callback for the specified flavor
1866  * of rcu_barrier()?
1867  */
1868 static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)
1869 {
1870         struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
1871         unsigned long ret;
1872 #ifdef CONFIG_PROVE_RCU
1873         struct rcu_head *rhp;
1874 #endif /* #ifdef CONFIG_PROVE_RCU */
1875 
1876         /*
1877          * Check count of all no-CBs callbacks awaiting invocation.
1878          * There needs to be a barrier before this function is called,
1879          * but associated with a prior determination that no more
1880          * callbacks would be posted.  In the worst case, the first
1881          * barrier in _rcu_barrier() suffices (but the caller cannot
1882          * necessarily rely on this, not a substitute for the caller
1883          * getting the concurrency design right!).  There must also be
1884          * a barrier between the following load an posting of a callback
1885          * (if a callback is in fact needed).  This is associated with an
1886          * atomic_inc() in the caller.
1887          */
1888         ret = atomic_long_read(&rdp->nocb_q_count);
1889 
1890 #ifdef CONFIG_PROVE_RCU
1891         rhp = READ_ONCE(rdp->nocb_head);
1892         if (!rhp)
1893                 rhp = READ_ONCE(rdp->nocb_gp_head);
1894         if (!rhp)
1895                 rhp = READ_ONCE(rdp->nocb_follower_head);
1896 
1897         /* Having no rcuo kthread but CBs after scheduler starts is bad! */
1898         if (!READ_ONCE(rdp->nocb_kthread) && rhp &&
1899             rcu_scheduler_fully_active) {
1900                 /* RCU callback enqueued before CPU first came online??? */
1901                 pr_err("RCU: Never-onlined no-CBs CPU %d has CB %p\n",
1902                        cpu, rhp->func);
1903                 WARN_ON_ONCE(1);
1904         }
1905 #endif /* #ifdef CONFIG_PROVE_RCU */
1906 
1907         return !!ret;
1908 }
1909 
1910 /*
1911  * Enqueue the specified string of rcu_head structures onto the specified
1912  * CPU's no-CBs lists.  The CPU is specified by rdp, the head of the
1913  * string by rhp, and the tail of the string by rhtp.  The non-lazy/lazy
1914  * counts are supplied by rhcount and rhcount_lazy.
1915  *
1916  * If warranted, also wake up the kthread servicing this CPUs queues.
1917  */
1918 static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,
1919                                     struct rcu_head *rhp,
1920                                     struct rcu_head **rhtp,
1921                                     int rhcount, int rhcount_lazy,
1922                                     unsigned long flags)
1923 {
1924         int len;
1925         struct rcu_head **old_rhpp;
1926         struct task_struct *t;
1927 
1928         /* Enqueue the callback on the nocb list and update counts. */
1929         atomic_long_add(rhcount, &rdp->nocb_q_count);
1930         /* rcu_barrier() relies on ->nocb_q_count add before xchg. */
1931         old_rhpp = xchg(&rdp->nocb_tail, rhtp);
1932         WRITE_ONCE(*old_rhpp, rhp);
1933         atomic_long_add(rhcount_lazy, &rdp->nocb_q_count_lazy);
1934         smp_mb__after_atomic(); /* Store *old_rhpp before _wake test. */
1935 
1936         /* If we are not being polled and there is a kthread, awaken it ... */
1937         t = READ_ONCE(rdp->nocb_kthread);
1938         if (rcu_nocb_poll || !t) {
1939                 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1940                                     TPS("WakeNotPoll"));
1941                 return;
1942         }
1943         len = atomic_long_read(&rdp->nocb_q_count);
1944         if (old_rhpp == &rdp->nocb_head) {
1945                 if (!irqs_disabled_flags(flags)) {
1946                         /* ... if queue was empty ... */
1947                         wake_nocb_leader(rdp, false);
1948                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1949                                             TPS("WakeEmpty"));
1950                 } else {
1951                         rdp->nocb_defer_wakeup = RCU_NOGP_WAKE;
1952                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1953                                             TPS("WakeEmptyIsDeferred"));
1954                 }
1955                 rdp->qlen_last_fqs_check = 0;
1956         } else if (len > rdp->qlen_last_fqs_check + qhimark) {
1957                 /* ... or if many callbacks queued. */
1958                 if (!irqs_disabled_flags(flags)) {
1959                         wake_nocb_leader(rdp, true);
1960                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1961                                             TPS("WakeOvf"));
1962                 } else {
1963                         rdp->nocb_defer_wakeup = RCU_NOGP_WAKE_FORCE;
1964                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1965                                             TPS("WakeOvfIsDeferred"));
1966                 }
1967                 rdp->qlen_last_fqs_check = LONG_MAX / 2;
1968         } else {
1969                 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WakeNot"));
1970         }
1971         return;
1972 }
1973 
1974 /*
1975  * This is a helper for __call_rcu(), which invokes this when the normal
1976  * callback queue is inoperable.  If this is not a no-CBs CPU, this
1977  * function returns failure back to __call_rcu(), which can complain
1978  * appropriately.
1979  *
1980  * Otherwise, this function queues the callback where the corresponding
1981  * "rcuo" kthread can find it.
1982  */
1983 static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
1984                             bool lazy, unsigned long flags)
1985 {
1986 
1987         if (!rcu_is_nocb_cpu(rdp->cpu))
1988                 return false;
1989         __call_rcu_nocb_enqueue(rdp, rhp, &rhp->next, 1, lazy, flags);
1990         if (__is_kfree_rcu_offset((unsigned long)rhp->func))
1991                 trace_rcu_kfree_callback(rdp->rsp->name, rhp,
1992                                          (unsigned long)rhp->func,
1993                                          -atomic_long_read(&rdp->nocb_q_count_lazy),
1994                                          -atomic_long_read(&rdp->nocb_q_count));
1995         else
1996                 trace_rcu_callback(rdp->rsp->name, rhp,
1997                                    -atomic_long_read(&rdp->nocb_q_count_lazy),
1998                                    -atomic_long_read(&rdp->nocb_q_count));
1999 
2000         /*
2001          * If called from an extended quiescent state with interrupts
2002          * disabled, invoke the RCU core in order to allow the idle-entry
2003          * deferred-wakeup check to function.
2004          */
2005         if (irqs_disabled_flags(flags) &&
2006             !rcu_is_watching() &&
2007             cpu_online(smp_processor_id()))
2008                 invoke_rcu_core();
2009 
2010         return true;
2011 }
2012 
2013 /*
2014  * Adopt orphaned callbacks on a no-CBs CPU, or return 0 if this is
2015  * not a no-CBs CPU.
2016  */
2017 static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
2018                                                      struct rcu_data *rdp,
2019                                                      unsigned long flags)
2020 {
2021         long ql = rsp->qlen;
2022         long qll = rsp->qlen_lazy;
2023 
2024         /* If this is not a no-CBs CPU, tell the caller to do it the old way. */
2025         if (!rcu_is_nocb_cpu(smp_processor_id()))
2026                 return false;
2027         rsp->qlen = 0;
2028         rsp->qlen_lazy = 0;
2029 
2030         /* First, enqueue the donelist, if any.  This preserves CB ordering. */
2031         if (rsp->orphan_donelist != NULL) {
2032                 __call_rcu_nocb_enqueue(rdp, rsp->orphan_donelist,
2033                                         rsp->orphan_donetail, ql, qll, flags);
2034                 ql = qll = 0;
2035                 rsp->orphan_donelist = NULL;
2036                 rsp->orphan_donetail = &rsp->orphan_donelist;
2037         }
2038         if (rsp->orphan_nxtlist != NULL) {
2039                 __call_rcu_nocb_enqueue(rdp, rsp->orphan_nxtlist,
2040                                         rsp->orphan_nxttail, ql, qll, flags);
2041                 ql = qll = 0;
2042                 rsp->orphan_nxtlist = NULL;
2043                 rsp->orphan_nxttail = &rsp->orphan_nxtlist;
2044         }
2045         return true;
2046 }
2047 
2048 /*
2049  * If necessary, kick off a new grace period, and either way wait
2050  * for a subsequent grace period to complete.
2051  */
2052 static void rcu_nocb_wait_gp(struct rcu_data *rdp)
2053 {
2054         unsigned long c;
2055         bool d;
2056         unsigned long flags;
2057         bool needwake;
2058         struct rcu_node *rnp = rdp->mynode;
2059 
2060         raw_spin_lock_irqsave_rcu_node(rnp, flags);
2061         needwake = rcu_start_future_gp(rnp, rdp, &c);
2062         raw_spin_unlock_irqrestore(&rnp->lock, flags);
2063         if (needwake)
2064                 rcu_gp_kthread_wake(rdp->rsp);
2065 
2066         /*
2067          * Wait for the grace period.  Do so interruptibly to avoid messing
2068          * up the load average.
2069          */
2070         trace_rcu_future_gp(rnp, rdp, c, TPS("StartWait"));
2071         for (;;) {
2072                 wait_event_interruptible(
2073                         rnp->nocb_gp_wq[c & 0x1],
2074                         (d = ULONG_CMP_GE(READ_ONCE(rnp->completed), c)));
2075                 if (likely(d))
2076                         break;
2077                 WARN_ON(signal_pending(current));
2078                 trace_rcu_future_gp(rnp, rdp, c, TPS("ResumeWait"));
2079         }
2080         trace_rcu_future_gp(rnp, rdp, c, TPS("EndWait"));
2081         smp_mb(); /* Ensure that CB invocation happens after GP end. */
2082 }
2083 
2084 /*
2085  * Leaders come here to wait for additional callbacks to show up.
2086  * This function does not return until callbacks appear.
2087  */
2088 static void nocb_leader_wait(struct rcu_data *my_rdp)
2089 {
2090         bool firsttime = true;
2091         bool gotcbs;
2092         struct rcu_data *rdp;
2093         struct rcu_head **tail;
2094 
2095 wait_again:
2096 
2097         /* Wait for callbacks to appear. */
2098         if (!rcu_nocb_poll) {
2099                 trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep");
2100                 wait_event_interruptible(my_rdp->nocb_wq,
2101                                 !READ_ONCE(my_rdp->nocb_leader_sleep));
2102                 /* Memory barrier handled by smp_mb() calls below and repoll. */
2103         } else if (firsttime) {
2104                 firsttime = false; /* Don't drown trace log with "Poll"! */
2105                 trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Poll");
2106         }
2107 
2108         /*
2109          * Each pass through the following loop checks a follower for CBs.
2110          * We are our own first follower.  Any CBs found are moved to
2111          * nocb_gp_head, where they await a grace period.
2112          */
2113         gotcbs = false;
2114         for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {
2115                 rdp->nocb_gp_head = READ_ONCE(rdp->nocb_head);
2116                 if (!rdp->nocb_gp_head)
2117                         continue;  /* No CBs here, try next follower. */
2118 
2119                 /* Move callbacks to wait-for-GP list, which is empty. */
2120                 WRITE_ONCE(rdp->nocb_head, NULL);
2121                 rdp->nocb_gp_tail = xchg(&rdp->nocb_tail, &rdp->nocb_head);
2122                 gotcbs = true;
2123         }
2124 
2125         /*
2126          * If there were no callbacks, sleep a bit, rescan after a
2127          * memory barrier, and go retry.
2128          */
2129         if (unlikely(!gotcbs)) {
2130                 if (!rcu_nocb_poll)
2131                         trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu,
2132                                             "WokeEmpty");
2133                 WARN_ON(signal_pending(current));
2134                 schedule_timeout_interruptible(1);
2135 
2136                 /* Rescan in case we were a victim of memory ordering. */
2137                 my_rdp->nocb_leader_sleep = true;
2138                 smp_mb();  /* Ensure _sleep true before scan. */
2139                 for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower)
2140                         if (READ_ONCE(rdp->nocb_head)) {
2141                                 /* Found CB, so short-circuit next wait. */
2142                                 my_rdp->nocb_leader_sleep = false;
2143                                 break;
2144                         }
2145                 goto wait_again;
2146         }
2147 
2148         /* Wait for one grace period. */
2149         rcu_nocb_wait_gp(my_rdp);
2150 
2151         /*
2152          * We left ->nocb_leader_sleep unset to reduce cache thrashing.
2153          * We set it now, but recheck for new callbacks while
2154          * traversing our follower list.
2155          */
2156         my_rdp->nocb_leader_sleep = true;
2157         smp_mb(); /* Ensure _sleep true before scan of ->nocb_head. */
2158 
2159         /* Each pass through the following loop wakes a follower, if needed. */
2160         for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {
2161                 if (READ_ONCE(rdp->nocb_head))
2162                         my_rdp->nocb_leader_sleep = false;/* No need to sleep.*/
2163                 if (!rdp->nocb_gp_head)
2164                         continue; /* No CBs, so no need to wake follower. */
2165 
2166                 /* Append callbacks to follower's "done" list. */
2167                 tail = xchg(&rdp->nocb_follower_tail, rdp->nocb_gp_tail);
2168                 *tail = rdp->nocb_gp_head;
2169                 smp_mb__after_atomic(); /* Store *tail before wakeup. */
2170                 if (rdp != my_rdp && tail == &rdp->nocb_follower_head) {
2171                         /*
2172                          * List was empty, wake up the follower.
2173                          * Memory barriers supplied by atomic_long_add().
2174                          */
2175                         wake_up(&rdp->nocb_wq);
2176                 }
2177         }
2178 
2179         /* If we (the leader) don't have CBs, go wait some more. */
2180         if (!my_rdp->nocb_follower_head)
2181                 goto wait_again;
2182 }
2183 
2184 /*
2185  * Followers come here to wait for additional callbacks to show up.
2186  * This function does not return until callbacks appear.
2187  */
2188 static void nocb_follower_wait(struct rcu_data *rdp)
2189 {
2190         bool firsttime = true;
2191 
2192         for (;;) {
2193                 if (!rcu_nocb_poll) {
2194                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2195                                             "FollowerSleep");
2196                         wait_event_interruptible(rdp->nocb_wq,
2197                                                  READ_ONCE(rdp->nocb_follower_head));
2198                 } else if (firsttime) {
2199                         /* Don't drown trace log with "Poll"! */
2200                         firsttime = false;
2201                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "Poll");
2202                 }
2203                 if (smp_load_acquire(&rdp->nocb_follower_head)) {
2204                         /* ^^^ Ensure CB invocation follows _head test. */
2205                         return;
2206                 }
2207                 if (!rcu_nocb_poll)
2208                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2209                                             "WokeEmpty");
2210                 WARN_ON(signal_pending(current));
2211                 schedule_timeout_interruptible(1);
2212         }
2213 }
2214 
2215 /*
2216  * Per-rcu_data kthread, but only for no-CBs CPUs.  Each kthread invokes
2217  * callbacks queued by the corresponding no-CBs CPU, however, there is
2218  * an optional leader-follower relationship so that the grace-period
2219  * kthreads don't have to do quite so many wakeups.
2220  */
2221 static int rcu_nocb_kthread(void *arg)
2222 {
2223         int c, cl;
2224         struct rcu_head *list;
2225         struct rcu_head *next;
2226         struct rcu_head **tail;
2227         struct rcu_data *rdp = arg;
2228 
2229         /* Each pass through this loop invokes one batch of callbacks */
2230         for (;;) {
2231                 /* Wait for callbacks. */
2232                 if (rdp->nocb_leader == rdp)
2233                         nocb_leader_wait(rdp);
2234                 else
2235                         nocb_follower_wait(rdp);
2236 
2237                 /* Pull the ready-to-invoke callbacks onto local list. */
2238                 list = READ_ONCE(rdp->nocb_follower_head);
2239                 BUG_ON(!list);
2240                 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "WokeNonEmpty");
2241                 WRITE_ONCE(rdp->nocb_follower_head, NULL);
2242                 tail = xchg(&rdp->nocb_follower_tail, &rdp->nocb_follower_head);
2243 
2244                 /* Each pass through the following loop invokes a callback. */
2245                 trace_rcu_batch_start(rdp->rsp->name,
2246                                       atomic_long_read(&rdp->nocb_q_count_lazy),
2247                                       atomic_long_read(&rdp->nocb_q_count), -1);
2248                 c = cl = 0;
2249                 while (list) {
2250                         next = list->next;
2251                         /* Wait for enqueuing to complete, if needed. */
2252                         while (next == NULL && &list->next != tail) {
2253                                 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2254                                                     TPS("WaitQueue"));
2255                                 schedule_timeout_interruptible(1);
2256                                 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2257                                                     TPS("WokeQueue"));
2258                                 next = list->next;
2259                         }
2260                         debug_rcu_head_unqueue(list);
2261                         local_bh_disable();
2262                         if (__rcu_reclaim(rdp->rsp->name, list))
2263                                 cl++;
2264                         c++;
2265                         local_bh_enable();
2266                         list = next;
2267                 }
2268                 trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1);
2269                 smp_mb__before_atomic();  /* _add after CB invocation. */
2270                 atomic_long_add(-c, &rdp->nocb_q_count);
2271                 atomic_long_add(-cl, &rdp->nocb_q_count_lazy);
2272                 rdp->n_nocbs_invoked += c;
2273         }
2274         return 0;
2275 }
2276 
2277 /* Is a deferred wakeup of rcu_nocb_kthread() required? */
2278 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
2279 {
2280         return READ_ONCE(rdp->nocb_defer_wakeup);
2281 }
2282 
2283 /* Do a deferred wakeup of rcu_nocb_kthread(). */
2284 static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
2285 {
2286         int ndw;
2287 
2288         if (!rcu_nocb_need_deferred_wakeup(rdp))
2289                 return;
2290         ndw = READ_ONCE(rdp->nocb_defer_wakeup);
2291         WRITE_ONCE(rdp->nocb_defer_wakeup, RCU_NOGP_WAKE_NOT);
2292         wake_nocb_leader(rdp, ndw == RCU_NOGP_WAKE_FORCE);
2293         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("DeferredWake"));
2294 }
2295 
2296 void __init rcu_init_nohz(void)
2297 {
2298         int cpu;
2299         bool need_rcu_nocb_mask = true;
2300         struct rcu_state *rsp;
2301 
2302 #ifdef CONFIG_RCU_NOCB_CPU_NONE
2303         need_rcu_nocb_mask = false;
2304 #endif /* #ifndef CONFIG_RCU_NOCB_CPU_NONE */
2305 
2306 #if defined(CONFIG_NO_HZ_FULL)
2307         if (tick_nohz_full_running && cpumask_weight(tick_nohz_full_mask))
2308                 need_rcu_nocb_mask = true;
2309 #endif /* #if defined(CONFIG_NO_HZ_FULL) */
2310 
2311         if (!have_rcu_nocb_mask && need_rcu_nocb_mask) {
2312                 if (!zalloc_cpumask_var(&rcu_nocb_mask, GFP_KERNEL)) {
2313                         pr_info("rcu_nocb_mask allocation failed, callback offloading disabled.\n");
2314                         return;
2315                 }
2316                 have_rcu_nocb_mask = true;
2317         }
2318         if (!have_rcu_nocb_mask)
2319                 return;
2320 
2321 #ifdef CONFIG_RCU_NOCB_CPU_ZERO
2322         pr_info("\tOffload RCU callbacks from CPU 0\n");
2323         cpumask_set_cpu(0, rcu_nocb_mask);
2324 #endif /* #ifdef CONFIG_RCU_NOCB_CPU_ZERO */
2325 #ifdef CONFIG_RCU_NOCB_CPU_ALL
2326         pr_info("\tOffload RCU callbacks from all CPUs\n");
2327         cpumask_copy(rcu_nocb_mask, cpu_possible_mask);
2328 #endif /* #ifdef CONFIG_RCU_NOCB_CPU_ALL */
2329 #if defined(CONFIG_NO_HZ_FULL)
2330         if (tick_nohz_full_running)
2331                 cpumask_or(rcu_nocb_mask, rcu_nocb_mask, tick_nohz_full_mask);
2332 #endif /* #if defined(CONFIG_NO_HZ_FULL) */
2333 
2334         if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) {
2335                 pr_info("\tNote: kernel parameter 'rcu_nocbs=' contains nonexistent CPUs.\n");
2336                 cpumask_and(rcu_nocb_mask, cpu_possible_mask,
2337                             rcu_nocb_mask);
2338         }
2339         pr_info("\tOffload RCU callbacks from CPUs: %*pbl.\n",
2340                 cpumask_pr_args(rcu_nocb_mask));
2341         if (rcu_nocb_poll)
2342                 pr_info("\tPoll for callbacks from no-CBs CPUs.\n");
2343 
2344         for_each_rcu_flavor(rsp) {
2345                 for_each_cpu(cpu, rcu_nocb_mask)
2346                         init_nocb_callback_list(per_cpu_ptr(rsp->rda, cpu));
2347                 rcu_organize_nocb_kthreads(rsp);
2348         }
2349 }
2350 
2351 /* Initialize per-rcu_data variables for no-CBs CPUs. */
2352 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
2353 {
2354         rdp->nocb_tail = &rdp->nocb_head;
2355         init_waitqueue_head(&rdp->nocb_wq);
2356         rdp->nocb_follower_tail = &rdp->nocb_follower_head;
2357 }
2358 
2359 /*
2360  * If the specified CPU is a no-CBs CPU that does not already have its
2361  * rcuo kthread for the specified RCU flavor, spawn it.  If the CPUs are
2362  * brought online out of order, this can require re-organizing the
2363  * leader-follower relationships.
2364  */
2365 static void rcu_spawn_one_nocb_kthread(struct rcu_state *rsp, int cpu)
2366 {
2367         struct rcu_data *rdp;
2368         struct rcu_data *rdp_last;
2369         struct rcu_data *rdp_old_leader;
2370         struct rcu_data *rdp_spawn = per_cpu_ptr(rsp->rda, cpu);
2371         struct task_struct *t;
2372 
2373         /*
2374          * If this isn't a no-CBs CPU or if it already has an rcuo kthread,
2375          * then nothing to do.
2376          */
2377         if (!rcu_is_nocb_cpu(cpu) || rdp_spawn->nocb_kthread)
2378                 return;
2379 
2380         /* If we didn't spawn the leader first, reorganize! */
2381         rdp_old_leader = rdp_spawn->nocb_leader;
2382         if (rdp_old_leader != rdp_spawn && !rdp_old_leader->nocb_kthread) {
2383                 rdp_last = NULL;
2384                 rdp = rdp_old_leader;
2385                 do {
2386                         rdp->nocb_leader = rdp_spawn;
2387                         if (rdp_last && rdp != rdp_spawn)
2388                                 rdp_last->nocb_next_follower = rdp;
2389                         if (rdp == rdp_spawn) {
2390                                 rdp = rdp->nocb_next_follower;
2391                         } else {
2392                                 rdp_last = rdp;
2393                                 rdp = rdp->nocb_next_follower;
2394                                 rdp_last->nocb_next_follower = NULL;
2395                         }
2396                 } while (rdp);
2397                 rdp_spawn->nocb_next_follower = rdp_old_leader;
2398         }
2399 
2400         /* Spawn the kthread for this CPU and RCU flavor. */
2401         t = kthread_run(rcu_nocb_kthread, rdp_spawn,
2402                         "rcuo%c/%d", rsp->abbr, cpu);
2403         BUG_ON(IS_ERR(t));
2404         WRITE_ONCE(rdp_spawn->nocb_kthread, t);
2405 }
2406 
2407 /*
2408  * If the specified CPU is a no-CBs CPU that does not already have its
2409  * rcuo kthreads, spawn them.
2410  */
2411 static void rcu_spawn_all_nocb_kthreads(int cpu)
2412 {
2413         struct rcu_state *rsp;
2414 
2415         if (rcu_scheduler_fully_active)
2416                 for_each_rcu_flavor(rsp)
2417                         rcu_spawn_one_nocb_kthread(rsp, cpu);
2418 }
2419 
2420 /*
2421  * Once the scheduler is running, spawn rcuo kthreads for all online
2422  * no-CBs CPUs.  This assumes that the early_initcall()s happen before
2423  * non-boot CPUs come online -- if this changes, we will need to add
2424  * some mutual exclusion.
2425  */
2426 static void __init rcu_spawn_nocb_kthreads(void)
2427 {
2428         int cpu;
2429 
2430         for_each_online_cpu(cpu)
2431                 rcu_spawn_all_nocb_kthreads(cpu);
2432 }
2433 
2434 /* How many follower CPU IDs per leader?  Default of -1 for sqrt(nr_cpu_ids). */
2435 static int rcu_nocb_leader_stride = -1;
2436 module_param(rcu_nocb_leader_stride, int, 0444);
2437 
2438 /*
2439  * Initialize leader-follower relationships for all no-CBs CPU.
2440  */
2441 static void __init rcu_organize_nocb_kthreads(struct rcu_state *rsp)
2442 {
2443         int cpu;
2444         int ls = rcu_nocb_leader_stride;
2445         int nl = 0;  /* Next leader. */
2446         struct rcu_data *rdp;
2447         struct rcu_data *rdp_leader = NULL;  /* Suppress misguided gcc warn. */
2448         struct rcu_data *rdp_prev = NULL;
2449 
2450         if (!have_rcu_nocb_mask)
2451                 return;
2452         if (ls == -1) {
2453                 ls = int_sqrt(nr_cpu_ids);
2454                 rcu_nocb_leader_stride = ls;
2455         }
2456 
2457         /*
2458          * Each pass through this loop sets up one rcu_data structure and
2459          * spawns one rcu_nocb_kthread().
2460          */
2461         for_each_cpu(cpu, rcu_nocb_mask) {
2462                 rdp = per_cpu_ptr(rsp->rda, cpu);
2463                 if (rdp->cpu >= nl) {
2464                         /* New leader, set up for followers & next leader. */
2465                         nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls;
2466                         rdp->nocb_leader = rdp;
2467                         rdp_leader = rdp;
2468                 } else {
2469                         /* Another follower, link to previous leader. */
2470                         rdp->nocb_leader = rdp_leader;
2471                         rdp_prev->nocb_next_follower = rdp;
2472                 }
2473                 rdp_prev = rdp;
2474         }
2475 }
2476 
2477 /* Prevent __call_rcu() from enqueuing callbacks on no-CBs CPUs */
2478 static bool init_nocb_callback_list(struct rcu_data *rdp)
2479 {
2480         if (!rcu_is_nocb_cpu(rdp->cpu))
2481                 return false;
2482 
2483         /* If there are early-boot callbacks, move them to nocb lists. */
2484         if (rdp->nxtlist) {
2485                 rdp->nocb_head = rdp->nxtlist;
2486                 rdp->nocb_tail = rdp->nxttail[RCU_NEXT_TAIL];
2487                 atomic_long_set(&rdp->nocb_q_count, rdp->qlen);
2488                 atomic_long_set(&rdp->nocb_q_count_lazy, rdp->qlen_lazy);
2489                 rdp->nxtlist = NULL;
2490                 rdp->qlen = 0;
2491                 rdp->qlen_lazy = 0;
2492         }
2493         rdp->nxttail[RCU_NEXT_TAIL] = NULL;
2494         return true;
2495 }
2496 
2497 #else /* #ifdef CONFIG_RCU_NOCB_CPU */
2498 
2499 static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)
2500 {
2501         WARN_ON_ONCE(1); /* Should be dead code. */
2502         return false;
2503 }
2504 
2505 static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)
2506 {
2507 }
2508 
2509 static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)
2510 {
2511 }
2512 
2513 static void rcu_init_one_nocb(struct rcu_node *rnp)
2514 {
2515 }
2516 
2517 static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
2518                             bool lazy, unsigned long flags)
2519 {
2520         return false;
2521 }
2522 
2523 static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
2524                                                      struct rcu_data *rdp,
2525                                                      unsigned long flags)
2526 {
2527         return false;
2528 }
2529 
2530 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
2531 {
2532 }
2533 
2534 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
2535 {
2536         return false;
2537 }
2538 
2539 static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
2540 {
2541 }
2542 
2543 static void rcu_spawn_all_nocb_kthreads(int cpu)
2544 {
2545 }
2546 
2547 static void __init rcu_spawn_nocb_kthreads(void)
2548 {
2549 }
2550 
2551 static bool init_nocb_callback_list(struct rcu_data *rdp)
2552 {
2553         return false;
2554 }
2555 
2556 #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
2557 
2558 /*
2559  * An adaptive-ticks CPU can potentially execute in kernel mode for an
2560  * arbitrarily long period of time with the scheduling-clock tick turned
2561  * off.  RCU will be paying attention to this CPU because it is in the
2562  * kernel, but the CPU cannot be guaranteed to be executing the RCU state
2563  * machine because the scheduling-clock tick has been disabled.  Therefore,
2564  * if an adaptive-ticks CPU is failing to respond to the current grace
2565  * period and has not be idle from an RCU perspective, kick it.
2566  */
2567 static void __maybe_unused rcu_kick_nohz_cpu(int cpu)
2568 {
2569 #ifdef CONFIG_NO_HZ_FULL
2570         if (tick_nohz_full_cpu(cpu))
2571                 smp_send_reschedule(cpu);
2572 #endif /* #ifdef CONFIG_NO_HZ_FULL */
2573 }
2574 
2575 
2576 #ifdef CONFIG_NO_HZ_FULL_SYSIDLE
2577 
2578 static int full_sysidle_state;          /* Current system-idle state. */
2579 #define RCU_SYSIDLE_NOT         0       /* Some CPU is not idle. */
2580 #define RCU_SYSIDLE_SHORT       1       /* All CPUs idle for brief period. */
2581 #define RCU_SYSIDLE_LONG        2       /* All CPUs idle for long enough. */
2582 #define RCU_SYSIDLE_FULL        3       /* All CPUs idle, ready for sysidle. */
2583 #define RCU_SYSIDLE_FULL_NOTED  4       /* Actually entered sysidle state. */
2584 
2585 /*
2586  * Invoked to note exit from irq or task transition to idle.  Note that
2587  * usermode execution does -not- count as idle here!  After all, we want
2588  * to detect full-system idle states, not RCU quiescent states and grace
2589  * periods.  The caller must have disabled interrupts.
2590  */
2591 static void rcu_sysidle_enter(int irq)
2592 {
2593         unsigned long j;
2594         struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
2595 
2596         /* If there are no nohz_full= CPUs, no need to track this. */
2597         if (!tick_nohz_full_enabled())
2598                 return;
2599 
2600         /* Adjust nesting, check for fully idle. */
2601         if (irq) {
2602                 rdtp->dynticks_idle_nesting--;
2603                 WARN_ON_ONCE(rdtp->dynticks_idle_nesting < 0);
2604                 if (rdtp->dynticks_idle_nesting != 0)
2605                         return;  /* Still not fully idle. */
2606         } else {
2607                 if ((rdtp->dynticks_idle_nesting & DYNTICK_TASK_NEST_MASK) ==
2608                     DYNTICK_TASK_NEST_VALUE) {
2609                         rdtp->dynticks_idle_nesting = 0;
2610                 } else {
2611                         rdtp->dynticks_idle_nesting -= DYNTICK_TASK_NEST_VALUE;
2612                         WARN_ON_ONCE(rdtp->dynticks_idle_nesting < 0);
2613                         return;  /* Still not fully idle. */
2614                 }
2615         }
2616 
2617         /* Record start of fully idle period. */
2618         j = jiffies;
2619         WRITE_ONCE(rdtp->dynticks_idle_jiffies, j);
2620         smp_mb__before_atomic();
2621         atomic_inc(&rdtp->dynticks_idle);
2622         smp_mb__after_atomic();
2623         WARN_ON_ONCE(atomic_read(&rdtp->dynticks_idle) & 0x1);
2624 }
2625 
2626 /*
2627  * Unconditionally force exit from full system-idle state.  This is
2628  * invoked when a normal CPU exits idle, but must be called separately
2629  * for the timekeeping CPU (tick_do_timer_cpu).  The reason for this
2630  * is that the timekeeping CPU is permitted to take scheduling-clock
2631  * interrupts while the system is in system-idle state, and of course
2632  * rcu_sysidle_exit() has no way of distinguishing a scheduling-clock
2633  * interrupt from any other type of interrupt.
2634  */
2635 void rcu_sysidle_force_exit(void)
2636 {
2637         int oldstate = READ_ONCE(full_sysidle_state);
2638         int newoldstate;
2639 
2640         /*
2641          * Each pass through the following loop attempts to exit full
2642          * system-idle state.  If contention proves to be a problem,
2643          * a trylock-based contention tree could be used here.
2644          */
2645         while (oldstate > RCU_SYSIDLE_SHORT) {
2646                 newoldstate = cmpxchg(&full_sysidle_state,
2647                                       oldstate, RCU_SYSIDLE_NOT);
2648                 if (oldstate == newoldstate &&
2649                     oldstate == RCU_SYSIDLE_FULL_NOTED) {
2650                         rcu_kick_nohz_cpu(tick_do_timer_cpu);
2651                         return; /* We cleared it, done! */
2652                 }
2653                 oldstate = newoldstate;
2654         }
2655         smp_mb(); /* Order initial oldstate fetch vs. later non-idle work. */
2656 }
2657 
2658 /*
2659  * Invoked to note entry to irq or task transition from idle.  Note that
2660  * usermode execution does -not- count as idle here!  The caller must
2661  * have disabled interrupts.
2662  */
2663 static void rcu_sysidle_exit(int irq)
2664 {
2665         struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
2666 
2667         /* If there are no nohz_full= CPUs, no need to track this. */
2668         if (!tick_nohz_full_enabled())
2669                 return;
2670 
2671         /* Adjust nesting, check for already non-idle. */
2672         if (irq) {
2673                 rdtp->dynticks_idle_nesting++;
2674                 WARN_ON_ONCE(rdtp->dynticks_idle_nesting <= 0);
2675                 if (rdtp->dynticks_idle_nesting != 1)
2676                         return; /* Already non-idle. */
2677         } else {
2678                 /*
2679                  * Allow for irq misnesting.  Yes, it really is possible
2680                  * to enter an irq handler then never leave it, and maybe
2681                  * also vice versa.  Handle both possibilities.
2682                  */
2683                 if (rdtp->dynticks_idle_nesting & DYNTICK_TASK_NEST_MASK) {
2684                         rdtp->dynticks_idle_nesting += DYNTICK_TASK_NEST_VALUE;
2685                         WARN_ON_ONCE(rdtp->dynticks_idle_nesting <= 0);
2686                         return; /* Already non-idle. */
2687                 } else {
2688                         rdtp->dynticks_idle_nesting = DYNTICK_TASK_EXIT_IDLE;
2689                 }
2690         }
2691 
2692         /* Record end of idle period. */
2693         smp_mb__before_atomic();
2694         atomic_inc(&rdtp->dynticks_idle);
2695         smp_mb__after_atomic();
2696         WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks_idle) & 0x1));
2697 
2698         /*
2699          * If we are the timekeeping CPU, we are permitted to be non-idle
2700          * during a system-idle state.  This must be the case, because
2701          * the timekeeping CPU has to take scheduling-clock interrupts
2702          * during the time that the system is transitioning to full
2703          * system-idle state.  This means that the timekeeping CPU must
2704          * invoke rcu_sysidle_force_exit() directly if it does anything
2705          * more than take a scheduling-clock interrupt.
2706          */
2707         if (smp_processor_id() == tick_do_timer_cpu)
2708                 return;
2709 
2710         /* Update system-idle state: We are clearly no longer fully idle! */
2711         rcu_sysidle_force_exit();
2712 }
2713 
2714 /*
2715  * Check to see if the current CPU is idle.  Note that usermode execution
2716  * does not count as idle.  The caller must have disabled interrupts,
2717  * and must be running on tick_do_timer_cpu.
2718  */
2719 static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
2720                                   unsigned long *maxj)
2721 {
2722         int cur;
2723         unsigned long j;
2724         struct rcu_dynticks *rdtp = rdp->dynticks;
2725 
2726         /* If there are no nohz_full= CPUs, don't check system-wide idleness. */
2727         if (!tick_nohz_full_enabled())
2728                 return;
2729 
2730         /*
2731          * If some other CPU has already reported non-idle, if this is
2732          * not the flavor of RCU that tracks sysidle state, or if this
2733          * is an offline or the timekeeping CPU, nothing to do.
2734          */
2735         if (!*isidle || rdp->rsp != rcu_state_p ||
2736             cpu_is_offline(rdp->cpu) || rdp->cpu == tick_do_timer_cpu)
2737                 return;
2738         /* Verify affinity of current kthread. */
2739         WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu);
2740 
2741         /* Pick up current idle and NMI-nesting counter and check. */
2742         cur = atomic_read(&rdtp->dynticks_idle);
2743         if (cur & 0x1) {
2744                 *isidle = false; /* We are not idle! */
2745                 return;
2746         }
2747         smp_mb(); /* Read counters before timestamps. */
2748 
2749         /* Pick up timestamps. */
2750         j = READ_ONCE(rdtp->dynticks_idle_jiffies);
2751         /* If this CPU entered idle more recently, update maxj timestamp. */
2752         if (ULONG_CMP_LT(*maxj, j))
2753                 *maxj = j;
2754 }
2755 
2756 /*
2757  * Is this the flavor of RCU that is handling full-system idle?
2758  */
2759 static bool is_sysidle_rcu_state(struct rcu_state *rsp)
2760 {
2761         return rsp == rcu_state_p;
2762 }
2763 
2764 /*
2765  * Return a delay in jiffies based on the number of CPUs, rcu_node
2766  * leaf fanout, and jiffies tick rate.  The idea is to allow larger
2767  * systems more time to transition to full-idle state in order to
2768  * avoid the cache thrashing that otherwise occur on the state variable.
2769  * Really small systems (less than a couple of tens of CPUs) should
2770  * instead use a single global atomically incremented counter, and later
2771  * versions of this will automatically reconfigure themselves accordingly.
2772  */
2773 static unsigned long rcu_sysidle_delay(void)
2774 {
2775         if (nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL)
2776                 return 0;
2777         return DIV_ROUND_UP(nr_cpu_ids * HZ, rcu_fanout_leaf * 1000);
2778 }
2779 
2780 /*
2781  * Advance the full-system-idle state.  This is invoked when all of
2782  * the non-timekeeping CPUs are idle.
2783  */
2784 static void rcu_sysidle(unsigned long j)
2785 {
2786         /* Check the current state. */
2787         switch (READ_ONCE(full_sysidle_state)) {
2788         case RCU_SYSIDLE_NOT:
2789 
2790                 /* First time all are idle, so note a short idle period. */
2791                 WRITE_ONCE(full_sysidle_state, RCU_SYSIDLE_SHORT);
2792                 break;
2793 
2794         case RCU_SYSIDLE_SHORT:
2795 
2796                 /*
2797                  * Idle for a bit, time to advance to next state?
2798                  * cmpxchg failure means race with non-idle, let them win.
2799                  */
2800                 if (ULONG_CMP_GE(jiffies, j + rcu_sysidle_delay()))
2801                         (void)cmpxchg(&full_sysidle_state,
2802                                       RCU_SYSIDLE_SHORT, RCU_SYSIDLE_LONG);
2803                 break;
2804 
2805         case RCU_SYSIDLE_LONG:
2806 
2807                 /*
2808                  * Do an additional check pass before advancing to full.
2809                  * cmpxchg failure means race with non-idle, let them win.
2810                  */
2811                 if (ULONG_CMP_GE(jiffies, j + rcu_sysidle_delay()))
2812                         (void)cmpxchg(&full_sysidle_state,
2813                                       RCU_SYSIDLE_LONG, RCU_SYSIDLE_FULL);
2814                 break;
2815 
2816         default:
2817                 break;
2818         }
2819 }
2820 
2821 /*
2822  * Found a non-idle non-timekeeping CPU, so kick the system-idle state
2823  * back to the beginning.
2824  */
2825 static void rcu_sysidle_cancel(void)
2826 {
2827         smp_mb();
2828         if (full_sysidle_state > RCU_SYSIDLE_SHORT)
2829                 WRITE_ONCE(full_sysidle_state, RCU_SYSIDLE_NOT);
2830 }
2831 
2832 /*
2833  * Update the sysidle state based on the results of a force-quiescent-state
2834  * scan of the CPUs' dyntick-idle state.
2835  */
2836 static void rcu_sysidle_report(struct rcu_state *rsp, int isidle,
2837                                unsigned long maxj, bool gpkt)
2838 {
2839         if (rsp != rcu_state_p)
2840                 return;  /* Wrong flavor, ignore. */
2841         if (gpkt && nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL)
2842                 return;  /* Running state machine from timekeeping CPU. */
2843         if (isidle)
2844                 rcu_sysidle(maxj);    /* More idle! */
2845         else
2846                 rcu_sysidle_cancel(); /* Idle is over. */
2847 }
2848 
2849 /*
2850  * Wrapper for rcu_sysidle_report() when called from the grace-period
2851  * kthread's context.
2852  */
2853 static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
2854                                   unsigned long maxj)
2855 {
2856         /* If there are no nohz_full= CPUs, no need to track this. */
2857         if (!tick_nohz_full_enabled())
2858                 return;
2859 
2860         rcu_sysidle_report(rsp, isidle, maxj, true);
2861 }
2862 
2863 /* Callback and function for forcing an RCU grace period. */
2864 struct rcu_sysidle_head {
2865         struct rcu_head rh;
2866         int inuse;
2867 };
2868 
2869 static void rcu_sysidle_cb(struct rcu_head *rhp)
2870 {
2871         struct rcu_sysidle_head *rshp;
2872 
2873         /*
2874          * The following memory barrier is needed to replace the
2875          * memory barriers that would normally be in the memory
2876          * allocator.
2877          */
2878         smp_mb();  /* grace period precedes setting inuse. */
2879 
2880         rshp = container_of(rhp, struct rcu_sysidle_head, rh);
2881         WRITE_ONCE(rshp->inuse, 0);
2882 }
2883 
2884 /*
2885  * Check to see if the system is fully idle, other than the timekeeping CPU.
2886  * The caller must have disabled interrupts.  This is not intended to be
2887  * called unless tick_nohz_full_enabled().
2888  */
2889 bool rcu_sys_is_idle(void)
2890 {
2891         static struct rcu_sysidle_head rsh;
2892         int rss = READ_ONCE(full_sysidle_state);
2893 
2894         if (WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu))
2895                 return false;
2896 
2897         /* Handle small-system case by doing a full scan of CPUs. */
2898         if (nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL) {
2899                 int oldrss = rss - 1;
2900 
2901                 /*
2902                  * One pass to advance to each state up to _FULL.
2903                  * Give up if any pass fails to advance the state.
2904                  */
2905                 while (rss < RCU_SYSIDLE_FULL && oldrss < rss) {
2906                         int cpu;
2907                         bool isidle = true;
2908                         unsigned long maxj = jiffies - ULONG_MAX / 4;
2909                         struct rcu_data *rdp;
2910 
2911                         /* Scan all the CPUs looking for nonidle CPUs. */
2912                         for_each_possible_cpu(cpu) {
2913                                 rdp = per_cpu_ptr(rcu_state_p->rda, cpu);
2914                                 rcu_sysidle_check_cpu(rdp, &isidle, &maxj);
2915                                 if (!isidle)
2916                                         break;
2917                         }
2918                         rcu_sysidle_report(rcu_state_p, isidle, maxj, false);
2919                         oldrss = rss;
2920                         rss = READ_ONCE(full_sysidle_state);
2921                 }
2922         }
2923 
2924         /* If this is the first observation of an idle period, record it. */
2925         if (rss == RCU_SYSIDLE_FULL) {
2926                 rss = cmpxchg(&full_sysidle_state,
2927                               RCU_SYSIDLE_FULL, RCU_SYSIDLE_FULL_NOTED);
2928                 return rss == RCU_SYSIDLE_FULL;
2929         }
2930 
2931         smp_mb(); /* ensure rss load happens before later caller actions. */
2932 
2933         /* If already fully idle, tell the caller (in case of races). */
2934         if (rss == RCU_SYSIDLE_FULL_NOTED)
2935                 return true;
2936 
2937         /*
2938          * If we aren't there yet, and a grace period is not in flight,
2939          * initiate a grace period.  Either way, tell the caller that
2940          * we are not there yet.  We use an xchg() rather than an assignment
2941          * to make up for the memory barriers that would otherwise be
2942          * provided by the memory allocator.
2943          */
2944         if (nr_cpu_ids > CONFIG_NO_HZ_FULL_SYSIDLE_SMALL &&
2945             !rcu_gp_in_progress(rcu_state_p) &&
2946             !rsh.inuse && xchg(&rsh.inuse, 1) == 0)
2947                 call_rcu(&rsh.rh, rcu_sysidle_cb);
2948         return false;
2949 }
2950 
2951 /*
2952  * Initialize dynticks sysidle state for CPUs coming online.
2953  */
2954 static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp)
2955 {
2956         rdtp->dynticks_idle_nesting = DYNTICK_TASK_NEST_VALUE;
2957 }
2958 
2959 #else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
2960 
2961 static void rcu_sysidle_enter(int irq)
2962 {
2963 }
2964 
2965 static void rcu_sysidle_exit(int irq)
2966 {
2967 }
2968 
2969 static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
2970                                   unsigned long *maxj)
2971 {
2972 }
2973 
2974 static bool is_sysidle_rcu_state(struct rcu_state *rsp)
2975 {
2976         return false;
2977 }
2978 
2979 static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
2980                                   unsigned long maxj)
2981 {
2982 }
2983 
2984 static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp)
2985 {
2986 }
2987 
2988 #endif /* #else #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
2989 
2990 /*
2991  * Is this CPU a NO_HZ_FULL CPU that should ignore RCU so that the
2992  * grace-period kthread will do force_quiescent_state() processing?
2993  * The idea is to avoid waking up RCU core processing on such a
2994  * CPU unless the grace period has extended for too long.
2995  *
2996  * This code relies on the fact that all NO_HZ_FULL CPUs are also
2997  * CONFIG_RCU_NOCB_CPU CPUs.
2998  */
2999 static bool rcu_nohz_full_cpu(struct rcu_state *rsp)
3000 {
3001 #ifdef CONFIG_NO_HZ_FULL
3002         if (tick_nohz_full_cpu(smp_processor_id()) &&
3003             (!rcu_gp_in_progress(rsp) ||
3004              ULONG_CMP_LT(jiffies, READ_ONCE(rsp->gp_start) + HZ)))
3005                 return true;
3006 #endif /* #ifdef CONFIG_NO_HZ_FULL */
3007         return false;
3008 }
3009 
3010 /*
3011  * Bind the grace-period kthread for the sysidle flavor of RCU to the
3012  * timekeeping CPU.
3013  */
3014 static void rcu_bind_gp_kthread(void)
3015 {
3016         int __maybe_unused cpu;
3017 
3018         if (!tick_nohz_full_enabled())
3019                 return;
3020 #ifdef CONFIG_NO_HZ_FULL_SYSIDLE
3021         cpu = tick_do_timer_cpu;
3022         if (cpu >= 0 && cpu < nr_cpu_ids)
3023                 set_cpus_allowed_ptr(current, cpumask_of(cpu));
3024 #else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
3025         housekeeping_affine(current);
3026 #endif /* #else #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
3027 }
3028 
3029 /* Record the current task on dyntick-idle entry. */
3030 static void rcu_dynticks_task_enter(void)
3031 {
3032 #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
3033         WRITE_ONCE(current->rcu_tasks_idle_cpu, smp_processor_id());
3034 #endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */
3035 }
3036 
3037 /* Record no current task on dyntick-idle exit. */
3038 static void rcu_dynticks_task_exit(void)
3039 {
3040 #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
3041         WRITE_ONCE(current->rcu_tasks_idle_cpu, -1);
3042 #endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */
3043 }
3044 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp