~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/rcu/tree_plugin.h

Version: ~ [ linux-5.1-rc5 ] ~ [ linux-5.0.7 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.34 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.111 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.168 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.178 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.138 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.65 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Read-Copy Update mechanism for mutual exclusion (tree-based version)
  3  * Internal non-public definitions that provide either classic
  4  * or preemptible semantics.
  5  *
  6  * This program is free software; you can redistribute it and/or modify
  7  * it under the terms of the GNU General Public License as published by
  8  * the Free Software Foundation; either version 2 of the License, or
  9  * (at your option) any later version.
 10  *
 11  * This program is distributed in the hope that it will be useful,
 12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 14  * GNU General Public License for more details.
 15  *
 16  * You should have received a copy of the GNU General Public License
 17  * along with this program; if not, you can access it online at
 18  * http://www.gnu.org/licenses/gpl-2.0.html.
 19  *
 20  * Copyright Red Hat, 2009
 21  * Copyright IBM Corporation, 2009
 22  *
 23  * Author: Ingo Molnar <mingo@elte.hu>
 24  *         Paul E. McKenney <paulmck@linux.vnet.ibm.com>
 25  */
 26 
 27 #include <linux/delay.h>
 28 #include <linux/gfp.h>
 29 #include <linux/oom.h>
 30 #include <linux/smpboot.h>
 31 #include "../time/tick-internal.h"
 32 
 33 #ifdef CONFIG_RCU_BOOST
 34 
 35 #include "../locking/rtmutex_common.h"
 36 
 37 /*
 38  * Control variables for per-CPU and per-rcu_node kthreads.  These
 39  * handle all flavors of RCU.
 40  */
 41 static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);
 42 DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
 43 DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
 44 DEFINE_PER_CPU(char, rcu_cpu_has_work);
 45 
 46 #else /* #ifdef CONFIG_RCU_BOOST */
 47 
 48 /*
 49  * Some architectures do not define rt_mutexes, but if !CONFIG_RCU_BOOST,
 50  * all uses are in dead code.  Provide a definition to keep the compiler
 51  * happy, but add WARN_ON_ONCE() to complain if used in the wrong place.
 52  * This probably needs to be excluded from -rt builds.
 53  */
 54 #define rt_mutex_owner(a) ({ WARN_ON_ONCE(1); NULL; })
 55 
 56 #endif /* #else #ifdef CONFIG_RCU_BOOST */
 57 
 58 #ifdef CONFIG_RCU_NOCB_CPU
 59 static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */
 60 static bool have_rcu_nocb_mask;     /* Was rcu_nocb_mask allocated? */
 61 static bool __read_mostly rcu_nocb_poll;    /* Offload kthread are to poll. */
 62 #endif /* #ifdef CONFIG_RCU_NOCB_CPU */
 63 
 64 /*
 65  * Check the RCU kernel configuration parameters and print informative
 66  * messages about anything out of the ordinary.
 67  */
 68 static void __init rcu_bootup_announce_oddness(void)
 69 {
 70         if (IS_ENABLED(CONFIG_RCU_TRACE))
 71                 pr_info("\tRCU debugfs-based tracing is enabled.\n");
 72         if ((IS_ENABLED(CONFIG_64BIT) && RCU_FANOUT != 64) ||
 73             (!IS_ENABLED(CONFIG_64BIT) && RCU_FANOUT != 32))
 74                 pr_info("\tCONFIG_RCU_FANOUT set to non-default value of %d\n",
 75                        RCU_FANOUT);
 76         if (rcu_fanout_exact)
 77                 pr_info("\tHierarchical RCU autobalancing is disabled.\n");
 78         if (IS_ENABLED(CONFIG_RCU_FAST_NO_HZ))
 79                 pr_info("\tRCU dyntick-idle grace-period acceleration is enabled.\n");
 80         if (IS_ENABLED(CONFIG_PROVE_RCU))
 81                 pr_info("\tRCU lockdep checking is enabled.\n");
 82         if (RCU_NUM_LVLS >= 4)
 83                 pr_info("\tFour(or more)-level hierarchy is enabled.\n");
 84         if (RCU_FANOUT_LEAF != 16)
 85                 pr_info("\tBuild-time adjustment of leaf fanout to %d.\n",
 86                         RCU_FANOUT_LEAF);
 87         if (rcu_fanout_leaf != RCU_FANOUT_LEAF)
 88                 pr_info("\tBoot-time adjustment of leaf fanout to %d.\n", rcu_fanout_leaf);
 89         if (nr_cpu_ids != NR_CPUS)
 90                 pr_info("\tRCU restricting CPUs from NR_CPUS=%d to nr_cpu_ids=%d.\n", NR_CPUS, nr_cpu_ids);
 91         if (IS_ENABLED(CONFIG_RCU_BOOST))
 92                 pr_info("\tRCU kthread priority: %d.\n", kthread_prio);
 93 }
 94 
 95 #ifdef CONFIG_PREEMPT_RCU
 96 
 97 RCU_STATE_INITIALIZER(rcu_preempt, 'p', call_rcu);
 98 static struct rcu_state *const rcu_state_p = &rcu_preempt_state;
 99 static struct rcu_data __percpu *const rcu_data_p = &rcu_preempt_data;
100 
101 static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
102                                bool wake);
103 
104 /*
105  * Tell them what RCU they are running.
106  */
107 static void __init rcu_bootup_announce(void)
108 {
109         pr_info("Preemptible hierarchical RCU implementation.\n");
110         rcu_bootup_announce_oddness();
111 }
112 
113 /* Flags for rcu_preempt_ctxt_queue() decision table. */
114 #define RCU_GP_TASKS    0x8
115 #define RCU_EXP_TASKS   0x4
116 #define RCU_GP_BLKD     0x2
117 #define RCU_EXP_BLKD    0x1
118 
119 /*
120  * Queues a task preempted within an RCU-preempt read-side critical
121  * section into the appropriate location within the ->blkd_tasks list,
122  * depending on the states of any ongoing normal and expedited grace
123  * periods.  The ->gp_tasks pointer indicates which element the normal
124  * grace period is waiting on (NULL if none), and the ->exp_tasks pointer
125  * indicates which element the expedited grace period is waiting on (again,
126  * NULL if none).  If a grace period is waiting on a given element in the
127  * ->blkd_tasks list, it also waits on all subsequent elements.  Thus,
128  * adding a task to the tail of the list blocks any grace period that is
129  * already waiting on one of the elements.  In contrast, adding a task
130  * to the head of the list won't block any grace period that is already
131  * waiting on one of the elements.
132  *
133  * This queuing is imprecise, and can sometimes make an ongoing grace
134  * period wait for a task that is not strictly speaking blocking it.
135  * Given the choice, we needlessly block a normal grace period rather than
136  * blocking an expedited grace period.
137  *
138  * Note that an endless sequence of expedited grace periods still cannot
139  * indefinitely postpone a normal grace period.  Eventually, all of the
140  * fixed number of preempted tasks blocking the normal grace period that are
141  * not also blocking the expedited grace period will resume and complete
142  * their RCU read-side critical sections.  At that point, the ->gp_tasks
143  * pointer will equal the ->exp_tasks pointer, at which point the end of
144  * the corresponding expedited grace period will also be the end of the
145  * normal grace period.
146  */
147 static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp)
148         __releases(rnp->lock) /* But leaves rrupts disabled. */
149 {
150         int blkd_state = (rnp->gp_tasks ? RCU_GP_TASKS : 0) +
151                          (rnp->exp_tasks ? RCU_EXP_TASKS : 0) +
152                          (rnp->qsmask & rdp->grpmask ? RCU_GP_BLKD : 0) +
153                          (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0);
154         struct task_struct *t = current;
155 
156         /*
157          * Decide where to queue the newly blocked task.  In theory,
158          * this could be an if-statement.  In practice, when I tried
159          * that, it was quite messy.
160          */
161         switch (blkd_state) {
162         case 0:
163         case                RCU_EXP_TASKS:
164         case                RCU_EXP_TASKS + RCU_GP_BLKD:
165         case RCU_GP_TASKS:
166         case RCU_GP_TASKS + RCU_EXP_TASKS:
167 
168                 /*
169                  * Blocking neither GP, or first task blocking the normal
170                  * GP but not blocking the already-waiting expedited GP.
171                  * Queue at the head of the list to avoid unnecessarily
172                  * blocking the already-waiting GPs.
173                  */
174                 list_add(&t->rcu_node_entry, &rnp->blkd_tasks);
175                 break;
176 
177         case                                              RCU_EXP_BLKD:
178         case                                RCU_GP_BLKD:
179         case                                RCU_GP_BLKD + RCU_EXP_BLKD:
180         case RCU_GP_TASKS +                               RCU_EXP_BLKD:
181         case RCU_GP_TASKS +                 RCU_GP_BLKD + RCU_EXP_BLKD:
182         case RCU_GP_TASKS + RCU_EXP_TASKS + RCU_GP_BLKD + RCU_EXP_BLKD:
183 
184                 /*
185                  * First task arriving that blocks either GP, or first task
186                  * arriving that blocks the expedited GP (with the normal
187                  * GP already waiting), or a task arriving that blocks
188                  * both GPs with both GPs already waiting.  Queue at the
189                  * tail of the list to avoid any GP waiting on any of the
190                  * already queued tasks that are not blocking it.
191                  */
192                 list_add_tail(&t->rcu_node_entry, &rnp->blkd_tasks);
193                 break;
194 
195         case                RCU_EXP_TASKS +               RCU_EXP_BLKD:
196         case                RCU_EXP_TASKS + RCU_GP_BLKD + RCU_EXP_BLKD:
197         case RCU_GP_TASKS + RCU_EXP_TASKS +               RCU_EXP_BLKD:
198 
199                 /*
200                  * Second or subsequent task blocking the expedited GP.
201                  * The task either does not block the normal GP, or is the
202                  * first task blocking the normal GP.  Queue just after
203                  * the first task blocking the expedited GP.
204                  */
205                 list_add(&t->rcu_node_entry, rnp->exp_tasks);
206                 break;
207 
208         case RCU_GP_TASKS +                 RCU_GP_BLKD:
209         case RCU_GP_TASKS + RCU_EXP_TASKS + RCU_GP_BLKD:
210 
211                 /*
212                  * Second or subsequent task blocking the normal GP.
213                  * The task does not block the expedited GP. Queue just
214                  * after the first task blocking the normal GP.
215                  */
216                 list_add(&t->rcu_node_entry, rnp->gp_tasks);
217                 break;
218 
219         default:
220 
221                 /* Yet another exercise in excessive paranoia. */
222                 WARN_ON_ONCE(1);
223                 break;
224         }
225 
226         /*
227          * We have now queued the task.  If it was the first one to
228          * block either grace period, update the ->gp_tasks and/or
229          * ->exp_tasks pointers, respectively, to reference the newly
230          * blocked tasks.
231          */
232         if (!rnp->gp_tasks && (blkd_state & RCU_GP_BLKD))
233                 rnp->gp_tasks = &t->rcu_node_entry;
234         if (!rnp->exp_tasks && (blkd_state & RCU_EXP_BLKD))
235                 rnp->exp_tasks = &t->rcu_node_entry;
236         raw_spin_unlock_rcu_node(rnp); /* interrupts remain disabled. */
237 
238         /*
239          * Report the quiescent state for the expedited GP.  This expedited
240          * GP should not be able to end until we report, so there should be
241          * no need to check for a subsequent expedited GP.  (Though we are
242          * still in a quiescent state in any case.)
243          */
244         if (blkd_state & RCU_EXP_BLKD &&
245             t->rcu_read_unlock_special.b.exp_need_qs) {
246                 t->rcu_read_unlock_special.b.exp_need_qs = false;
247                 rcu_report_exp_rdp(rdp->rsp, rdp, true);
248         } else {
249                 WARN_ON_ONCE(t->rcu_read_unlock_special.b.exp_need_qs);
250         }
251 }
252 
253 /*
254  * Record a preemptible-RCU quiescent state for the specified CPU.  Note
255  * that this just means that the task currently running on the CPU is
256  * not in a quiescent state.  There might be any number of tasks blocked
257  * while in an RCU read-side critical section.
258  *
259  * As with the other rcu_*_qs() functions, callers to this function
260  * must disable preemption.
261  */
262 static void rcu_preempt_qs(void)
263 {
264         if (__this_cpu_read(rcu_data_p->cpu_no_qs.s)) {
265                 trace_rcu_grace_period(TPS("rcu_preempt"),
266                                        __this_cpu_read(rcu_data_p->gpnum),
267                                        TPS("cpuqs"));
268                 __this_cpu_write(rcu_data_p->cpu_no_qs.b.norm, false);
269                 barrier(); /* Coordinate with rcu_preempt_check_callbacks(). */
270                 current->rcu_read_unlock_special.b.need_qs = false;
271         }
272 }
273 
274 /*
275  * We have entered the scheduler, and the current task might soon be
276  * context-switched away from.  If this task is in an RCU read-side
277  * critical section, we will no longer be able to rely on the CPU to
278  * record that fact, so we enqueue the task on the blkd_tasks list.
279  * The task will dequeue itself when it exits the outermost enclosing
280  * RCU read-side critical section.  Therefore, the current grace period
281  * cannot be permitted to complete until the blkd_tasks list entries
282  * predating the current grace period drain, in other words, until
283  * rnp->gp_tasks becomes NULL.
284  *
285  * Caller must disable interrupts.
286  */
287 static void rcu_preempt_note_context_switch(void)
288 {
289         struct task_struct *t = current;
290         struct rcu_data *rdp;
291         struct rcu_node *rnp;
292 
293         if (t->rcu_read_lock_nesting > 0 &&
294             !t->rcu_read_unlock_special.b.blocked) {
295 
296                 /* Possibly blocking in an RCU read-side critical section. */
297                 rdp = this_cpu_ptr(rcu_state_p->rda);
298                 rnp = rdp->mynode;
299                 raw_spin_lock_rcu_node(rnp);
300                 t->rcu_read_unlock_special.b.blocked = true;
301                 t->rcu_blocked_node = rnp;
302 
303                 /*
304                  * Verify the CPU's sanity, trace the preemption, and
305                  * then queue the task as required based on the states
306                  * of any ongoing and expedited grace periods.
307                  */
308                 WARN_ON_ONCE((rdp->grpmask & rcu_rnp_online_cpus(rnp)) == 0);
309                 WARN_ON_ONCE(!list_empty(&t->rcu_node_entry));
310                 trace_rcu_preempt_task(rdp->rsp->name,
311                                        t->pid,
312                                        (rnp->qsmask & rdp->grpmask)
313                                        ? rnp->gpnum
314                                        : rnp->gpnum + 1);
315                 rcu_preempt_ctxt_queue(rnp, rdp);
316         } else if (t->rcu_read_lock_nesting < 0 &&
317                    t->rcu_read_unlock_special.s) {
318 
319                 /*
320                  * Complete exit from RCU read-side critical section on
321                  * behalf of preempted instance of __rcu_read_unlock().
322                  */
323                 rcu_read_unlock_special(t);
324         }
325 
326         /*
327          * Either we were not in an RCU read-side critical section to
328          * begin with, or we have now recorded that critical section
329          * globally.  Either way, we can now note a quiescent state
330          * for this CPU.  Again, if we were in an RCU read-side critical
331          * section, and if that critical section was blocking the current
332          * grace period, then the fact that the task has been enqueued
333          * means that we continue to block the current grace period.
334          */
335         rcu_preempt_qs();
336 }
337 
338 /*
339  * Check for preempted RCU readers blocking the current grace period
340  * for the specified rcu_node structure.  If the caller needs a reliable
341  * answer, it must hold the rcu_node's ->lock.
342  */
343 static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
344 {
345         return rnp->gp_tasks != NULL;
346 }
347 
348 /*
349  * Advance a ->blkd_tasks-list pointer to the next entry, instead
350  * returning NULL if at the end of the list.
351  */
352 static struct list_head *rcu_next_node_entry(struct task_struct *t,
353                                              struct rcu_node *rnp)
354 {
355         struct list_head *np;
356 
357         np = t->rcu_node_entry.next;
358         if (np == &rnp->blkd_tasks)
359                 np = NULL;
360         return np;
361 }
362 
363 /*
364  * Return true if the specified rcu_node structure has tasks that were
365  * preempted within an RCU read-side critical section.
366  */
367 static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
368 {
369         return !list_empty(&rnp->blkd_tasks);
370 }
371 
372 /*
373  * Handle special cases during rcu_read_unlock(), such as needing to
374  * notify RCU core processing or task having blocked during the RCU
375  * read-side critical section.
376  */
377 void rcu_read_unlock_special(struct task_struct *t)
378 {
379         bool empty_exp;
380         bool empty_norm;
381         bool empty_exp_now;
382         unsigned long flags;
383         struct list_head *np;
384         bool drop_boost_mutex = false;
385         struct rcu_data *rdp;
386         struct rcu_node *rnp;
387         union rcu_special special;
388 
389         /* NMI handlers cannot block and cannot safely manipulate state. */
390         if (in_nmi())
391                 return;
392 
393         local_irq_save(flags);
394 
395         /*
396          * If RCU core is waiting for this CPU to exit its critical section,
397          * report the fact that it has exited.  Because irqs are disabled,
398          * t->rcu_read_unlock_special cannot change.
399          */
400         special = t->rcu_read_unlock_special;
401         if (special.b.need_qs) {
402                 rcu_preempt_qs();
403                 t->rcu_read_unlock_special.b.need_qs = false;
404                 if (!t->rcu_read_unlock_special.s) {
405                         local_irq_restore(flags);
406                         return;
407                 }
408         }
409 
410         /*
411          * Respond to a request for an expedited grace period, but only if
412          * we were not preempted, meaning that we were running on the same
413          * CPU throughout.  If we were preempted, the exp_need_qs flag
414          * would have been cleared at the time of the first preemption,
415          * and the quiescent state would be reported when we were dequeued.
416          */
417         if (special.b.exp_need_qs) {
418                 WARN_ON_ONCE(special.b.blocked);
419                 t->rcu_read_unlock_special.b.exp_need_qs = false;
420                 rdp = this_cpu_ptr(rcu_state_p->rda);
421                 rcu_report_exp_rdp(rcu_state_p, rdp, true);
422                 if (!t->rcu_read_unlock_special.s) {
423                         local_irq_restore(flags);
424                         return;
425                 }
426         }
427 
428         /* Hardware IRQ handlers cannot block, complain if they get here. */
429         if (in_irq() || in_serving_softirq()) {
430                 lockdep_rcu_suspicious(__FILE__, __LINE__,
431                                        "rcu_read_unlock() from irq or softirq with blocking in critical section!!!\n");
432                 pr_alert("->rcu_read_unlock_special: %#x (b: %d, enq: %d nq: %d)\n",
433                          t->rcu_read_unlock_special.s,
434                          t->rcu_read_unlock_special.b.blocked,
435                          t->rcu_read_unlock_special.b.exp_need_qs,
436                          t->rcu_read_unlock_special.b.need_qs);
437                 local_irq_restore(flags);
438                 return;
439         }
440 
441         /* Clean up if blocked during RCU read-side critical section. */
442         if (special.b.blocked) {
443                 t->rcu_read_unlock_special.b.blocked = false;
444 
445                 /*
446                  * Remove this task from the list it blocked on.  The task
447                  * now remains queued on the rcu_node corresponding to the
448                  * CPU it first blocked on, so there is no longer any need
449                  * to loop.  Retain a WARN_ON_ONCE() out of sheer paranoia.
450                  */
451                 rnp = t->rcu_blocked_node;
452                 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
453                 WARN_ON_ONCE(rnp != t->rcu_blocked_node);
454                 empty_norm = !rcu_preempt_blocked_readers_cgp(rnp);
455                 empty_exp = sync_rcu_preempt_exp_done(rnp);
456                 smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */
457                 np = rcu_next_node_entry(t, rnp);
458                 list_del_init(&t->rcu_node_entry);
459                 t->rcu_blocked_node = NULL;
460                 trace_rcu_unlock_preempted_task(TPS("rcu_preempt"),
461                                                 rnp->gpnum, t->pid);
462                 if (&t->rcu_node_entry == rnp->gp_tasks)
463                         rnp->gp_tasks = np;
464                 if (&t->rcu_node_entry == rnp->exp_tasks)
465                         rnp->exp_tasks = np;
466                 if (IS_ENABLED(CONFIG_RCU_BOOST)) {
467                         if (&t->rcu_node_entry == rnp->boost_tasks)
468                                 rnp->boost_tasks = np;
469                         /* Snapshot ->boost_mtx ownership w/rnp->lock held. */
470                         drop_boost_mutex = rt_mutex_owner(&rnp->boost_mtx) == t;
471                 }
472 
473                 /*
474                  * If this was the last task on the current list, and if
475                  * we aren't waiting on any CPUs, report the quiescent state.
476                  * Note that rcu_report_unblock_qs_rnp() releases rnp->lock,
477                  * so we must take a snapshot of the expedited state.
478                  */
479                 empty_exp_now = sync_rcu_preempt_exp_done(rnp);
480                 if (!empty_norm && !rcu_preempt_blocked_readers_cgp(rnp)) {
481                         trace_rcu_quiescent_state_report(TPS("preempt_rcu"),
482                                                          rnp->gpnum,
483                                                          0, rnp->qsmask,
484                                                          rnp->level,
485                                                          rnp->grplo,
486                                                          rnp->grphi,
487                                                          !!rnp->gp_tasks);
488                         rcu_report_unblock_qs_rnp(rcu_state_p, rnp, flags);
489                 } else {
490                         raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
491                 }
492 
493                 /* Unboost if we were boosted. */
494                 if (IS_ENABLED(CONFIG_RCU_BOOST) && drop_boost_mutex)
495                         rt_mutex_unlock(&rnp->boost_mtx);
496 
497                 /*
498                  * If this was the last task on the expedited lists,
499                  * then we need to report up the rcu_node hierarchy.
500                  */
501                 if (!empty_exp && empty_exp_now)
502                         rcu_report_exp_rnp(rcu_state_p, rnp, true);
503         } else {
504                 local_irq_restore(flags);
505         }
506 }
507 
508 /*
509  * Dump detailed information for all tasks blocking the current RCU
510  * grace period on the specified rcu_node structure.
511  */
512 static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
513 {
514         unsigned long flags;
515         struct task_struct *t;
516 
517         raw_spin_lock_irqsave_rcu_node(rnp, flags);
518         if (!rcu_preempt_blocked_readers_cgp(rnp)) {
519                 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
520                 return;
521         }
522         t = list_entry(rnp->gp_tasks->prev,
523                        struct task_struct, rcu_node_entry);
524         list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry)
525                 sched_show_task(t);
526         raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
527 }
528 
529 /*
530  * Dump detailed information for all tasks blocking the current RCU
531  * grace period.
532  */
533 static void rcu_print_detail_task_stall(struct rcu_state *rsp)
534 {
535         struct rcu_node *rnp = rcu_get_root(rsp);
536 
537         rcu_print_detail_task_stall_rnp(rnp);
538         rcu_for_each_leaf_node(rsp, rnp)
539                 rcu_print_detail_task_stall_rnp(rnp);
540 }
541 
542 static void rcu_print_task_stall_begin(struct rcu_node *rnp)
543 {
544         pr_err("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):",
545                rnp->level, rnp->grplo, rnp->grphi);
546 }
547 
548 static void rcu_print_task_stall_end(void)
549 {
550         pr_cont("\n");
551 }
552 
553 /*
554  * Scan the current list of tasks blocked within RCU read-side critical
555  * sections, printing out the tid of each.
556  */
557 static int rcu_print_task_stall(struct rcu_node *rnp)
558 {
559         struct task_struct *t;
560         int ndetected = 0;
561 
562         if (!rcu_preempt_blocked_readers_cgp(rnp))
563                 return 0;
564         rcu_print_task_stall_begin(rnp);
565         t = list_entry(rnp->gp_tasks->prev,
566                        struct task_struct, rcu_node_entry);
567         list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
568                 pr_cont(" P%d", t->pid);
569                 ndetected++;
570         }
571         rcu_print_task_stall_end();
572         return ndetected;
573 }
574 
575 /*
576  * Scan the current list of tasks blocked within RCU read-side critical
577  * sections, printing out the tid of each that is blocking the current
578  * expedited grace period.
579  */
580 static int rcu_print_task_exp_stall(struct rcu_node *rnp)
581 {
582         struct task_struct *t;
583         int ndetected = 0;
584 
585         if (!rnp->exp_tasks)
586                 return 0;
587         t = list_entry(rnp->exp_tasks->prev,
588                        struct task_struct, rcu_node_entry);
589         list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
590                 pr_cont(" P%d", t->pid);
591                 ndetected++;
592         }
593         return ndetected;
594 }
595 
596 /*
597  * Check that the list of blocked tasks for the newly completed grace
598  * period is in fact empty.  It is a serious bug to complete a grace
599  * period that still has RCU readers blocked!  This function must be
600  * invoked -before- updating this rnp's ->gpnum, and the rnp's ->lock
601  * must be held by the caller.
602  *
603  * Also, if there are blocked tasks on the list, they automatically
604  * block the newly created grace period, so set up ->gp_tasks accordingly.
605  */
606 static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
607 {
608         WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp));
609         if (rcu_preempt_has_tasks(rnp))
610                 rnp->gp_tasks = rnp->blkd_tasks.next;
611         WARN_ON_ONCE(rnp->qsmask);
612 }
613 
614 /*
615  * Check for a quiescent state from the current CPU.  When a task blocks,
616  * the task is recorded in the corresponding CPU's rcu_node structure,
617  * which is checked elsewhere.
618  *
619  * Caller must disable hard irqs.
620  */
621 static void rcu_preempt_check_callbacks(void)
622 {
623         struct task_struct *t = current;
624 
625         if (t->rcu_read_lock_nesting == 0) {
626                 rcu_preempt_qs();
627                 return;
628         }
629         if (t->rcu_read_lock_nesting > 0 &&
630             __this_cpu_read(rcu_data_p->core_needs_qs) &&
631             __this_cpu_read(rcu_data_p->cpu_no_qs.b.norm))
632                 t->rcu_read_unlock_special.b.need_qs = true;
633 }
634 
635 #ifdef CONFIG_RCU_BOOST
636 
637 static void rcu_preempt_do_callbacks(void)
638 {
639         rcu_do_batch(rcu_state_p, this_cpu_ptr(rcu_data_p));
640 }
641 
642 #endif /* #ifdef CONFIG_RCU_BOOST */
643 
644 /*
645  * Queue a preemptible-RCU callback for invocation after a grace period.
646  */
647 void call_rcu(struct rcu_head *head, rcu_callback_t func)
648 {
649         __call_rcu(head, func, rcu_state_p, -1, 0);
650 }
651 EXPORT_SYMBOL_GPL(call_rcu);
652 
653 /**
654  * synchronize_rcu - wait until a grace period has elapsed.
655  *
656  * Control will return to the caller some time after a full grace
657  * period has elapsed, in other words after all currently executing RCU
658  * read-side critical sections have completed.  Note, however, that
659  * upon return from synchronize_rcu(), the caller might well be executing
660  * concurrently with new RCU read-side critical sections that began while
661  * synchronize_rcu() was waiting.  RCU read-side critical sections are
662  * delimited by rcu_read_lock() and rcu_read_unlock(), and may be nested.
663  *
664  * See the description of synchronize_sched() for more detailed information
665  * on memory ordering guarantees.
666  */
667 void synchronize_rcu(void)
668 {
669         RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
670                          lock_is_held(&rcu_lock_map) ||
671                          lock_is_held(&rcu_sched_lock_map),
672                          "Illegal synchronize_rcu() in RCU read-side critical section");
673         if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE)
674                 return;
675         if (rcu_gp_is_expedited())
676                 synchronize_rcu_expedited();
677         else
678                 wait_rcu_gp(call_rcu);
679 }
680 EXPORT_SYMBOL_GPL(synchronize_rcu);
681 
682 /**
683  * rcu_barrier - Wait until all in-flight call_rcu() callbacks complete.
684  *
685  * Note that this primitive does not necessarily wait for an RCU grace period
686  * to complete.  For example, if there are no RCU callbacks queued anywhere
687  * in the system, then rcu_barrier() is within its rights to return
688  * immediately, without waiting for anything, much less an RCU grace period.
689  */
690 void rcu_barrier(void)
691 {
692         _rcu_barrier(rcu_state_p);
693 }
694 EXPORT_SYMBOL_GPL(rcu_barrier);
695 
696 /*
697  * Initialize preemptible RCU's state structures.
698  */
699 static void __init __rcu_init_preempt(void)
700 {
701         rcu_init_one(rcu_state_p);
702 }
703 
704 /*
705  * Check for a task exiting while in a preemptible-RCU read-side
706  * critical section, clean up if so.  No need to issue warnings,
707  * as debug_check_no_locks_held() already does this if lockdep
708  * is enabled.
709  */
710 void exit_rcu(void)
711 {
712         struct task_struct *t = current;
713 
714         if (likely(list_empty(&current->rcu_node_entry)))
715                 return;
716         t->rcu_read_lock_nesting = 1;
717         barrier();
718         t->rcu_read_unlock_special.b.blocked = true;
719         __rcu_read_unlock();
720 }
721 
722 #else /* #ifdef CONFIG_PREEMPT_RCU */
723 
724 static struct rcu_state *const rcu_state_p = &rcu_sched_state;
725 
726 /*
727  * Tell them what RCU they are running.
728  */
729 static void __init rcu_bootup_announce(void)
730 {
731         pr_info("Hierarchical RCU implementation.\n");
732         rcu_bootup_announce_oddness();
733 }
734 
735 /*
736  * Because preemptible RCU does not exist, we never have to check for
737  * CPUs being in quiescent states.
738  */
739 static void rcu_preempt_note_context_switch(void)
740 {
741 }
742 
743 /*
744  * Because preemptible RCU does not exist, there are never any preempted
745  * RCU readers.
746  */
747 static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
748 {
749         return 0;
750 }
751 
752 /*
753  * Because there is no preemptible RCU, there can be no readers blocked.
754  */
755 static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
756 {
757         return false;
758 }
759 
760 /*
761  * Because preemptible RCU does not exist, we never have to check for
762  * tasks blocked within RCU read-side critical sections.
763  */
764 static void rcu_print_detail_task_stall(struct rcu_state *rsp)
765 {
766 }
767 
768 /*
769  * Because preemptible RCU does not exist, we never have to check for
770  * tasks blocked within RCU read-side critical sections.
771  */
772 static int rcu_print_task_stall(struct rcu_node *rnp)
773 {
774         return 0;
775 }
776 
777 /*
778  * Because preemptible RCU does not exist, we never have to check for
779  * tasks blocked within RCU read-side critical sections that are
780  * blocking the current expedited grace period.
781  */
782 static int rcu_print_task_exp_stall(struct rcu_node *rnp)
783 {
784         return 0;
785 }
786 
787 /*
788  * Because there is no preemptible RCU, there can be no readers blocked,
789  * so there is no need to check for blocked tasks.  So check only for
790  * bogus qsmask values.
791  */
792 static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
793 {
794         WARN_ON_ONCE(rnp->qsmask);
795 }
796 
797 /*
798  * Because preemptible RCU does not exist, it never has any callbacks
799  * to check.
800  */
801 static void rcu_preempt_check_callbacks(void)
802 {
803 }
804 
805 /*
806  * Because preemptible RCU does not exist, rcu_barrier() is just
807  * another name for rcu_barrier_sched().
808  */
809 void rcu_barrier(void)
810 {
811         rcu_barrier_sched();
812 }
813 EXPORT_SYMBOL_GPL(rcu_barrier);
814 
815 /*
816  * Because preemptible RCU does not exist, it need not be initialized.
817  */
818 static void __init __rcu_init_preempt(void)
819 {
820 }
821 
822 /*
823  * Because preemptible RCU does not exist, tasks cannot possibly exit
824  * while in preemptible RCU read-side critical sections.
825  */
826 void exit_rcu(void)
827 {
828 }
829 
830 #endif /* #else #ifdef CONFIG_PREEMPT_RCU */
831 
832 #ifdef CONFIG_RCU_BOOST
833 
834 #include "../locking/rtmutex_common.h"
835 
836 #ifdef CONFIG_RCU_TRACE
837 
838 static void rcu_initiate_boost_trace(struct rcu_node *rnp)
839 {
840         if (!rcu_preempt_has_tasks(rnp))
841                 rnp->n_balk_blkd_tasks++;
842         else if (rnp->exp_tasks == NULL && rnp->gp_tasks == NULL)
843                 rnp->n_balk_exp_gp_tasks++;
844         else if (rnp->gp_tasks != NULL && rnp->boost_tasks != NULL)
845                 rnp->n_balk_boost_tasks++;
846         else if (rnp->gp_tasks != NULL && rnp->qsmask != 0)
847                 rnp->n_balk_notblocked++;
848         else if (rnp->gp_tasks != NULL &&
849                  ULONG_CMP_LT(jiffies, rnp->boost_time))
850                 rnp->n_balk_notyet++;
851         else
852                 rnp->n_balk_nos++;
853 }
854 
855 #else /* #ifdef CONFIG_RCU_TRACE */
856 
857 static void rcu_initiate_boost_trace(struct rcu_node *rnp)
858 {
859 }
860 
861 #endif /* #else #ifdef CONFIG_RCU_TRACE */
862 
863 static void rcu_wake_cond(struct task_struct *t, int status)
864 {
865         /*
866          * If the thread is yielding, only wake it when this
867          * is invoked from idle
868          */
869         if (status != RCU_KTHREAD_YIELDING || is_idle_task(current))
870                 wake_up_process(t);
871 }
872 
873 /*
874  * Carry out RCU priority boosting on the task indicated by ->exp_tasks
875  * or ->boost_tasks, advancing the pointer to the next task in the
876  * ->blkd_tasks list.
877  *
878  * Note that irqs must be enabled: boosting the task can block.
879  * Returns 1 if there are more tasks needing to be boosted.
880  */
881 static int rcu_boost(struct rcu_node *rnp)
882 {
883         unsigned long flags;
884         struct task_struct *t;
885         struct list_head *tb;
886 
887         if (READ_ONCE(rnp->exp_tasks) == NULL &&
888             READ_ONCE(rnp->boost_tasks) == NULL)
889                 return 0;  /* Nothing left to boost. */
890 
891         raw_spin_lock_irqsave_rcu_node(rnp, flags);
892 
893         /*
894          * Recheck under the lock: all tasks in need of boosting
895          * might exit their RCU read-side critical sections on their own.
896          */
897         if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL) {
898                 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
899                 return 0;
900         }
901 
902         /*
903          * Preferentially boost tasks blocking expedited grace periods.
904          * This cannot starve the normal grace periods because a second
905          * expedited grace period must boost all blocked tasks, including
906          * those blocking the pre-existing normal grace period.
907          */
908         if (rnp->exp_tasks != NULL) {
909                 tb = rnp->exp_tasks;
910                 rnp->n_exp_boosts++;
911         } else {
912                 tb = rnp->boost_tasks;
913                 rnp->n_normal_boosts++;
914         }
915         rnp->n_tasks_boosted++;
916 
917         /*
918          * We boost task t by manufacturing an rt_mutex that appears to
919          * be held by task t.  We leave a pointer to that rt_mutex where
920          * task t can find it, and task t will release the mutex when it
921          * exits its outermost RCU read-side critical section.  Then
922          * simply acquiring this artificial rt_mutex will boost task
923          * t's priority.  (Thanks to tglx for suggesting this approach!)
924          *
925          * Note that task t must acquire rnp->lock to remove itself from
926          * the ->blkd_tasks list, which it will do from exit() if from
927          * nowhere else.  We therefore are guaranteed that task t will
928          * stay around at least until we drop rnp->lock.  Note that
929          * rnp->lock also resolves races between our priority boosting
930          * and task t's exiting its outermost RCU read-side critical
931          * section.
932          */
933         t = container_of(tb, struct task_struct, rcu_node_entry);
934         rt_mutex_init_proxy_locked(&rnp->boost_mtx, t);
935         raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
936         /* Lock only for side effect: boosts task t's priority. */
937         rt_mutex_lock(&rnp->boost_mtx);
938         rt_mutex_unlock(&rnp->boost_mtx);  /* Then keep lockdep happy. */
939 
940         return READ_ONCE(rnp->exp_tasks) != NULL ||
941                READ_ONCE(rnp->boost_tasks) != NULL;
942 }
943 
944 /*
945  * Priority-boosting kthread, one per leaf rcu_node.
946  */
947 static int rcu_boost_kthread(void *arg)
948 {
949         struct rcu_node *rnp = (struct rcu_node *)arg;
950         int spincnt = 0;
951         int more2boost;
952 
953         trace_rcu_utilization(TPS("Start boost kthread@init"));
954         for (;;) {
955                 rnp->boost_kthread_status = RCU_KTHREAD_WAITING;
956                 trace_rcu_utilization(TPS("End boost kthread@rcu_wait"));
957                 rcu_wait(rnp->boost_tasks || rnp->exp_tasks);
958                 trace_rcu_utilization(TPS("Start boost kthread@rcu_wait"));
959                 rnp->boost_kthread_status = RCU_KTHREAD_RUNNING;
960                 more2boost = rcu_boost(rnp);
961                 if (more2boost)
962                         spincnt++;
963                 else
964                         spincnt = 0;
965                 if (spincnt > 10) {
966                         rnp->boost_kthread_status = RCU_KTHREAD_YIELDING;
967                         trace_rcu_utilization(TPS("End boost kthread@rcu_yield"));
968                         schedule_timeout_interruptible(2);
969                         trace_rcu_utilization(TPS("Start boost kthread@rcu_yield"));
970                         spincnt = 0;
971                 }
972         }
973         /* NOTREACHED */
974         trace_rcu_utilization(TPS("End boost kthread@notreached"));
975         return 0;
976 }
977 
978 /*
979  * Check to see if it is time to start boosting RCU readers that are
980  * blocking the current grace period, and, if so, tell the per-rcu_node
981  * kthread to start boosting them.  If there is an expedited grace
982  * period in progress, it is always time to boost.
983  *
984  * The caller must hold rnp->lock, which this function releases.
985  * The ->boost_kthread_task is immortal, so we don't need to worry
986  * about it going away.
987  */
988 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
989         __releases(rnp->lock)
990 {
991         struct task_struct *t;
992 
993         if (!rcu_preempt_blocked_readers_cgp(rnp) && rnp->exp_tasks == NULL) {
994                 rnp->n_balk_exp_gp_tasks++;
995                 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
996                 return;
997         }
998         if (rnp->exp_tasks != NULL ||
999             (rnp->gp_tasks != NULL &&
1000              rnp->boost_tasks == NULL &&
1001              rnp->qsmask == 0 &&
1002              ULONG_CMP_GE(jiffies, rnp->boost_time))) {
1003                 if (rnp->exp_tasks == NULL)
1004                         rnp->boost_tasks = rnp->gp_tasks;
1005                 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1006                 t = rnp->boost_kthread_task;
1007                 if (t)
1008                         rcu_wake_cond(t, rnp->boost_kthread_status);
1009         } else {
1010                 rcu_initiate_boost_trace(rnp);
1011                 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1012         }
1013 }
1014 
1015 /*
1016  * Wake up the per-CPU kthread to invoke RCU callbacks.
1017  */
1018 static void invoke_rcu_callbacks_kthread(void)
1019 {
1020         unsigned long flags;
1021 
1022         local_irq_save(flags);
1023         __this_cpu_write(rcu_cpu_has_work, 1);
1024         if (__this_cpu_read(rcu_cpu_kthread_task) != NULL &&
1025             current != __this_cpu_read(rcu_cpu_kthread_task)) {
1026                 rcu_wake_cond(__this_cpu_read(rcu_cpu_kthread_task),
1027                               __this_cpu_read(rcu_cpu_kthread_status));
1028         }
1029         local_irq_restore(flags);
1030 }
1031 
1032 /*
1033  * Is the current CPU running the RCU-callbacks kthread?
1034  * Caller must have preemption disabled.
1035  */
1036 static bool rcu_is_callbacks_kthread(void)
1037 {
1038         return __this_cpu_read(rcu_cpu_kthread_task) == current;
1039 }
1040 
1041 #define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)
1042 
1043 /*
1044  * Do priority-boost accounting for the start of a new grace period.
1045  */
1046 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1047 {
1048         rnp->boost_time = jiffies + RCU_BOOST_DELAY_JIFFIES;
1049 }
1050 
1051 /*
1052  * Create an RCU-boost kthread for the specified node if one does not
1053  * already exist.  We only create this kthread for preemptible RCU.
1054  * Returns zero if all is well, a negated errno otherwise.
1055  */
1056 static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
1057                                        struct rcu_node *rnp)
1058 {
1059         int rnp_index = rnp - &rsp->node[0];
1060         unsigned long flags;
1061         struct sched_param sp;
1062         struct task_struct *t;
1063 
1064         if (rcu_state_p != rsp)
1065                 return 0;
1066 
1067         if (!rcu_scheduler_fully_active || rcu_rnp_online_cpus(rnp) == 0)
1068                 return 0;
1069 
1070         rsp->boost = 1;
1071         if (rnp->boost_kthread_task != NULL)
1072                 return 0;
1073         t = kthread_create(rcu_boost_kthread, (void *)rnp,
1074                            "rcub/%d", rnp_index);
1075         if (IS_ERR(t))
1076                 return PTR_ERR(t);
1077         raw_spin_lock_irqsave_rcu_node(rnp, flags);
1078         rnp->boost_kthread_task = t;
1079         raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1080         sp.sched_priority = kthread_prio;
1081         sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
1082         wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */
1083         return 0;
1084 }
1085 
1086 static void rcu_kthread_do_work(void)
1087 {
1088         rcu_do_batch(&rcu_sched_state, this_cpu_ptr(&rcu_sched_data));
1089         rcu_do_batch(&rcu_bh_state, this_cpu_ptr(&rcu_bh_data));
1090         rcu_preempt_do_callbacks();
1091 }
1092 
1093 static void rcu_cpu_kthread_setup(unsigned int cpu)
1094 {
1095         struct sched_param sp;
1096 
1097         sp.sched_priority = kthread_prio;
1098         sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1099 }
1100 
1101 static void rcu_cpu_kthread_park(unsigned int cpu)
1102 {
1103         per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
1104 }
1105 
1106 static int rcu_cpu_kthread_should_run(unsigned int cpu)
1107 {
1108         return __this_cpu_read(rcu_cpu_has_work);
1109 }
1110 
1111 /*
1112  * Per-CPU kernel thread that invokes RCU callbacks.  This replaces the
1113  * RCU softirq used in flavors and configurations of RCU that do not
1114  * support RCU priority boosting.
1115  */
1116 static void rcu_cpu_kthread(unsigned int cpu)
1117 {
1118         unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status);
1119         char work, *workp = this_cpu_ptr(&rcu_cpu_has_work);
1120         int spincnt;
1121 
1122         for (spincnt = 0; spincnt < 10; spincnt++) {
1123                 trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait"));
1124                 local_bh_disable();
1125                 *statusp = RCU_KTHREAD_RUNNING;
1126                 this_cpu_inc(rcu_cpu_kthread_loops);
1127                 local_irq_disable();
1128                 work = *workp;
1129                 *workp = 0;
1130                 local_irq_enable();
1131                 if (work)
1132                         rcu_kthread_do_work();
1133                 local_bh_enable();
1134                 if (*workp == 0) {
1135                         trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
1136                         *statusp = RCU_KTHREAD_WAITING;
1137                         return;
1138                 }
1139         }
1140         *statusp = RCU_KTHREAD_YIELDING;
1141         trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
1142         schedule_timeout_interruptible(2);
1143         trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
1144         *statusp = RCU_KTHREAD_WAITING;
1145 }
1146 
1147 /*
1148  * Set the per-rcu_node kthread's affinity to cover all CPUs that are
1149  * served by the rcu_node in question.  The CPU hotplug lock is still
1150  * held, so the value of rnp->qsmaskinit will be stable.
1151  *
1152  * We don't include outgoingcpu in the affinity set, use -1 if there is
1153  * no outgoing CPU.  If there are no CPUs left in the affinity set,
1154  * this function allows the kthread to execute on any CPU.
1155  */
1156 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1157 {
1158         struct task_struct *t = rnp->boost_kthread_task;
1159         unsigned long mask = rcu_rnp_online_cpus(rnp);
1160         cpumask_var_t cm;
1161         int cpu;
1162 
1163         if (!t)
1164                 return;
1165         if (!zalloc_cpumask_var(&cm, GFP_KERNEL))
1166                 return;
1167         for_each_leaf_node_possible_cpu(rnp, cpu)
1168                 if ((mask & leaf_node_cpu_bit(rnp, cpu)) &&
1169                     cpu != outgoingcpu)
1170                         cpumask_set_cpu(cpu, cm);
1171         if (cpumask_weight(cm) == 0)
1172                 cpumask_setall(cm);
1173         set_cpus_allowed_ptr(t, cm);
1174         free_cpumask_var(cm);
1175 }
1176 
1177 static struct smp_hotplug_thread rcu_cpu_thread_spec = {
1178         .store                  = &rcu_cpu_kthread_task,
1179         .thread_should_run      = rcu_cpu_kthread_should_run,
1180         .thread_fn              = rcu_cpu_kthread,
1181         .thread_comm            = "rcuc/%u",
1182         .setup                  = rcu_cpu_kthread_setup,
1183         .park                   = rcu_cpu_kthread_park,
1184 };
1185 
1186 /*
1187  * Spawn boost kthreads -- called as soon as the scheduler is running.
1188  */
1189 static void __init rcu_spawn_boost_kthreads(void)
1190 {
1191         struct rcu_node *rnp;
1192         int cpu;
1193 
1194         for_each_possible_cpu(cpu)
1195                 per_cpu(rcu_cpu_has_work, cpu) = 0;
1196         BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec));
1197         rcu_for_each_leaf_node(rcu_state_p, rnp)
1198                 (void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp);
1199 }
1200 
1201 static void rcu_prepare_kthreads(int cpu)
1202 {
1203         struct rcu_data *rdp = per_cpu_ptr(rcu_state_p->rda, cpu);
1204         struct rcu_node *rnp = rdp->mynode;
1205 
1206         /* Fire up the incoming CPU's kthread and leaf rcu_node kthread. */
1207         if (rcu_scheduler_fully_active)
1208                 (void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp);
1209 }
1210 
1211 #else /* #ifdef CONFIG_RCU_BOOST */
1212 
1213 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
1214         __releases(rnp->lock)
1215 {
1216         raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1217 }
1218 
1219 static void invoke_rcu_callbacks_kthread(void)
1220 {
1221         WARN_ON_ONCE(1);
1222 }
1223 
1224 static bool rcu_is_callbacks_kthread(void)
1225 {
1226         return false;
1227 }
1228 
1229 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1230 {
1231 }
1232 
1233 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1234 {
1235 }
1236 
1237 static void __init rcu_spawn_boost_kthreads(void)
1238 {
1239 }
1240 
1241 static void rcu_prepare_kthreads(int cpu)
1242 {
1243 }
1244 
1245 #endif /* #else #ifdef CONFIG_RCU_BOOST */
1246 
1247 #if !defined(CONFIG_RCU_FAST_NO_HZ)
1248 
1249 /*
1250  * Check to see if any future RCU-related work will need to be done
1251  * by the current CPU, even if none need be done immediately, returning
1252  * 1 if so.  This function is part of the RCU implementation; it is -not-
1253  * an exported member of the RCU API.
1254  *
1255  * Because we not have RCU_FAST_NO_HZ, just check whether this CPU needs
1256  * any flavor of RCU.
1257  */
1258 int rcu_needs_cpu(u64 basemono, u64 *nextevt)
1259 {
1260         *nextevt = KTIME_MAX;
1261         return IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL)
1262                ? 0 : rcu_cpu_has_callbacks(NULL);
1263 }
1264 
1265 /*
1266  * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
1267  * after it.
1268  */
1269 static void rcu_cleanup_after_idle(void)
1270 {
1271 }
1272 
1273 /*
1274  * Do the idle-entry grace-period work, which, because CONFIG_RCU_FAST_NO_HZ=n,
1275  * is nothing.
1276  */
1277 static void rcu_prepare_for_idle(void)
1278 {
1279 }
1280 
1281 /*
1282  * Don't bother keeping a running count of the number of RCU callbacks
1283  * posted because CONFIG_RCU_FAST_NO_HZ=n.
1284  */
1285 static void rcu_idle_count_callbacks_posted(void)
1286 {
1287 }
1288 
1289 #else /* #if !defined(CONFIG_RCU_FAST_NO_HZ) */
1290 
1291 /*
1292  * This code is invoked when a CPU goes idle, at which point we want
1293  * to have the CPU do everything required for RCU so that it can enter
1294  * the energy-efficient dyntick-idle mode.  This is handled by a
1295  * state machine implemented by rcu_prepare_for_idle() below.
1296  *
1297  * The following three proprocessor symbols control this state machine:
1298  *
1299  * RCU_IDLE_GP_DELAY gives the number of jiffies that a CPU is permitted
1300  *      to sleep in dyntick-idle mode with RCU callbacks pending.  This
1301  *      is sized to be roughly one RCU grace period.  Those energy-efficiency
1302  *      benchmarkers who might otherwise be tempted to set this to a large
1303  *      number, be warned: Setting RCU_IDLE_GP_DELAY too high can hang your
1304  *      system.  And if you are -that- concerned about energy efficiency,
1305  *      just power the system down and be done with it!
1306  * RCU_IDLE_LAZY_GP_DELAY gives the number of jiffies that a CPU is
1307  *      permitted to sleep in dyntick-idle mode with only lazy RCU
1308  *      callbacks pending.  Setting this too high can OOM your system.
1309  *
1310  * The values below work well in practice.  If future workloads require
1311  * adjustment, they can be converted into kernel config parameters, though
1312  * making the state machine smarter might be a better option.
1313  */
1314 #define RCU_IDLE_GP_DELAY 4             /* Roughly one grace period. */
1315 #define RCU_IDLE_LAZY_GP_DELAY (6 * HZ) /* Roughly six seconds. */
1316 
1317 static int rcu_idle_gp_delay = RCU_IDLE_GP_DELAY;
1318 module_param(rcu_idle_gp_delay, int, 0644);
1319 static int rcu_idle_lazy_gp_delay = RCU_IDLE_LAZY_GP_DELAY;
1320 module_param(rcu_idle_lazy_gp_delay, int, 0644);
1321 
1322 /*
1323  * Try to advance callbacks for all flavors of RCU on the current CPU, but
1324  * only if it has been awhile since the last time we did so.  Afterwards,
1325  * if there are any callbacks ready for immediate invocation, return true.
1326  */
1327 static bool __maybe_unused rcu_try_advance_all_cbs(void)
1328 {
1329         bool cbs_ready = false;
1330         struct rcu_data *rdp;
1331         struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1332         struct rcu_node *rnp;
1333         struct rcu_state *rsp;
1334 
1335         /* Exit early if we advanced recently. */
1336         if (jiffies == rdtp->last_advance_all)
1337                 return false;
1338         rdtp->last_advance_all = jiffies;
1339 
1340         for_each_rcu_flavor(rsp) {
1341                 rdp = this_cpu_ptr(rsp->rda);
1342                 rnp = rdp->mynode;
1343 
1344                 /*
1345                  * Don't bother checking unless a grace period has
1346                  * completed since we last checked and there are
1347                  * callbacks not yet ready to invoke.
1348                  */
1349                 if ((rdp->completed != rnp->completed ||
1350                      unlikely(READ_ONCE(rdp->gpwrap))) &&
1351                     rdp->nxttail[RCU_DONE_TAIL] != rdp->nxttail[RCU_NEXT_TAIL])
1352                         note_gp_changes(rsp, rdp);
1353 
1354                 if (cpu_has_callbacks_ready_to_invoke(rdp))
1355                         cbs_ready = true;
1356         }
1357         return cbs_ready;
1358 }
1359 
1360 /*
1361  * Allow the CPU to enter dyntick-idle mode unless it has callbacks ready
1362  * to invoke.  If the CPU has callbacks, try to advance them.  Tell the
1363  * caller to set the timeout based on whether or not there are non-lazy
1364  * callbacks.
1365  *
1366  * The caller must have disabled interrupts.
1367  */
1368 int rcu_needs_cpu(u64 basemono, u64 *nextevt)
1369 {
1370         struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1371         unsigned long dj;
1372 
1373         if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL)) {
1374                 *nextevt = KTIME_MAX;
1375                 return 0;
1376         }
1377 
1378         /* Snapshot to detect later posting of non-lazy callback. */
1379         rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
1380 
1381         /* If no callbacks, RCU doesn't need the CPU. */
1382         if (!rcu_cpu_has_callbacks(&rdtp->all_lazy)) {
1383                 *nextevt = KTIME_MAX;
1384                 return 0;
1385         }
1386 
1387         /* Attempt to advance callbacks. */
1388         if (rcu_try_advance_all_cbs()) {
1389                 /* Some ready to invoke, so initiate later invocation. */
1390                 invoke_rcu_core();
1391                 return 1;
1392         }
1393         rdtp->last_accelerate = jiffies;
1394 
1395         /* Request timer delay depending on laziness, and round. */
1396         if (!rdtp->all_lazy) {
1397                 dj = round_up(rcu_idle_gp_delay + jiffies,
1398                                rcu_idle_gp_delay) - jiffies;
1399         } else {
1400                 dj = round_jiffies(rcu_idle_lazy_gp_delay + jiffies) - jiffies;
1401         }
1402         *nextevt = basemono + dj * TICK_NSEC;
1403         return 0;
1404 }
1405 
1406 /*
1407  * Prepare a CPU for idle from an RCU perspective.  The first major task
1408  * is to sense whether nohz mode has been enabled or disabled via sysfs.
1409  * The second major task is to check to see if a non-lazy callback has
1410  * arrived at a CPU that previously had only lazy callbacks.  The third
1411  * major task is to accelerate (that is, assign grace-period numbers to)
1412  * any recently arrived callbacks.
1413  *
1414  * The caller must have disabled interrupts.
1415  */
1416 static void rcu_prepare_for_idle(void)
1417 {
1418         bool needwake;
1419         struct rcu_data *rdp;
1420         struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1421         struct rcu_node *rnp;
1422         struct rcu_state *rsp;
1423         int tne;
1424 
1425         if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL) ||
1426             rcu_is_nocb_cpu(smp_processor_id()))
1427                 return;
1428 
1429         /* Handle nohz enablement switches conservatively. */
1430         tne = READ_ONCE(tick_nohz_active);
1431         if (tne != rdtp->tick_nohz_enabled_snap) {
1432                 if (rcu_cpu_has_callbacks(NULL))
1433                         invoke_rcu_core(); /* force nohz to see update. */
1434                 rdtp->tick_nohz_enabled_snap = tne;
1435                 return;
1436         }
1437         if (!tne)
1438                 return;
1439 
1440         /*
1441          * If a non-lazy callback arrived at a CPU having only lazy
1442          * callbacks, invoke RCU core for the side-effect of recalculating
1443          * idle duration on re-entry to idle.
1444          */
1445         if (rdtp->all_lazy &&
1446             rdtp->nonlazy_posted != rdtp->nonlazy_posted_snap) {
1447                 rdtp->all_lazy = false;
1448                 rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
1449                 invoke_rcu_core();
1450                 return;
1451         }
1452 
1453         /*
1454          * If we have not yet accelerated this jiffy, accelerate all
1455          * callbacks on this CPU.
1456          */
1457         if (rdtp->last_accelerate == jiffies)
1458                 return;
1459         rdtp->last_accelerate = jiffies;
1460         for_each_rcu_flavor(rsp) {
1461                 rdp = this_cpu_ptr(rsp->rda);
1462                 if (!*rdp->nxttail[RCU_DONE_TAIL])
1463                         continue;
1464                 rnp = rdp->mynode;
1465                 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
1466                 needwake = rcu_accelerate_cbs(rsp, rnp, rdp);
1467                 raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
1468                 if (needwake)
1469                         rcu_gp_kthread_wake(rsp);
1470         }
1471 }
1472 
1473 /*
1474  * Clean up for exit from idle.  Attempt to advance callbacks based on
1475  * any grace periods that elapsed while the CPU was idle, and if any
1476  * callbacks are now ready to invoke, initiate invocation.
1477  */
1478 static void rcu_cleanup_after_idle(void)
1479 {
1480         if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL) ||
1481             rcu_is_nocb_cpu(smp_processor_id()))
1482                 return;
1483         if (rcu_try_advance_all_cbs())
1484                 invoke_rcu_core();
1485 }
1486 
1487 /*
1488  * Keep a running count of the number of non-lazy callbacks posted
1489  * on this CPU.  This running counter (which is never decremented) allows
1490  * rcu_prepare_for_idle() to detect when something out of the idle loop
1491  * posts a callback, even if an equal number of callbacks are invoked.
1492  * Of course, callbacks should only be posted from within a trace event
1493  * designed to be called from idle or from within RCU_NONIDLE().
1494  */
1495 static void rcu_idle_count_callbacks_posted(void)
1496 {
1497         __this_cpu_add(rcu_dynticks.nonlazy_posted, 1);
1498 }
1499 
1500 /*
1501  * Data for flushing lazy RCU callbacks at OOM time.
1502  */
1503 static atomic_t oom_callback_count;
1504 static DECLARE_WAIT_QUEUE_HEAD(oom_callback_wq);
1505 
1506 /*
1507  * RCU OOM callback -- decrement the outstanding count and deliver the
1508  * wake-up if we are the last one.
1509  */
1510 static void rcu_oom_callback(struct rcu_head *rhp)
1511 {
1512         if (atomic_dec_and_test(&oom_callback_count))
1513                 wake_up(&oom_callback_wq);
1514 }
1515 
1516 /*
1517  * Post an rcu_oom_notify callback on the current CPU if it has at
1518  * least one lazy callback.  This will unnecessarily post callbacks
1519  * to CPUs that already have a non-lazy callback at the end of their
1520  * callback list, but this is an infrequent operation, so accept some
1521  * extra overhead to keep things simple.
1522  */
1523 static void rcu_oom_notify_cpu(void *unused)
1524 {
1525         struct rcu_state *rsp;
1526         struct rcu_data *rdp;
1527 
1528         for_each_rcu_flavor(rsp) {
1529                 rdp = raw_cpu_ptr(rsp->rda);
1530                 if (rdp->qlen_lazy != 0) {
1531                         atomic_inc(&oom_callback_count);
1532                         rsp->call(&rdp->oom_head, rcu_oom_callback);
1533                 }
1534         }
1535 }
1536 
1537 /*
1538  * If low on memory, ensure that each CPU has a non-lazy callback.
1539  * This will wake up CPUs that have only lazy callbacks, in turn
1540  * ensuring that they free up the corresponding memory in a timely manner.
1541  * Because an uncertain amount of memory will be freed in some uncertain
1542  * timeframe, we do not claim to have freed anything.
1543  */
1544 static int rcu_oom_notify(struct notifier_block *self,
1545                           unsigned long notused, void *nfreed)
1546 {
1547         int cpu;
1548 
1549         /* Wait for callbacks from earlier instance to complete. */
1550         wait_event(oom_callback_wq, atomic_read(&oom_callback_count) == 0);
1551         smp_mb(); /* Ensure callback reuse happens after callback invocation. */
1552 
1553         /*
1554          * Prevent premature wakeup: ensure that all increments happen
1555          * before there is a chance of the counter reaching zero.
1556          */
1557         atomic_set(&oom_callback_count, 1);
1558 
1559         for_each_online_cpu(cpu) {
1560                 smp_call_function_single(cpu, rcu_oom_notify_cpu, NULL, 1);
1561                 cond_resched_rcu_qs();
1562         }
1563 
1564         /* Unconditionally decrement: no need to wake ourselves up. */
1565         atomic_dec(&oom_callback_count);
1566 
1567         return NOTIFY_OK;
1568 }
1569 
1570 static struct notifier_block rcu_oom_nb = {
1571         .notifier_call = rcu_oom_notify
1572 };
1573 
1574 static int __init rcu_register_oom_notifier(void)
1575 {
1576         register_oom_notifier(&rcu_oom_nb);
1577         return 0;
1578 }
1579 early_initcall(rcu_register_oom_notifier);
1580 
1581 #endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */
1582 
1583 #ifdef CONFIG_RCU_FAST_NO_HZ
1584 
1585 static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
1586 {
1587         struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
1588         unsigned long nlpd = rdtp->nonlazy_posted - rdtp->nonlazy_posted_snap;
1589 
1590         sprintf(cp, "last_accelerate: %04lx/%04lx, nonlazy_posted: %ld, %c%c",
1591                 rdtp->last_accelerate & 0xffff, jiffies & 0xffff,
1592                 ulong2long(nlpd),
1593                 rdtp->all_lazy ? 'L' : '.',
1594                 rdtp->tick_nohz_enabled_snap ? '.' : 'D');
1595 }
1596 
1597 #else /* #ifdef CONFIG_RCU_FAST_NO_HZ */
1598 
1599 static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
1600 {
1601         *cp = '\0';
1602 }
1603 
1604 #endif /* #else #ifdef CONFIG_RCU_FAST_NO_HZ */
1605 
1606 /* Initiate the stall-info list. */
1607 static void print_cpu_stall_info_begin(void)
1608 {
1609         pr_cont("\n");
1610 }
1611 
1612 /*
1613  * Print out diagnostic information for the specified stalled CPU.
1614  *
1615  * If the specified CPU is aware of the current RCU grace period
1616  * (flavor specified by rsp), then print the number of scheduling
1617  * clock interrupts the CPU has taken during the time that it has
1618  * been aware.  Otherwise, print the number of RCU grace periods
1619  * that this CPU is ignorant of, for example, "1" if the CPU was
1620  * aware of the previous grace period.
1621  *
1622  * Also print out idle and (if CONFIG_RCU_FAST_NO_HZ) idle-entry info.
1623  */
1624 static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
1625 {
1626         char fast_no_hz[72];
1627         struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
1628         struct rcu_dynticks *rdtp = rdp->dynticks;
1629         char *ticks_title;
1630         unsigned long ticks_value;
1631 
1632         if (rsp->gpnum == rdp->gpnum) {
1633                 ticks_title = "ticks this GP";
1634                 ticks_value = rdp->ticks_this_gp;
1635         } else {
1636                 ticks_title = "GPs behind";
1637                 ticks_value = rsp->gpnum - rdp->gpnum;
1638         }
1639         print_cpu_stall_fast_no_hz(fast_no_hz, cpu);
1640         pr_err("\t%d-%c%c%c: (%lu %s) idle=%03x/%llx/%d softirq=%u/%u fqs=%ld %s\n",
1641                cpu,
1642                "O."[!!cpu_online(cpu)],
1643                "o."[!!(rdp->grpmask & rdp->mynode->qsmaskinit)],
1644                "N."[!!(rdp->grpmask & rdp->mynode->qsmaskinitnext)],
1645                ticks_value, ticks_title,
1646                atomic_read(&rdtp->dynticks) & 0xfff,
1647                rdtp->dynticks_nesting, rdtp->dynticks_nmi_nesting,
1648                rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
1649                READ_ONCE(rsp->n_force_qs) - rsp->n_force_qs_gpstart,
1650                fast_no_hz);
1651 }
1652 
1653 /* Terminate the stall-info list. */
1654 static void print_cpu_stall_info_end(void)
1655 {
1656         pr_err("\t");
1657 }
1658 
1659 /* Zero ->ticks_this_gp for all flavors of RCU. */
1660 static void zero_cpu_stall_ticks(struct rcu_data *rdp)
1661 {
1662         rdp->ticks_this_gp = 0;
1663         rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id());
1664 }
1665 
1666 /* Increment ->ticks_this_gp for all flavors of RCU. */
1667 static void increment_cpu_stall_ticks(void)
1668 {
1669         struct rcu_state *rsp;
1670 
1671         for_each_rcu_flavor(rsp)
1672                 raw_cpu_inc(rsp->rda->ticks_this_gp);
1673 }
1674 
1675 #ifdef CONFIG_RCU_NOCB_CPU
1676 
1677 /*
1678  * Offload callback processing from the boot-time-specified set of CPUs
1679  * specified by rcu_nocb_mask.  For each CPU in the set, there is a
1680  * kthread created that pulls the callbacks from the corresponding CPU,
1681  * waits for a grace period to elapse, and invokes the callbacks.
1682  * The no-CBs CPUs do a wake_up() on their kthread when they insert
1683  * a callback into any empty list, unless the rcu_nocb_poll boot parameter
1684  * has been specified, in which case each kthread actively polls its
1685  * CPU.  (Which isn't so great for energy efficiency, but which does
1686  * reduce RCU's overhead on that CPU.)
1687  *
1688  * This is intended to be used in conjunction with Frederic Weisbecker's
1689  * adaptive-idle work, which would seriously reduce OS jitter on CPUs
1690  * running CPU-bound user-mode computations.
1691  *
1692  * Offloading of callback processing could also in theory be used as
1693  * an energy-efficiency measure because CPUs with no RCU callbacks
1694  * queued are more aggressive about entering dyntick-idle mode.
1695  */
1696 
1697 
1698 /* Parse the boot-time rcu_nocb_mask CPU list from the kernel parameters. */
1699 static int __init rcu_nocb_setup(char *str)
1700 {
1701         alloc_bootmem_cpumask_var(&rcu_nocb_mask);
1702         have_rcu_nocb_mask = true;
1703         cpulist_parse(str, rcu_nocb_mask);
1704         return 1;
1705 }
1706 __setup("rcu_nocbs=", rcu_nocb_setup);
1707 
1708 static int __init parse_rcu_nocb_poll(char *arg)
1709 {
1710         rcu_nocb_poll = 1;
1711         return 0;
1712 }
1713 early_param("rcu_nocb_poll", parse_rcu_nocb_poll);
1714 
1715 /*
1716  * Wake up any no-CBs CPUs' kthreads that were waiting on the just-ended
1717  * grace period.
1718  */
1719 static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq)
1720 {
1721         swake_up_all(sq);
1722 }
1723 
1724 /*
1725  * Set the root rcu_node structure's ->need_future_gp field
1726  * based on the sum of those of all rcu_node structures.  This does
1727  * double-count the root rcu_node structure's requests, but this
1728  * is necessary to handle the possibility of a rcu_nocb_kthread()
1729  * having awakened during the time that the rcu_node structures
1730  * were being updated for the end of the previous grace period.
1731  */
1732 static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)
1733 {
1734         rnp->need_future_gp[(rnp->completed + 1) & 0x1] += nrq;
1735 }
1736 
1737 static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)
1738 {
1739         return &rnp->nocb_gp_wq[rnp->completed & 0x1];
1740 }
1741 
1742 static void rcu_init_one_nocb(struct rcu_node *rnp)
1743 {
1744         init_swait_queue_head(&rnp->nocb_gp_wq[0]);
1745         init_swait_queue_head(&rnp->nocb_gp_wq[1]);
1746 }
1747 
1748 #ifndef CONFIG_RCU_NOCB_CPU_ALL
1749 /* Is the specified CPU a no-CBs CPU? */
1750 bool rcu_is_nocb_cpu(int cpu)
1751 {
1752         if (have_rcu_nocb_mask)
1753                 return cpumask_test_cpu(cpu, rcu_nocb_mask);
1754         return false;
1755 }
1756 #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
1757 
1758 /*
1759  * Kick the leader kthread for this NOCB group.
1760  */
1761 static void wake_nocb_leader(struct rcu_data *rdp, bool force)
1762 {
1763         struct rcu_data *rdp_leader = rdp->nocb_leader;
1764 
1765         if (!READ_ONCE(rdp_leader->nocb_kthread))
1766                 return;
1767         if (READ_ONCE(rdp_leader->nocb_leader_sleep) || force) {
1768                 /* Prior smp_mb__after_atomic() orders against prior enqueue. */
1769                 WRITE_ONCE(rdp_leader->nocb_leader_sleep, false);
1770                 swake_up(&rdp_leader->nocb_wq);
1771         }
1772 }
1773 
1774 /*
1775  * Does the specified CPU need an RCU callback for the specified flavor
1776  * of rcu_barrier()?
1777  */
1778 static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)
1779 {
1780         struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
1781         unsigned long ret;
1782 #ifdef CONFIG_PROVE_RCU
1783         struct rcu_head *rhp;
1784 #endif /* #ifdef CONFIG_PROVE_RCU */
1785 
1786         /*
1787          * Check count of all no-CBs callbacks awaiting invocation.
1788          * There needs to be a barrier before this function is called,
1789          * but associated with a prior determination that no more
1790          * callbacks would be posted.  In the worst case, the first
1791          * barrier in _rcu_barrier() suffices (but the caller cannot
1792          * necessarily rely on this, not a substitute for the caller
1793          * getting the concurrency design right!).  There must also be
1794          * a barrier between the following load an posting of a callback
1795          * (if a callback is in fact needed).  This is associated with an
1796          * atomic_inc() in the caller.
1797          */
1798         ret = atomic_long_read(&rdp->nocb_q_count);
1799 
1800 #ifdef CONFIG_PROVE_RCU
1801         rhp = READ_ONCE(rdp->nocb_head);
1802         if (!rhp)
1803                 rhp = READ_ONCE(rdp->nocb_gp_head);
1804         if (!rhp)
1805                 rhp = READ_ONCE(rdp->nocb_follower_head);
1806 
1807         /* Having no rcuo kthread but CBs after scheduler starts is bad! */
1808         if (!READ_ONCE(rdp->nocb_kthread) && rhp &&
1809             rcu_scheduler_fully_active) {
1810                 /* RCU callback enqueued before CPU first came online??? */
1811                 pr_err("RCU: Never-onlined no-CBs CPU %d has CB %p\n",
1812                        cpu, rhp->func);
1813                 WARN_ON_ONCE(1);
1814         }
1815 #endif /* #ifdef CONFIG_PROVE_RCU */
1816 
1817         return !!ret;
1818 }
1819 
1820 /*
1821  * Enqueue the specified string of rcu_head structures onto the specified
1822  * CPU's no-CBs lists.  The CPU is specified by rdp, the head of the
1823  * string by rhp, and the tail of the string by rhtp.  The non-lazy/lazy
1824  * counts are supplied by rhcount and rhcount_lazy.
1825  *
1826  * If warranted, also wake up the kthread servicing this CPUs queues.
1827  */
1828 static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,
1829                                     struct rcu_head *rhp,
1830                                     struct rcu_head **rhtp,
1831                                     int rhcount, int rhcount_lazy,
1832                                     unsigned long flags)
1833 {
1834         int len;
1835         struct rcu_head **old_rhpp;
1836         struct task_struct *t;
1837 
1838         /* Enqueue the callback on the nocb list and update counts. */
1839         atomic_long_add(rhcount, &rdp->nocb_q_count);
1840         /* rcu_barrier() relies on ->nocb_q_count add before xchg. */
1841         old_rhpp = xchg(&rdp->nocb_tail, rhtp);
1842         WRITE_ONCE(*old_rhpp, rhp);
1843         atomic_long_add(rhcount_lazy, &rdp->nocb_q_count_lazy);
1844         smp_mb__after_atomic(); /* Store *old_rhpp before _wake test. */
1845 
1846         /* If we are not being polled and there is a kthread, awaken it ... */
1847         t = READ_ONCE(rdp->nocb_kthread);
1848         if (rcu_nocb_poll || !t) {
1849                 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1850                                     TPS("WakeNotPoll"));
1851                 return;
1852         }
1853         len = atomic_long_read(&rdp->nocb_q_count);
1854         if (old_rhpp == &rdp->nocb_head) {
1855                 if (!irqs_disabled_flags(flags)) {
1856                         /* ... if queue was empty ... */
1857                         wake_nocb_leader(rdp, false);
1858                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1859                                             TPS("WakeEmpty"));
1860                 } else {
1861                         rdp->nocb_defer_wakeup = RCU_NOGP_WAKE;
1862                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1863                                             TPS("WakeEmptyIsDeferred"));
1864                 }
1865                 rdp->qlen_last_fqs_check = 0;
1866         } else if (len > rdp->qlen_last_fqs_check + qhimark) {
1867                 /* ... or if many callbacks queued. */
1868                 if (!irqs_disabled_flags(flags)) {
1869                         wake_nocb_leader(rdp, true);
1870                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1871                                             TPS("WakeOvf"));
1872                 } else {
1873                         rdp->nocb_defer_wakeup = RCU_NOGP_WAKE_FORCE;
1874                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1875                                             TPS("WakeOvfIsDeferred"));
1876                 }
1877                 rdp->qlen_last_fqs_check = LONG_MAX / 2;
1878         } else {
1879                 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WakeNot"));
1880         }
1881         return;
1882 }
1883 
1884 /*
1885  * This is a helper for __call_rcu(), which invokes this when the normal
1886  * callback queue is inoperable.  If this is not a no-CBs CPU, this
1887  * function returns failure back to __call_rcu(), which can complain
1888  * appropriately.
1889  *
1890  * Otherwise, this function queues the callback where the corresponding
1891  * "rcuo" kthread can find it.
1892  */
1893 static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
1894                             bool lazy, unsigned long flags)
1895 {
1896 
1897         if (!rcu_is_nocb_cpu(rdp->cpu))
1898                 return false;
1899         __call_rcu_nocb_enqueue(rdp, rhp, &rhp->next, 1, lazy, flags);
1900         if (__is_kfree_rcu_offset((unsigned long)rhp->func))
1901                 trace_rcu_kfree_callback(rdp->rsp->name, rhp,
1902                                          (unsigned long)rhp->func,
1903                                          -atomic_long_read(&rdp->nocb_q_count_lazy),
1904                                          -atomic_long_read(&rdp->nocb_q_count));
1905         else
1906                 trace_rcu_callback(rdp->rsp->name, rhp,
1907                                    -atomic_long_read(&rdp->nocb_q_count_lazy),
1908                                    -atomic_long_read(&rdp->nocb_q_count));
1909 
1910         /*
1911          * If called from an extended quiescent state with interrupts
1912          * disabled, invoke the RCU core in order to allow the idle-entry
1913          * deferred-wakeup check to function.
1914          */
1915         if (irqs_disabled_flags(flags) &&
1916             !rcu_is_watching() &&
1917             cpu_online(smp_processor_id()))
1918                 invoke_rcu_core();
1919 
1920         return true;
1921 }
1922 
1923 /*
1924  * Adopt orphaned callbacks on a no-CBs CPU, or return 0 if this is
1925  * not a no-CBs CPU.
1926  */
1927 static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
1928                                                      struct rcu_data *rdp,
1929                                                      unsigned long flags)
1930 {
1931         long ql = rsp->qlen;
1932         long qll = rsp->qlen_lazy;
1933 
1934         /* If this is not a no-CBs CPU, tell the caller to do it the old way. */
1935         if (!rcu_is_nocb_cpu(smp_processor_id()))
1936                 return false;
1937         rsp->qlen = 0;
1938         rsp->qlen_lazy = 0;
1939 
1940         /* First, enqueue the donelist, if any.  This preserves CB ordering. */
1941         if (rsp->orphan_donelist != NULL) {
1942                 __call_rcu_nocb_enqueue(rdp, rsp->orphan_donelist,
1943                                         rsp->orphan_donetail, ql, qll, flags);
1944                 ql = qll = 0;
1945                 rsp->orphan_donelist = NULL;
1946                 rsp->orphan_donetail = &rsp->orphan_donelist;
1947         }
1948         if (rsp->orphan_nxtlist != NULL) {
1949                 __call_rcu_nocb_enqueue(rdp, rsp->orphan_nxtlist,
1950                                         rsp->orphan_nxttail, ql, qll, flags);
1951                 ql = qll = 0;
1952                 rsp->orphan_nxtlist = NULL;
1953                 rsp->orphan_nxttail = &rsp->orphan_nxtlist;
1954         }
1955         return true;
1956 }
1957 
1958 /*
1959  * If necessary, kick off a new grace period, and either way wait
1960  * for a subsequent grace period to complete.
1961  */
1962 static void rcu_nocb_wait_gp(struct rcu_data *rdp)
1963 {
1964         unsigned long c;
1965         bool d;
1966         unsigned long flags;
1967         bool needwake;
1968         struct rcu_node *rnp = rdp->mynode;
1969 
1970         raw_spin_lock_irqsave_rcu_node(rnp, flags);
1971         needwake = rcu_start_future_gp(rnp, rdp, &c);
1972         raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1973         if (needwake)
1974                 rcu_gp_kthread_wake(rdp->rsp);
1975 
1976         /*
1977          * Wait for the grace period.  Do so interruptibly to avoid messing
1978          * up the load average.
1979          */
1980         trace_rcu_future_gp(rnp, rdp, c, TPS("StartWait"));
1981         for (;;) {
1982                 swait_event_interruptible(
1983                         rnp->nocb_gp_wq[c & 0x1],
1984                         (d = ULONG_CMP_GE(READ_ONCE(rnp->completed), c)));
1985                 if (likely(d))
1986                         break;
1987                 WARN_ON(signal_pending(current));
1988                 trace_rcu_future_gp(rnp, rdp, c, TPS("ResumeWait"));
1989         }
1990         trace_rcu_future_gp(rnp, rdp, c, TPS("EndWait"));
1991         smp_mb(); /* Ensure that CB invocation happens after GP end. */
1992 }
1993 
1994 /*
1995  * Leaders come here to wait for additional callbacks to show up.
1996  * This function does not return until callbacks appear.
1997  */
1998 static void nocb_leader_wait(struct rcu_data *my_rdp)
1999 {
2000         bool firsttime = true;
2001         bool gotcbs;
2002         struct rcu_data *rdp;
2003         struct rcu_head **tail;
2004 
2005 wait_again:
2006 
2007         /* Wait for callbacks to appear. */
2008         if (!rcu_nocb_poll) {
2009                 trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep");
2010                 swait_event_interruptible(my_rdp->nocb_wq,
2011                                 !READ_ONCE(my_rdp->nocb_leader_sleep));
2012                 /* Memory barrier handled by smp_mb() calls below and repoll. */
2013         } else if (firsttime) {
2014                 firsttime = false; /* Don't drown trace log with "Poll"! */
2015                 trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Poll");
2016         }
2017 
2018         /*
2019          * Each pass through the following loop checks a follower for CBs.
2020          * We are our own first follower.  Any CBs found are moved to
2021          * nocb_gp_head, where they await a grace period.
2022          */
2023         gotcbs = false;
2024         for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {
2025                 rdp->nocb_gp_head = READ_ONCE(rdp->nocb_head);
2026                 if (!rdp->nocb_gp_head)
2027                         continue;  /* No CBs here, try next follower. */
2028 
2029                 /* Move callbacks to wait-for-GP list, which is empty. */
2030                 WRITE_ONCE(rdp->nocb_head, NULL);
2031                 rdp->nocb_gp_tail = xchg(&rdp->nocb_tail, &rdp->nocb_head);
2032                 gotcbs = true;
2033         }
2034 
2035         /*
2036          * If there were no callbacks, sleep a bit, rescan after a
2037          * memory barrier, and go retry.
2038          */
2039         if (unlikely(!gotcbs)) {
2040                 if (!rcu_nocb_poll)
2041                         trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu,
2042                                             "WokeEmpty");
2043                 WARN_ON(signal_pending(current));
2044                 schedule_timeout_interruptible(1);
2045 
2046                 /* Rescan in case we were a victim of memory ordering. */
2047                 my_rdp->nocb_leader_sleep = true;
2048                 smp_mb();  /* Ensure _sleep true before scan. */
2049                 for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower)
2050                         if (READ_ONCE(rdp->nocb_head)) {
2051                                 /* Found CB, so short-circuit next wait. */
2052                                 my_rdp->nocb_leader_sleep = false;
2053                                 break;
2054                         }
2055                 goto wait_again;
2056         }
2057 
2058         /* Wait for one grace period. */
2059         rcu_nocb_wait_gp(my_rdp);
2060 
2061         /*
2062          * We left ->nocb_leader_sleep unset to reduce cache thrashing.
2063          * We set it now, but recheck for new callbacks while
2064          * traversing our follower list.
2065          */
2066         my_rdp->nocb_leader_sleep = true;
2067         smp_mb(); /* Ensure _sleep true before scan of ->nocb_head. */
2068 
2069         /* Each pass through the following loop wakes a follower, if needed. */
2070         for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {
2071                 if (READ_ONCE(rdp->nocb_head))
2072                         my_rdp->nocb_leader_sleep = false;/* No need to sleep.*/
2073                 if (!rdp->nocb_gp_head)
2074                         continue; /* No CBs, so no need to wake follower. */
2075 
2076                 /* Append callbacks to follower's "done" list. */
2077                 tail = xchg(&rdp->nocb_follower_tail, rdp->nocb_gp_tail);
2078                 *tail = rdp->nocb_gp_head;
2079                 smp_mb__after_atomic(); /* Store *tail before wakeup. */
2080                 if (rdp != my_rdp && tail == &rdp->nocb_follower_head) {
2081                         /*
2082                          * List was empty, wake up the follower.
2083                          * Memory barriers supplied by atomic_long_add().
2084                          */
2085                         swake_up(&rdp->nocb_wq);
2086                 }
2087         }
2088 
2089         /* If we (the leader) don't have CBs, go wait some more. */
2090         if (!my_rdp->nocb_follower_head)
2091                 goto wait_again;
2092 }
2093 
2094 /*
2095  * Followers come here to wait for additional callbacks to show up.
2096  * This function does not return until callbacks appear.
2097  */
2098 static void nocb_follower_wait(struct rcu_data *rdp)
2099 {
2100         bool firsttime = true;
2101 
2102         for (;;) {
2103                 if (!rcu_nocb_poll) {
2104                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2105                                             "FollowerSleep");
2106                         swait_event_interruptible(rdp->nocb_wq,
2107                                                  READ_ONCE(rdp->nocb_follower_head));
2108                 } else if (firsttime) {
2109                         /* Don't drown trace log with "Poll"! */
2110                         firsttime = false;
2111                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "Poll");
2112                 }
2113                 if (smp_load_acquire(&rdp->nocb_follower_head)) {
2114                         /* ^^^ Ensure CB invocation follows _head test. */
2115                         return;
2116                 }
2117                 if (!rcu_nocb_poll)
2118                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2119                                             "WokeEmpty");
2120                 WARN_ON(signal_pending(current));
2121                 schedule_timeout_interruptible(1);
2122         }
2123 }
2124 
2125 /*
2126  * Per-rcu_data kthread, but only for no-CBs CPUs.  Each kthread invokes
2127  * callbacks queued by the corresponding no-CBs CPU, however, there is
2128  * an optional leader-follower relationship so that the grace-period
2129  * kthreads don't have to do quite so many wakeups.
2130  */
2131 static int rcu_nocb_kthread(void *arg)
2132 {
2133         int c, cl;
2134         struct rcu_head *list;
2135         struct rcu_head *next;
2136         struct rcu_head **tail;
2137         struct rcu_data *rdp = arg;
2138 
2139         /* Each pass through this loop invokes one batch of callbacks */
2140         for (;;) {
2141                 /* Wait for callbacks. */
2142                 if (rdp->nocb_leader == rdp)
2143                         nocb_leader_wait(rdp);
2144                 else
2145                         nocb_follower_wait(rdp);
2146 
2147                 /* Pull the ready-to-invoke callbacks onto local list. */
2148                 list = READ_ONCE(rdp->nocb_follower_head);
2149                 BUG_ON(!list);
2150                 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "WokeNonEmpty");
2151                 WRITE_ONCE(rdp->nocb_follower_head, NULL);
2152                 tail = xchg(&rdp->nocb_follower_tail, &rdp->nocb_follower_head);
2153 
2154                 /* Each pass through the following loop invokes a callback. */
2155                 trace_rcu_batch_start(rdp->rsp->name,
2156                                       atomic_long_read(&rdp->nocb_q_count_lazy),
2157                                       atomic_long_read(&rdp->nocb_q_count), -1);
2158                 c = cl = 0;
2159                 while (list) {
2160                         next = list->next;
2161                         /* Wait for enqueuing to complete, if needed. */
2162                         while (next == NULL && &list->next != tail) {
2163                                 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2164                                                     TPS("WaitQueue"));
2165                                 schedule_timeout_interruptible(1);
2166                                 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2167                                                     TPS("WokeQueue"));
2168                                 next = list->next;
2169                         }
2170                         debug_rcu_head_unqueue(list);
2171                         local_bh_disable();
2172                         if (__rcu_reclaim(rdp->rsp->name, list))
2173                                 cl++;
2174                         c++;
2175                         local_bh_enable();
2176                         cond_resched_rcu_qs();
2177                         list = next;
2178                 }
2179                 trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1);
2180                 smp_mb__before_atomic();  /* _add after CB invocation. */
2181                 atomic_long_add(-c, &rdp->nocb_q_count);
2182                 atomic_long_add(-cl, &rdp->nocb_q_count_lazy);
2183                 rdp->n_nocbs_invoked += c;
2184         }
2185         return 0;
2186 }
2187 
2188 /* Is a deferred wakeup of rcu_nocb_kthread() required? */
2189 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
2190 {
2191         return READ_ONCE(rdp->nocb_defer_wakeup);
2192 }
2193 
2194 /* Do a deferred wakeup of rcu_nocb_kthread(). */
2195 static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
2196 {
2197         int ndw;
2198 
2199         if (!rcu_nocb_need_deferred_wakeup(rdp))
2200                 return;
2201         ndw = READ_ONCE(rdp->nocb_defer_wakeup);
2202         WRITE_ONCE(rdp->nocb_defer_wakeup, RCU_NOGP_WAKE_NOT);
2203         wake_nocb_leader(rdp, ndw == RCU_NOGP_WAKE_FORCE);
2204         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("DeferredWake"));
2205 }
2206 
2207 void __init rcu_init_nohz(void)
2208 {
2209         int cpu;
2210         bool need_rcu_nocb_mask = true;
2211         struct rcu_state *rsp;
2212 
2213 #ifdef CONFIG_RCU_NOCB_CPU_NONE
2214         need_rcu_nocb_mask = false;
2215 #endif /* #ifndef CONFIG_RCU_NOCB_CPU_NONE */
2216 
2217 #if defined(CONFIG_NO_HZ_FULL)
2218         if (tick_nohz_full_running && cpumask_weight(tick_nohz_full_mask))
2219                 need_rcu_nocb_mask = true;
2220 #endif /* #if defined(CONFIG_NO_HZ_FULL) */
2221 
2222         if (!have_rcu_nocb_mask && need_rcu_nocb_mask) {
2223                 if (!zalloc_cpumask_var(&rcu_nocb_mask, GFP_KERNEL)) {
2224                         pr_info("rcu_nocb_mask allocation failed, callback offloading disabled.\n");
2225                         return;
2226                 }
2227                 have_rcu_nocb_mask = true;
2228         }
2229         if (!have_rcu_nocb_mask)
2230                 return;
2231 
2232 #ifdef CONFIG_RCU_NOCB_CPU_ZERO
2233         pr_info("\tOffload RCU callbacks from CPU 0\n");
2234         cpumask_set_cpu(0, rcu_nocb_mask);
2235 #endif /* #ifdef CONFIG_RCU_NOCB_CPU_ZERO */
2236 #ifdef CONFIG_RCU_NOCB_CPU_ALL
2237         pr_info("\tOffload RCU callbacks from all CPUs\n");
2238         cpumask_copy(rcu_nocb_mask, cpu_possible_mask);
2239 #endif /* #ifdef CONFIG_RCU_NOCB_CPU_ALL */
2240 #if defined(CONFIG_NO_HZ_FULL)
2241         if (tick_nohz_full_running)
2242                 cpumask_or(rcu_nocb_mask, rcu_nocb_mask, tick_nohz_full_mask);
2243 #endif /* #if defined(CONFIG_NO_HZ_FULL) */
2244 
2245         if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) {
2246                 pr_info("\tNote: kernel parameter 'rcu_nocbs=' contains nonexistent CPUs.\n");
2247                 cpumask_and(rcu_nocb_mask, cpu_possible_mask,
2248                             rcu_nocb_mask);
2249         }
2250         pr_info("\tOffload RCU callbacks from CPUs: %*pbl.\n",
2251                 cpumask_pr_args(rcu_nocb_mask));
2252         if (rcu_nocb_poll)
2253                 pr_info("\tPoll for callbacks from no-CBs CPUs.\n");
2254 
2255         for_each_rcu_flavor(rsp) {
2256                 for_each_cpu(cpu, rcu_nocb_mask)
2257                         init_nocb_callback_list(per_cpu_ptr(rsp->rda, cpu));
2258                 rcu_organize_nocb_kthreads(rsp);
2259         }
2260 }
2261 
2262 /* Initialize per-rcu_data variables for no-CBs CPUs. */
2263 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
2264 {
2265         rdp->nocb_tail = &rdp->nocb_head;
2266         init_swait_queue_head(&rdp->nocb_wq);
2267         rdp->nocb_follower_tail = &rdp->nocb_follower_head;
2268 }
2269 
2270 /*
2271  * If the specified CPU is a no-CBs CPU that does not already have its
2272  * rcuo kthread for the specified RCU flavor, spawn it.  If the CPUs are
2273  * brought online out of order, this can require re-organizing the
2274  * leader-follower relationships.
2275  */
2276 static void rcu_spawn_one_nocb_kthread(struct rcu_state *rsp, int cpu)
2277 {
2278         struct rcu_data *rdp;
2279         struct rcu_data *rdp_last;
2280         struct rcu_data *rdp_old_leader;
2281         struct rcu_data *rdp_spawn = per_cpu_ptr(rsp->rda, cpu);
2282         struct task_struct *t;
2283 
2284         /*
2285          * If this isn't a no-CBs CPU or if it already has an rcuo kthread,
2286          * then nothing to do.
2287          */
2288         if (!rcu_is_nocb_cpu(cpu) || rdp_spawn->nocb_kthread)
2289                 return;
2290 
2291         /* If we didn't spawn the leader first, reorganize! */
2292         rdp_old_leader = rdp_spawn->nocb_leader;
2293         if (rdp_old_leader != rdp_spawn && !rdp_old_leader->nocb_kthread) {
2294                 rdp_last = NULL;
2295                 rdp = rdp_old_leader;
2296                 do {
2297                         rdp->nocb_leader = rdp_spawn;
2298                         if (rdp_last && rdp != rdp_spawn)
2299                                 rdp_last->nocb_next_follower = rdp;
2300                         if (rdp == rdp_spawn) {
2301                                 rdp = rdp->nocb_next_follower;
2302                         } else {
2303                                 rdp_last = rdp;
2304                                 rdp = rdp->nocb_next_follower;
2305                                 rdp_last->nocb_next_follower = NULL;
2306                         }
2307                 } while (rdp);
2308                 rdp_spawn->nocb_next_follower = rdp_old_leader;
2309         }
2310 
2311         /* Spawn the kthread for this CPU and RCU flavor. */
2312         t = kthread_run(rcu_nocb_kthread, rdp_spawn,
2313                         "rcuo%c/%d", rsp->abbr, cpu);
2314         BUG_ON(IS_ERR(t));
2315         WRITE_ONCE(rdp_spawn->nocb_kthread, t);
2316 }
2317 
2318 /*
2319  * If the specified CPU is a no-CBs CPU that does not already have its
2320  * rcuo kthreads, spawn them.
2321  */
2322 static void rcu_spawn_all_nocb_kthreads(int cpu)
2323 {
2324         struct rcu_state *rsp;
2325 
2326         if (rcu_scheduler_fully_active)
2327                 for_each_rcu_flavor(rsp)
2328                         rcu_spawn_one_nocb_kthread(rsp, cpu);
2329 }
2330 
2331 /*
2332  * Once the scheduler is running, spawn rcuo kthreads for all online
2333  * no-CBs CPUs.  This assumes that the early_initcall()s happen before
2334  * non-boot CPUs come online -- if this changes, we will need to add
2335  * some mutual exclusion.
2336  */
2337 static void __init rcu_spawn_nocb_kthreads(void)
2338 {
2339         int cpu;
2340 
2341         for_each_online_cpu(cpu)
2342                 rcu_spawn_all_nocb_kthreads(cpu);
2343 }
2344 
2345 /* How many follower CPU IDs per leader?  Default of -1 for sqrt(nr_cpu_ids). */
2346 static int rcu_nocb_leader_stride = -1;
2347 module_param(rcu_nocb_leader_stride, int, 0444);
2348 
2349 /*
2350  * Initialize leader-follower relationships for all no-CBs CPU.
2351  */
2352 static void __init rcu_organize_nocb_kthreads(struct rcu_state *rsp)
2353 {
2354         int cpu;
2355         int ls = rcu_nocb_leader_stride;
2356         int nl = 0;  /* Next leader. */
2357         struct rcu_data *rdp;
2358         struct rcu_data *rdp_leader = NULL;  /* Suppress misguided gcc warn. */
2359         struct rcu_data *rdp_prev = NULL;
2360 
2361         if (!have_rcu_nocb_mask)
2362                 return;
2363         if (ls == -1) {
2364                 ls = int_sqrt(nr_cpu_ids);
2365                 rcu_nocb_leader_stride = ls;
2366         }
2367 
2368         /*
2369          * Each pass through this loop sets up one rcu_data structure and
2370          * spawns one rcu_nocb_kthread().
2371          */
2372         for_each_cpu(cpu, rcu_nocb_mask) {
2373                 rdp = per_cpu_ptr(rsp->rda, cpu);
2374                 if (rdp->cpu >= nl) {
2375                         /* New leader, set up for followers & next leader. */
2376                         nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls;
2377                         rdp->nocb_leader = rdp;
2378                         rdp_leader = rdp;
2379                 } else {
2380                         /* Another follower, link to previous leader. */
2381                         rdp->nocb_leader = rdp_leader;
2382                         rdp_prev->nocb_next_follower = rdp;
2383                 }
2384                 rdp_prev = rdp;
2385         }
2386 }
2387 
2388 /* Prevent __call_rcu() from enqueuing callbacks on no-CBs CPUs */
2389 static bool init_nocb_callback_list(struct rcu_data *rdp)
2390 {
2391         if (!rcu_is_nocb_cpu(rdp->cpu))
2392                 return false;
2393 
2394         /* If there are early-boot callbacks, move them to nocb lists. */
2395         if (rdp->nxtlist) {
2396                 rdp->nocb_head = rdp->nxtlist;
2397                 rdp->nocb_tail = rdp->nxttail[RCU_NEXT_TAIL];
2398                 atomic_long_set(&rdp->nocb_q_count, rdp->qlen);
2399                 atomic_long_set(&rdp->nocb_q_count_lazy, rdp->qlen_lazy);
2400                 rdp->nxtlist = NULL;
2401                 rdp->qlen = 0;
2402                 rdp->qlen_lazy = 0;
2403         }
2404         rdp->nxttail[RCU_NEXT_TAIL] = NULL;
2405         return true;
2406 }
2407 
2408 #else /* #ifdef CONFIG_RCU_NOCB_CPU */
2409 
2410 static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)
2411 {
2412         WARN_ON_ONCE(1); /* Should be dead code. */
2413         return false;
2414 }
2415 
2416 static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq)
2417 {
2418 }
2419 
2420 static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)
2421 {
2422 }
2423 
2424 static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)
2425 {
2426         return NULL;
2427 }
2428 
2429 static void rcu_init_one_nocb(struct rcu_node *rnp)
2430 {
2431 }
2432 
2433 static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
2434                             bool lazy, unsigned long flags)
2435 {
2436         return false;
2437 }
2438 
2439 static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
2440                                                      struct rcu_data *rdp,
2441                                                      unsigned long flags)
2442 {
2443         return false;
2444 }
2445 
2446 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
2447 {
2448 }
2449 
2450 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
2451 {
2452         return false;
2453 }
2454 
2455 static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
2456 {
2457 }
2458 
2459 static void rcu_spawn_all_nocb_kthreads(int cpu)
2460 {
2461 }
2462 
2463 static void __init rcu_spawn_nocb_kthreads(void)
2464 {
2465 }
2466 
2467 static bool init_nocb_callback_list(struct rcu_data *rdp)
2468 {
2469         return false;
2470 }
2471 
2472 #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
2473 
2474 /*
2475  * An adaptive-ticks CPU can potentially execute in kernel mode for an
2476  * arbitrarily long period of time with the scheduling-clock tick turned
2477  * off.  RCU will be paying attention to this CPU because it is in the
2478  * kernel, but the CPU cannot be guaranteed to be executing the RCU state
2479  * machine because the scheduling-clock tick has been disabled.  Therefore,
2480  * if an adaptive-ticks CPU is failing to respond to the current grace
2481  * period and has not be idle from an RCU perspective, kick it.
2482  */
2483 static void __maybe_unused rcu_kick_nohz_cpu(int cpu)
2484 {
2485 #ifdef CONFIG_NO_HZ_FULL
2486         if (tick_nohz_full_cpu(cpu))
2487                 smp_send_reschedule(cpu);
2488 #endif /* #ifdef CONFIG_NO_HZ_FULL */
2489 }
2490 
2491 
2492 #ifdef CONFIG_NO_HZ_FULL_SYSIDLE
2493 
2494 static int full_sysidle_state;          /* Current system-idle state. */
2495 #define RCU_SYSIDLE_NOT         0       /* Some CPU is not idle. */
2496 #define RCU_SYSIDLE_SHORT       1       /* All CPUs idle for brief period. */
2497 #define RCU_SYSIDLE_LONG        2       /* All CPUs idle for long enough. */
2498 #define RCU_SYSIDLE_FULL        3       /* All CPUs idle, ready for sysidle. */
2499 #define RCU_SYSIDLE_FULL_NOTED  4       /* Actually entered sysidle state. */
2500 
2501 /*
2502  * Invoked to note exit from irq or task transition to idle.  Note that
2503  * usermode execution does -not- count as idle here!  After all, we want
2504  * to detect full-system idle states, not RCU quiescent states and grace
2505  * periods.  The caller must have disabled interrupts.
2506  */
2507 static void rcu_sysidle_enter(int irq)
2508 {
2509         unsigned long j;
2510         struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
2511 
2512         /* If there are no nohz_full= CPUs, no need to track this. */
2513         if (!tick_nohz_full_enabled())
2514                 return;
2515 
2516         /* Adjust nesting, check for fully idle. */
2517         if (irq) {
2518                 rdtp->dynticks_idle_nesting--;
2519                 WARN_ON_ONCE(rdtp->dynticks_idle_nesting < 0);
2520                 if (rdtp->dynticks_idle_nesting != 0)
2521                         return;  /* Still not fully idle. */
2522         } else {
2523                 if ((rdtp->dynticks_idle_nesting & DYNTICK_TASK_NEST_MASK) ==
2524                     DYNTICK_TASK_NEST_VALUE) {
2525                         rdtp->dynticks_idle_nesting = 0;
2526                 } else {
2527                         rdtp->dynticks_idle_nesting -= DYNTICK_TASK_NEST_VALUE;
2528                         WARN_ON_ONCE(rdtp->dynticks_idle_nesting < 0);
2529                         return;  /* Still not fully idle. */
2530                 }
2531         }
2532 
2533         /* Record start of fully idle period. */
2534         j = jiffies;
2535         WRITE_ONCE(rdtp->dynticks_idle_jiffies, j);
2536         smp_mb__before_atomic();
2537         atomic_inc(&rdtp->dynticks_idle);
2538         smp_mb__after_atomic();
2539         WARN_ON_ONCE(atomic_read(&rdtp->dynticks_idle) & 0x1);
2540 }
2541 
2542 /*
2543  * Unconditionally force exit from full system-idle state.  This is
2544  * invoked when a normal CPU exits idle, but must be called separately
2545  * for the timekeeping CPU (tick_do_timer_cpu).  The reason for this
2546  * is that the timekeeping CPU is permitted to take scheduling-clock
2547  * interrupts while the system is in system-idle state, and of course
2548  * rcu_sysidle_exit() has no way of distinguishing a scheduling-clock
2549  * interrupt from any other type of interrupt.
2550  */
2551 void rcu_sysidle_force_exit(void)
2552 {
2553         int oldstate = READ_ONCE(full_sysidle_state);
2554         int newoldstate;
2555 
2556         /*
2557          * Each pass through the following loop attempts to exit full
2558          * system-idle state.  If contention proves to be a problem,
2559          * a trylock-based contention tree could be used here.
2560          */
2561         while (oldstate > RCU_SYSIDLE_SHORT) {
2562                 newoldstate = cmpxchg(&full_sysidle_state,
2563                                       oldstate, RCU_SYSIDLE_NOT);
2564                 if (oldstate == newoldstate &&
2565                     oldstate == RCU_SYSIDLE_FULL_NOTED) {
2566                         rcu_kick_nohz_cpu(tick_do_timer_cpu);
2567                         return; /* We cleared it, done! */
2568                 }
2569                 oldstate = newoldstate;
2570         }
2571         smp_mb(); /* Order initial oldstate fetch vs. later non-idle work. */
2572 }
2573 
2574 /*
2575  * Invoked to note entry to irq or task transition from idle.  Note that
2576  * usermode execution does -not- count as idle here!  The caller must
2577  * have disabled interrupts.
2578  */
2579 static void rcu_sysidle_exit(int irq)
2580 {
2581         struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
2582 
2583         /* If there are no nohz_full= CPUs, no need to track this. */
2584         if (!tick_nohz_full_enabled())
2585                 return;
2586 
2587         /* Adjust nesting, check for already non-idle. */
2588         if (irq) {
2589                 rdtp->dynticks_idle_nesting++;
2590                 WARN_ON_ONCE(rdtp->dynticks_idle_nesting <= 0);
2591                 if (rdtp->dynticks_idle_nesting != 1)
2592                         return; /* Already non-idle. */
2593         } else {
2594                 /*
2595                  * Allow for irq misnesting.  Yes, it really is possible
2596                  * to enter an irq handler then never leave it, and maybe
2597                  * also vice versa.  Handle both possibilities.
2598                  */
2599                 if (rdtp->dynticks_idle_nesting & DYNTICK_TASK_NEST_MASK) {
2600                         rdtp->dynticks_idle_nesting += DYNTICK_TASK_NEST_VALUE;
2601                         WARN_ON_ONCE(rdtp->dynticks_idle_nesting <= 0);
2602                         return; /* Already non-idle. */
2603                 } else {
2604                         rdtp->dynticks_idle_nesting = DYNTICK_TASK_EXIT_IDLE;
2605                 }
2606         }
2607 
2608         /* Record end of idle period. */
2609         smp_mb__before_atomic();
2610         atomic_inc(&rdtp->dynticks_idle);
2611         smp_mb__after_atomic();
2612         WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks_idle) & 0x1));
2613 
2614         /*
2615          * If we are the timekeeping CPU, we are permitted to be non-idle
2616          * during a system-idle state.  This must be the case, because
2617          * the timekeeping CPU has to take scheduling-clock interrupts
2618          * during the time that the system is transitioning to full
2619          * system-idle state.  This means that the timekeeping CPU must
2620          * invoke rcu_sysidle_force_exit() directly if it does anything
2621          * more than take a scheduling-clock interrupt.
2622          */
2623         if (smp_processor_id() == tick_do_timer_cpu)
2624                 return;
2625 
2626         /* Update system-idle state: We are clearly no longer fully idle! */
2627         rcu_sysidle_force_exit();
2628 }
2629 
2630 /*
2631  * Check to see if the current CPU is idle.  Note that usermode execution
2632  * does not count as idle.  The caller must have disabled interrupts,
2633  * and must be running on tick_do_timer_cpu.
2634  */
2635 static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
2636                                   unsigned long *maxj)
2637 {
2638         int cur;
2639         unsigned long j;
2640         struct rcu_dynticks *rdtp = rdp->dynticks;
2641 
2642         /* If there are no nohz_full= CPUs, don't check system-wide idleness. */
2643         if (!tick_nohz_full_enabled())
2644                 return;
2645 
2646         /*
2647          * If some other CPU has already reported non-idle, if this is
2648          * not the flavor of RCU that tracks sysidle state, or if this
2649          * is an offline or the timekeeping CPU, nothing to do.
2650          */
2651         if (!*isidle || rdp->rsp != rcu_state_p ||
2652             cpu_is_offline(rdp->cpu) || rdp->cpu == tick_do_timer_cpu)
2653                 return;
2654         /* Verify affinity of current kthread. */
2655         WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu);
2656 
2657         /* Pick up current idle and NMI-nesting counter and check. */
2658         cur = atomic_read(&rdtp->dynticks_idle);
2659         if (cur & 0x1) {
2660                 *isidle = false; /* We are not idle! */
2661                 return;
2662         }
2663         smp_mb(); /* Read counters before timestamps. */
2664 
2665         /* Pick up timestamps. */
2666         j = READ_ONCE(rdtp->dynticks_idle_jiffies);
2667         /* If this CPU entered idle more recently, update maxj timestamp. */
2668         if (ULONG_CMP_LT(*maxj, j))
2669                 *maxj = j;
2670 }
2671 
2672 /*
2673  * Is this the flavor of RCU that is handling full-system idle?
2674  */
2675 static bool is_sysidle_rcu_state(struct rcu_state *rsp)
2676 {
2677         return rsp == rcu_state_p;
2678 }
2679 
2680 /*
2681  * Return a delay in jiffies based on the number of CPUs, rcu_node
2682  * leaf fanout, and jiffies tick rate.  The idea is to allow larger
2683  * systems more time to transition to full-idle state in order to
2684  * avoid the cache thrashing that otherwise occur on the state variable.
2685  * Really small systems (less than a couple of tens of CPUs) should
2686  * instead use a single global atomically incremented counter, and later
2687  * versions of this will automatically reconfigure themselves accordingly.
2688  */
2689 static unsigned long rcu_sysidle_delay(void)
2690 {
2691         if (nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL)
2692                 return 0;
2693         return DIV_ROUND_UP(nr_cpu_ids * HZ, rcu_fanout_leaf * 1000);
2694 }
2695 
2696 /*
2697  * Advance the full-system-idle state.  This is invoked when all of
2698  * the non-timekeeping CPUs are idle.
2699  */
2700 static void rcu_sysidle(unsigned long j)
2701 {
2702         /* Check the current state. */
2703         switch (READ_ONCE(full_sysidle_state)) {
2704         case RCU_SYSIDLE_NOT:
2705 
2706                 /* First time all are idle, so note a short idle period. */
2707                 WRITE_ONCE(full_sysidle_state, RCU_SYSIDLE_SHORT);
2708                 break;
2709 
2710         case RCU_SYSIDLE_SHORT:
2711 
2712                 /*
2713                  * Idle for a bit, time to advance to next state?
2714                  * cmpxchg failure means race with non-idle, let them win.
2715                  */
2716                 if (ULONG_CMP_GE(jiffies, j + rcu_sysidle_delay()))
2717                         (void)cmpxchg(&full_sysidle_state,
2718                                       RCU_SYSIDLE_SHORT, RCU_SYSIDLE_LONG);
2719                 break;
2720 
2721         case RCU_SYSIDLE_LONG:
2722 
2723                 /*
2724                  * Do an additional check pass before advancing to full.
2725                  * cmpxchg failure means race with non-idle, let them win.
2726                  */
2727                 if (ULONG_CMP_GE(jiffies, j + rcu_sysidle_delay()))
2728                         (void)cmpxchg(&full_sysidle_state,
2729                                       RCU_SYSIDLE_LONG, RCU_SYSIDLE_FULL);
2730                 break;
2731 
2732         default:
2733                 break;
2734         }
2735 }
2736 
2737 /*
2738  * Found a non-idle non-timekeeping CPU, so kick the system-idle state
2739  * back to the beginning.
2740  */
2741 static void rcu_sysidle_cancel(void)
2742 {
2743         smp_mb();
2744         if (full_sysidle_state > RCU_SYSIDLE_SHORT)
2745                 WRITE_ONCE(full_sysidle_state, RCU_SYSIDLE_NOT);
2746 }
2747 
2748 /*
2749  * Update the sysidle state based on the results of a force-quiescent-state
2750  * scan of the CPUs' dyntick-idle state.
2751  */
2752 static void rcu_sysidle_report(struct rcu_state *rsp, int isidle,
2753                                unsigned long maxj, bool gpkt)
2754 {
2755         if (rsp != rcu_state_p)
2756                 return;  /* Wrong flavor, ignore. */
2757         if (gpkt && nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL)
2758                 return;  /* Running state machine from timekeeping CPU. */
2759         if (isidle)
2760                 rcu_sysidle(maxj);    /* More idle! */
2761         else
2762                 rcu_sysidle_cancel(); /* Idle is over. */
2763 }
2764 
2765 /*
2766  * Wrapper for rcu_sysidle_report() when called from the grace-period
2767  * kthread's context.
2768  */
2769 static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
2770                                   unsigned long maxj)
2771 {
2772         /* If there are no nohz_full= CPUs, no need to track this. */
2773         if (!tick_nohz_full_enabled())
2774                 return;
2775 
2776         rcu_sysidle_report(rsp, isidle, maxj, true);
2777 }
2778 
2779 /* Callback and function for forcing an RCU grace period. */
2780 struct rcu_sysidle_head {
2781         struct rcu_head rh;
2782         int inuse;
2783 };
2784 
2785 static void rcu_sysidle_cb(struct rcu_head *rhp)
2786 {
2787         struct rcu_sysidle_head *rshp;
2788 
2789         /*
2790          * The following memory barrier is needed to replace the
2791          * memory barriers that would normally be in the memory
2792          * allocator.
2793          */
2794         smp_mb();  /* grace period precedes setting inuse. */
2795 
2796         rshp = container_of(rhp, struct rcu_sysidle_head, rh);
2797         WRITE_ONCE(rshp->inuse, 0);
2798 }
2799 
2800 /*
2801  * Check to see if the system is fully idle, other than the timekeeping CPU.
2802  * The caller must have disabled interrupts.  This is not intended to be
2803  * called unless tick_nohz_full_enabled().
2804  */
2805 bool rcu_sys_is_idle(void)
2806 {
2807         static struct rcu_sysidle_head rsh;
2808         int rss = READ_ONCE(full_sysidle_state);
2809 
2810         if (WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu))
2811                 return false;
2812 
2813         /* Handle small-system case by doing a full scan of CPUs. */
2814         if (nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL) {
2815                 int oldrss = rss - 1;
2816 
2817                 /*
2818                  * One pass to advance to each state up to _FULL.
2819                  * Give up if any pass fails to advance the state.
2820                  */
2821                 while (rss < RCU_SYSIDLE_FULL && oldrss < rss) {
2822                         int cpu;
2823                         bool isidle = true;
2824                         unsigned long maxj = jiffies - ULONG_MAX / 4;
2825                         struct rcu_data *rdp;
2826 
2827                         /* Scan all the CPUs looking for nonidle CPUs. */
2828                         for_each_possible_cpu(cpu) {
2829                                 rdp = per_cpu_ptr(rcu_state_p->rda, cpu);
2830                                 rcu_sysidle_check_cpu(rdp, &isidle, &maxj);
2831                                 if (!isidle)
2832                                         break;
2833                         }
2834                         rcu_sysidle_report(rcu_state_p, isidle, maxj, false);
2835                         oldrss = rss;
2836                         rss = READ_ONCE(full_sysidle_state);
2837                 }
2838         }
2839 
2840         /* If this is the first observation of an idle period, record it. */
2841         if (rss == RCU_SYSIDLE_FULL) {
2842                 rss = cmpxchg(&full_sysidle_state,
2843                               RCU_SYSIDLE_FULL, RCU_SYSIDLE_FULL_NOTED);
2844                 return rss == RCU_SYSIDLE_FULL;
2845         }
2846 
2847         smp_mb(); /* ensure rss load happens before later caller actions. */
2848 
2849         /* If already fully idle, tell the caller (in case of races). */
2850         if (rss == RCU_SYSIDLE_FULL_NOTED)
2851                 return true;
2852 
2853         /*
2854          * If we aren't there yet, and a grace period is not in flight,
2855          * initiate a grace period.  Either way, tell the caller that
2856          * we are not there yet.  We use an xchg() rather than an assignment
2857          * to make up for the memory barriers that would otherwise be
2858          * provided by the memory allocator.
2859          */
2860         if (nr_cpu_ids > CONFIG_NO_HZ_FULL_SYSIDLE_SMALL &&
2861             !rcu_gp_in_progress(rcu_state_p) &&
2862             !rsh.inuse && xchg(&rsh.inuse, 1) == 0)
2863                 call_rcu(&rsh.rh, rcu_sysidle_cb);
2864         return false;
2865 }
2866 
2867 /*
2868  * Initialize dynticks sysidle state for CPUs coming online.
2869  */
2870 static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp)
2871 {
2872         rdtp->dynticks_idle_nesting = DYNTICK_TASK_NEST_VALUE;
2873 }
2874 
2875 #else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
2876 
2877 static void rcu_sysidle_enter(int irq)
2878 {
2879 }
2880 
2881 static void rcu_sysidle_exit(int irq)
2882 {
2883 }
2884 
2885 static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
2886                                   unsigned long *maxj)
2887 {
2888 }
2889 
2890 static bool is_sysidle_rcu_state(struct rcu_state *rsp)
2891 {
2892         return false;
2893 }
2894 
2895 static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
2896                                   unsigned long maxj)
2897 {
2898 }
2899 
2900 static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp)
2901 {
2902 }
2903 
2904 #endif /* #else #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
2905 
2906 /*
2907  * Is this CPU a NO_HZ_FULL CPU that should ignore RCU so that the
2908  * grace-period kthread will do force_quiescent_state() processing?
2909  * The idea is to avoid waking up RCU core processing on such a
2910  * CPU unless the grace period has extended for too long.
2911  *
2912  * This code relies on the fact that all NO_HZ_FULL CPUs are also
2913  * CONFIG_RCU_NOCB_CPU CPUs.
2914  */
2915 static bool rcu_nohz_full_cpu(struct rcu_state *rsp)
2916 {
2917 #ifdef CONFIG_NO_HZ_FULL
2918         if (tick_nohz_full_cpu(smp_processor_id()) &&
2919             (!rcu_gp_in_progress(rsp) ||
2920              ULONG_CMP_LT(jiffies, READ_ONCE(rsp->gp_start) + HZ)))
2921                 return true;
2922 #endif /* #ifdef CONFIG_NO_HZ_FULL */
2923         return false;
2924 }
2925 
2926 /*
2927  * Bind the grace-period kthread for the sysidle flavor of RCU to the
2928  * timekeeping CPU.
2929  */
2930 static void rcu_bind_gp_kthread(void)
2931 {
2932         int __maybe_unused cpu;
2933 
2934         if (!tick_nohz_full_enabled())
2935                 return;
2936 #ifdef CONFIG_NO_HZ_FULL_SYSIDLE
2937         cpu = tick_do_timer_cpu;
2938         if (cpu >= 0 && cpu < nr_cpu_ids)
2939                 set_cpus_allowed_ptr(current, cpumask_of(cpu));
2940 #else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
2941         housekeeping_affine(current);
2942 #endif /* #else #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
2943 }
2944 
2945 /* Record the current task on dyntick-idle entry. */
2946 static void rcu_dynticks_task_enter(void)
2947 {
2948 #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
2949         WRITE_ONCE(current->rcu_tasks_idle_cpu, smp_processor_id());
2950 #endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */
2951 }
2952 
2953 /* Record no current task on dyntick-idle exit. */
2954 static void rcu_dynticks_task_exit(void)
2955 {
2956 #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
2957         WRITE_ONCE(current->rcu_tasks_idle_cpu, -1);
2958 #endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */
2959 }
2960 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp