~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/rcu/tree_plugin.h

Version: ~ [ linux-6.1-rc7 ] ~ [ linux-6.0.10 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.80 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.156 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.225 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.267 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.300 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.334 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Read-Copy Update mechanism for mutual exclusion (tree-based version)
  3  * Internal non-public definitions that provide either classic
  4  * or preemptible semantics.
  5  *
  6  * This program is free software; you can redistribute it and/or modify
  7  * it under the terms of the GNU General Public License as published by
  8  * the Free Software Foundation; either version 2 of the License, or
  9  * (at your option) any later version.
 10  *
 11  * This program is distributed in the hope that it will be useful,
 12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 14  * GNU General Public License for more details.
 15  *
 16  * You should have received a copy of the GNU General Public License
 17  * along with this program; if not, you can access it online at
 18  * http://www.gnu.org/licenses/gpl-2.0.html.
 19  *
 20  * Copyright Red Hat, 2009
 21  * Copyright IBM Corporation, 2009
 22  *
 23  * Author: Ingo Molnar <mingo@elte.hu>
 24  *         Paul E. McKenney <paulmck@linux.vnet.ibm.com>
 25  */
 26 
 27 #include <linux/delay.h>
 28 #include <linux/gfp.h>
 29 #include <linux/oom.h>
 30 #include <linux/smpboot.h>
 31 #include "../time/tick-internal.h"
 32 
 33 #ifdef CONFIG_RCU_BOOST
 34 
 35 #include "../locking/rtmutex_common.h"
 36 
 37 /*
 38  * Control variables for per-CPU and per-rcu_node kthreads.  These
 39  * handle all flavors of RCU.
 40  */
 41 static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);
 42 DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
 43 DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
 44 DEFINE_PER_CPU(char, rcu_cpu_has_work);
 45 
 46 #endif /* #ifdef CONFIG_RCU_BOOST */
 47 
 48 #ifdef CONFIG_RCU_NOCB_CPU
 49 static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */
 50 static bool have_rcu_nocb_mask;     /* Was rcu_nocb_mask allocated? */
 51 static bool __read_mostly rcu_nocb_poll;    /* Offload kthread are to poll. */
 52 #endif /* #ifdef CONFIG_RCU_NOCB_CPU */
 53 
 54 /*
 55  * Check the RCU kernel configuration parameters and print informative
 56  * messages about anything out of the ordinary.  If you like #ifdef, you
 57  * will love this function.
 58  */
 59 static void __init rcu_bootup_announce_oddness(void)
 60 {
 61         if (IS_ENABLED(CONFIG_RCU_TRACE))
 62                 pr_info("\tRCU debugfs-based tracing is enabled.\n");
 63         if ((IS_ENABLED(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 64) ||
 64             (!IS_ENABLED(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 32))
 65                 pr_info("\tCONFIG_RCU_FANOUT set to non-default value of %d\n",
 66                        CONFIG_RCU_FANOUT);
 67         if (IS_ENABLED(CONFIG_RCU_FANOUT_EXACT))
 68                 pr_info("\tHierarchical RCU autobalancing is disabled.\n");
 69         if (IS_ENABLED(CONFIG_RCU_FAST_NO_HZ))
 70                 pr_info("\tRCU dyntick-idle grace-period acceleration is enabled.\n");
 71         if (IS_ENABLED(CONFIG_PROVE_RCU))
 72                 pr_info("\tRCU lockdep checking is enabled.\n");
 73         if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_RUNNABLE))
 74                 pr_info("\tRCU torture testing starts during boot.\n");
 75         if (IS_ENABLED(CONFIG_RCU_CPU_STALL_INFO))
 76                 pr_info("\tAdditional per-CPU info printed with stalls.\n");
 77         if (NUM_RCU_LVL_4 != 0)
 78                 pr_info("\tFour-level hierarchy is enabled.\n");
 79         if (CONFIG_RCU_FANOUT_LEAF != 16)
 80                 pr_info("\tBuild-time adjustment of leaf fanout to %d.\n",
 81                         CONFIG_RCU_FANOUT_LEAF);
 82         if (rcu_fanout_leaf != CONFIG_RCU_FANOUT_LEAF)
 83                 pr_info("\tBoot-time adjustment of leaf fanout to %d.\n", rcu_fanout_leaf);
 84         if (nr_cpu_ids != NR_CPUS)
 85                 pr_info("\tRCU restricting CPUs from NR_CPUS=%d to nr_cpu_ids=%d.\n", NR_CPUS, nr_cpu_ids);
 86         if (IS_ENABLED(CONFIG_RCU_BOOST))
 87                 pr_info("\tRCU kthread priority: %d.\n", kthread_prio);
 88 }
 89 
 90 #ifdef CONFIG_PREEMPT_RCU
 91 
 92 RCU_STATE_INITIALIZER(rcu_preempt, 'p', call_rcu);
 93 static struct rcu_state *rcu_state_p = &rcu_preempt_state;
 94 
 95 static int rcu_preempted_readers_exp(struct rcu_node *rnp);
 96 static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
 97                                bool wake);
 98 
 99 /*
100  * Tell them what RCU they are running.
101  */
102 static void __init rcu_bootup_announce(void)
103 {
104         pr_info("Preemptible hierarchical RCU implementation.\n");
105         rcu_bootup_announce_oddness();
106 }
107 
108 /*
109  * Record a preemptible-RCU quiescent state for the specified CPU.  Note
110  * that this just means that the task currently running on the CPU is
111  * not in a quiescent state.  There might be any number of tasks blocked
112  * while in an RCU read-side critical section.
113  *
114  * As with the other rcu_*_qs() functions, callers to this function
115  * must disable preemption.
116  */
117 static void rcu_preempt_qs(void)
118 {
119         if (!__this_cpu_read(rcu_preempt_data.passed_quiesce)) {
120                 trace_rcu_grace_period(TPS("rcu_preempt"),
121                                        __this_cpu_read(rcu_preempt_data.gpnum),
122                                        TPS("cpuqs"));
123                 __this_cpu_write(rcu_preempt_data.passed_quiesce, 1);
124                 barrier(); /* Coordinate with rcu_preempt_check_callbacks(). */
125                 current->rcu_read_unlock_special.b.need_qs = false;
126         }
127 }
128 
129 /*
130  * We have entered the scheduler, and the current task might soon be
131  * context-switched away from.  If this task is in an RCU read-side
132  * critical section, we will no longer be able to rely on the CPU to
133  * record that fact, so we enqueue the task on the blkd_tasks list.
134  * The task will dequeue itself when it exits the outermost enclosing
135  * RCU read-side critical section.  Therefore, the current grace period
136  * cannot be permitted to complete until the blkd_tasks list entries
137  * predating the current grace period drain, in other words, until
138  * rnp->gp_tasks becomes NULL.
139  *
140  * Caller must disable preemption.
141  */
142 static void rcu_preempt_note_context_switch(void)
143 {
144         struct task_struct *t = current;
145         unsigned long flags;
146         struct rcu_data *rdp;
147         struct rcu_node *rnp;
148 
149         if (t->rcu_read_lock_nesting > 0 &&
150             !t->rcu_read_unlock_special.b.blocked) {
151 
152                 /* Possibly blocking in an RCU read-side critical section. */
153                 rdp = this_cpu_ptr(rcu_preempt_state.rda);
154                 rnp = rdp->mynode;
155                 raw_spin_lock_irqsave(&rnp->lock, flags);
156                 smp_mb__after_unlock_lock();
157                 t->rcu_read_unlock_special.b.blocked = true;
158                 t->rcu_blocked_node = rnp;
159 
160                 /*
161                  * If this CPU has already checked in, then this task
162                  * will hold up the next grace period rather than the
163                  * current grace period.  Queue the task accordingly.
164                  * If the task is queued for the current grace period
165                  * (i.e., this CPU has not yet passed through a quiescent
166                  * state for the current grace period), then as long
167                  * as that task remains queued, the current grace period
168                  * cannot end.  Note that there is some uncertainty as
169                  * to exactly when the current grace period started.
170                  * We take a conservative approach, which can result
171                  * in unnecessarily waiting on tasks that started very
172                  * slightly after the current grace period began.  C'est
173                  * la vie!!!
174                  *
175                  * But first, note that the current CPU must still be
176                  * on line!
177                  */
178                 WARN_ON_ONCE((rdp->grpmask & rcu_rnp_online_cpus(rnp)) == 0);
179                 WARN_ON_ONCE(!list_empty(&t->rcu_node_entry));
180                 if ((rnp->qsmask & rdp->grpmask) && rnp->gp_tasks != NULL) {
181                         list_add(&t->rcu_node_entry, rnp->gp_tasks->prev);
182                         rnp->gp_tasks = &t->rcu_node_entry;
183 #ifdef CONFIG_RCU_BOOST
184                         if (rnp->boost_tasks != NULL)
185                                 rnp->boost_tasks = rnp->gp_tasks;
186 #endif /* #ifdef CONFIG_RCU_BOOST */
187                 } else {
188                         list_add(&t->rcu_node_entry, &rnp->blkd_tasks);
189                         if (rnp->qsmask & rdp->grpmask)
190                                 rnp->gp_tasks = &t->rcu_node_entry;
191                 }
192                 trace_rcu_preempt_task(rdp->rsp->name,
193                                        t->pid,
194                                        (rnp->qsmask & rdp->grpmask)
195                                        ? rnp->gpnum
196                                        : rnp->gpnum + 1);
197                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
198         } else if (t->rcu_read_lock_nesting < 0 &&
199                    t->rcu_read_unlock_special.s) {
200 
201                 /*
202                  * Complete exit from RCU read-side critical section on
203                  * behalf of preempted instance of __rcu_read_unlock().
204                  */
205                 rcu_read_unlock_special(t);
206         }
207 
208         /*
209          * Either we were not in an RCU read-side critical section to
210          * begin with, or we have now recorded that critical section
211          * globally.  Either way, we can now note a quiescent state
212          * for this CPU.  Again, if we were in an RCU read-side critical
213          * section, and if that critical section was blocking the current
214          * grace period, then the fact that the task has been enqueued
215          * means that we continue to block the current grace period.
216          */
217         rcu_preempt_qs();
218 }
219 
220 /*
221  * Check for preempted RCU readers blocking the current grace period
222  * for the specified rcu_node structure.  If the caller needs a reliable
223  * answer, it must hold the rcu_node's ->lock.
224  */
225 static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
226 {
227         return rnp->gp_tasks != NULL;
228 }
229 
230 /*
231  * Advance a ->blkd_tasks-list pointer to the next entry, instead
232  * returning NULL if at the end of the list.
233  */
234 static struct list_head *rcu_next_node_entry(struct task_struct *t,
235                                              struct rcu_node *rnp)
236 {
237         struct list_head *np;
238 
239         np = t->rcu_node_entry.next;
240         if (np == &rnp->blkd_tasks)
241                 np = NULL;
242         return np;
243 }
244 
245 /*
246  * Return true if the specified rcu_node structure has tasks that were
247  * preempted within an RCU read-side critical section.
248  */
249 static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
250 {
251         return !list_empty(&rnp->blkd_tasks);
252 }
253 
254 /*
255  * Handle special cases during rcu_read_unlock(), such as needing to
256  * notify RCU core processing or task having blocked during the RCU
257  * read-side critical section.
258  */
259 void rcu_read_unlock_special(struct task_struct *t)
260 {
261         bool empty_exp;
262         bool empty_norm;
263         bool empty_exp_now;
264         unsigned long flags;
265         struct list_head *np;
266 #ifdef CONFIG_RCU_BOOST
267         bool drop_boost_mutex = false;
268 #endif /* #ifdef CONFIG_RCU_BOOST */
269         struct rcu_node *rnp;
270         union rcu_special special;
271 
272         /* NMI handlers cannot block and cannot safely manipulate state. */
273         if (in_nmi())
274                 return;
275 
276         local_irq_save(flags);
277 
278         /*
279          * If RCU core is waiting for this CPU to exit critical section,
280          * let it know that we have done so.  Because irqs are disabled,
281          * t->rcu_read_unlock_special cannot change.
282          */
283         special = t->rcu_read_unlock_special;
284         if (special.b.need_qs) {
285                 rcu_preempt_qs();
286                 t->rcu_read_unlock_special.b.need_qs = false;
287                 if (!t->rcu_read_unlock_special.s) {
288                         local_irq_restore(flags);
289                         return;
290                 }
291         }
292 
293         /* Hardware IRQ handlers cannot block, complain if they get here. */
294         if (in_irq() || in_serving_softirq()) {
295                 lockdep_rcu_suspicious(__FILE__, __LINE__,
296                                        "rcu_read_unlock() from irq or softirq with blocking in critical section!!!\n");
297                 pr_alert("->rcu_read_unlock_special: %#x (b: %d, nq: %d)\n",
298                          t->rcu_read_unlock_special.s,
299                          t->rcu_read_unlock_special.b.blocked,
300                          t->rcu_read_unlock_special.b.need_qs);
301                 local_irq_restore(flags);
302                 return;
303         }
304 
305         /* Clean up if blocked during RCU read-side critical section. */
306         if (special.b.blocked) {
307                 t->rcu_read_unlock_special.b.blocked = false;
308 
309                 /*
310                  * Remove this task from the list it blocked on.  The
311                  * task can migrate while we acquire the lock, but at
312                  * most one time.  So at most two passes through loop.
313                  */
314                 for (;;) {
315                         rnp = t->rcu_blocked_node;
316                         raw_spin_lock(&rnp->lock);  /* irqs already disabled. */
317                         smp_mb__after_unlock_lock();
318                         if (rnp == t->rcu_blocked_node)
319                                 break;
320                         raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
321                 }
322                 empty_norm = !rcu_preempt_blocked_readers_cgp(rnp);
323                 empty_exp = !rcu_preempted_readers_exp(rnp);
324                 smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */
325                 np = rcu_next_node_entry(t, rnp);
326                 list_del_init(&t->rcu_node_entry);
327                 t->rcu_blocked_node = NULL;
328                 trace_rcu_unlock_preempted_task(TPS("rcu_preempt"),
329                                                 rnp->gpnum, t->pid);
330                 if (&t->rcu_node_entry == rnp->gp_tasks)
331                         rnp->gp_tasks = np;
332                 if (&t->rcu_node_entry == rnp->exp_tasks)
333                         rnp->exp_tasks = np;
334 #ifdef CONFIG_RCU_BOOST
335                 if (&t->rcu_node_entry == rnp->boost_tasks)
336                         rnp->boost_tasks = np;
337                 /* Snapshot ->boost_mtx ownership with rcu_node lock held. */
338                 drop_boost_mutex = rt_mutex_owner(&rnp->boost_mtx) == t;
339 #endif /* #ifdef CONFIG_RCU_BOOST */
340 
341                 /*
342                  * If this was the last task on the current list, and if
343                  * we aren't waiting on any CPUs, report the quiescent state.
344                  * Note that rcu_report_unblock_qs_rnp() releases rnp->lock,
345                  * so we must take a snapshot of the expedited state.
346                  */
347                 empty_exp_now = !rcu_preempted_readers_exp(rnp);
348                 if (!empty_norm && !rcu_preempt_blocked_readers_cgp(rnp)) {
349                         trace_rcu_quiescent_state_report(TPS("preempt_rcu"),
350                                                          rnp->gpnum,
351                                                          0, rnp->qsmask,
352                                                          rnp->level,
353                                                          rnp->grplo,
354                                                          rnp->grphi,
355                                                          !!rnp->gp_tasks);
356                         rcu_report_unblock_qs_rnp(&rcu_preempt_state,
357                                                   rnp, flags);
358                 } else {
359                         raw_spin_unlock_irqrestore(&rnp->lock, flags);
360                 }
361 
362 #ifdef CONFIG_RCU_BOOST
363                 /* Unboost if we were boosted. */
364                 if (drop_boost_mutex)
365                         rt_mutex_unlock(&rnp->boost_mtx);
366 #endif /* #ifdef CONFIG_RCU_BOOST */
367 
368                 /*
369                  * If this was the last task on the expedited lists,
370                  * then we need to report up the rcu_node hierarchy.
371                  */
372                 if (!empty_exp && empty_exp_now)
373                         rcu_report_exp_rnp(&rcu_preempt_state, rnp, true);
374         } else {
375                 local_irq_restore(flags);
376         }
377 }
378 
379 /*
380  * Dump detailed information for all tasks blocking the current RCU
381  * grace period on the specified rcu_node structure.
382  */
383 static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
384 {
385         unsigned long flags;
386         struct task_struct *t;
387 
388         raw_spin_lock_irqsave(&rnp->lock, flags);
389         if (!rcu_preempt_blocked_readers_cgp(rnp)) {
390                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
391                 return;
392         }
393         t = list_entry(rnp->gp_tasks,
394                        struct task_struct, rcu_node_entry);
395         list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry)
396                 sched_show_task(t);
397         raw_spin_unlock_irqrestore(&rnp->lock, flags);
398 }
399 
400 /*
401  * Dump detailed information for all tasks blocking the current RCU
402  * grace period.
403  */
404 static void rcu_print_detail_task_stall(struct rcu_state *rsp)
405 {
406         struct rcu_node *rnp = rcu_get_root(rsp);
407 
408         rcu_print_detail_task_stall_rnp(rnp);
409         rcu_for_each_leaf_node(rsp, rnp)
410                 rcu_print_detail_task_stall_rnp(rnp);
411 }
412 
413 #ifdef CONFIG_RCU_CPU_STALL_INFO
414 
415 static void rcu_print_task_stall_begin(struct rcu_node *rnp)
416 {
417         pr_err("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):",
418                rnp->level, rnp->grplo, rnp->grphi);
419 }
420 
421 static void rcu_print_task_stall_end(void)
422 {
423         pr_cont("\n");
424 }
425 
426 #else /* #ifdef CONFIG_RCU_CPU_STALL_INFO */
427 
428 static void rcu_print_task_stall_begin(struct rcu_node *rnp)
429 {
430 }
431 
432 static void rcu_print_task_stall_end(void)
433 {
434 }
435 
436 #endif /* #else #ifdef CONFIG_RCU_CPU_STALL_INFO */
437 
438 /*
439  * Scan the current list of tasks blocked within RCU read-side critical
440  * sections, printing out the tid of each.
441  */
442 static int rcu_print_task_stall(struct rcu_node *rnp)
443 {
444         struct task_struct *t;
445         int ndetected = 0;
446 
447         if (!rcu_preempt_blocked_readers_cgp(rnp))
448                 return 0;
449         rcu_print_task_stall_begin(rnp);
450         t = list_entry(rnp->gp_tasks,
451                        struct task_struct, rcu_node_entry);
452         list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
453                 pr_cont(" P%d", t->pid);
454                 ndetected++;
455         }
456         rcu_print_task_stall_end();
457         return ndetected;
458 }
459 
460 /*
461  * Check that the list of blocked tasks for the newly completed grace
462  * period is in fact empty.  It is a serious bug to complete a grace
463  * period that still has RCU readers blocked!  This function must be
464  * invoked -before- updating this rnp's ->gpnum, and the rnp's ->lock
465  * must be held by the caller.
466  *
467  * Also, if there are blocked tasks on the list, they automatically
468  * block the newly created grace period, so set up ->gp_tasks accordingly.
469  */
470 static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
471 {
472         WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp));
473         if (rcu_preempt_has_tasks(rnp))
474                 rnp->gp_tasks = rnp->blkd_tasks.next;
475         WARN_ON_ONCE(rnp->qsmask);
476 }
477 
478 /*
479  * Check for a quiescent state from the current CPU.  When a task blocks,
480  * the task is recorded in the corresponding CPU's rcu_node structure,
481  * which is checked elsewhere.
482  *
483  * Caller must disable hard irqs.
484  */
485 static void rcu_preempt_check_callbacks(void)
486 {
487         struct task_struct *t = current;
488 
489         if (t->rcu_read_lock_nesting == 0) {
490                 rcu_preempt_qs();
491                 return;
492         }
493         if (t->rcu_read_lock_nesting > 0 &&
494             __this_cpu_read(rcu_preempt_data.qs_pending) &&
495             !__this_cpu_read(rcu_preempt_data.passed_quiesce))
496                 t->rcu_read_unlock_special.b.need_qs = true;
497 }
498 
499 #ifdef CONFIG_RCU_BOOST
500 
501 static void rcu_preempt_do_callbacks(void)
502 {
503         rcu_do_batch(&rcu_preempt_state, this_cpu_ptr(&rcu_preempt_data));
504 }
505 
506 #endif /* #ifdef CONFIG_RCU_BOOST */
507 
508 /*
509  * Queue a preemptible-RCU callback for invocation after a grace period.
510  */
511 void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
512 {
513         __call_rcu(head, func, &rcu_preempt_state, -1, 0);
514 }
515 EXPORT_SYMBOL_GPL(call_rcu);
516 
517 /**
518  * synchronize_rcu - wait until a grace period has elapsed.
519  *
520  * Control will return to the caller some time after a full grace
521  * period has elapsed, in other words after all currently executing RCU
522  * read-side critical sections have completed.  Note, however, that
523  * upon return from synchronize_rcu(), the caller might well be executing
524  * concurrently with new RCU read-side critical sections that began while
525  * synchronize_rcu() was waiting.  RCU read-side critical sections are
526  * delimited by rcu_read_lock() and rcu_read_unlock(), and may be nested.
527  *
528  * See the description of synchronize_sched() for more detailed information
529  * on memory ordering guarantees.
530  */
531 void synchronize_rcu(void)
532 {
533         rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map) &&
534                            !lock_is_held(&rcu_lock_map) &&
535                            !lock_is_held(&rcu_sched_lock_map),
536                            "Illegal synchronize_rcu() in RCU read-side critical section");
537         if (!rcu_scheduler_active)
538                 return;
539         if (rcu_gp_is_expedited())
540                 synchronize_rcu_expedited();
541         else
542                 wait_rcu_gp(call_rcu);
543 }
544 EXPORT_SYMBOL_GPL(synchronize_rcu);
545 
546 static DECLARE_WAIT_QUEUE_HEAD(sync_rcu_preempt_exp_wq);
547 static unsigned long sync_rcu_preempt_exp_count;
548 static DEFINE_MUTEX(sync_rcu_preempt_exp_mutex);
549 
550 /*
551  * Return non-zero if there are any tasks in RCU read-side critical
552  * sections blocking the current preemptible-RCU expedited grace period.
553  * If there is no preemptible-RCU expedited grace period currently in
554  * progress, returns zero unconditionally.
555  */
556 static int rcu_preempted_readers_exp(struct rcu_node *rnp)
557 {
558         return rnp->exp_tasks != NULL;
559 }
560 
561 /*
562  * return non-zero if there is no RCU expedited grace period in progress
563  * for the specified rcu_node structure, in other words, if all CPUs and
564  * tasks covered by the specified rcu_node structure have done their bit
565  * for the current expedited grace period.  Works only for preemptible
566  * RCU -- other RCU implementation use other means.
567  *
568  * Caller must hold sync_rcu_preempt_exp_mutex.
569  */
570 static int sync_rcu_preempt_exp_done(struct rcu_node *rnp)
571 {
572         return !rcu_preempted_readers_exp(rnp) &&
573                ACCESS_ONCE(rnp->expmask) == 0;
574 }
575 
576 /*
577  * Report the exit from RCU read-side critical section for the last task
578  * that queued itself during or before the current expedited preemptible-RCU
579  * grace period.  This event is reported either to the rcu_node structure on
580  * which the task was queued or to one of that rcu_node structure's ancestors,
581  * recursively up the tree.  (Calm down, calm down, we do the recursion
582  * iteratively!)
583  *
584  * Caller must hold sync_rcu_preempt_exp_mutex.
585  */
586 static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
587                                bool wake)
588 {
589         unsigned long flags;
590         unsigned long mask;
591 
592         raw_spin_lock_irqsave(&rnp->lock, flags);
593         smp_mb__after_unlock_lock();
594         for (;;) {
595                 if (!sync_rcu_preempt_exp_done(rnp)) {
596                         raw_spin_unlock_irqrestore(&rnp->lock, flags);
597                         break;
598                 }
599                 if (rnp->parent == NULL) {
600                         raw_spin_unlock_irqrestore(&rnp->lock, flags);
601                         if (wake) {
602                                 smp_mb(); /* EGP done before wake_up(). */
603                                 wake_up(&sync_rcu_preempt_exp_wq);
604                         }
605                         break;
606                 }
607                 mask = rnp->grpmask;
608                 raw_spin_unlock(&rnp->lock); /* irqs remain disabled */
609                 rnp = rnp->parent;
610                 raw_spin_lock(&rnp->lock); /* irqs already disabled */
611                 smp_mb__after_unlock_lock();
612                 rnp->expmask &= ~mask;
613         }
614 }
615 
616 /*
617  * Snapshot the tasks blocking the newly started preemptible-RCU expedited
618  * grace period for the specified rcu_node structure, phase 1.  If there
619  * are such tasks, set the ->expmask bits up the rcu_node tree and also
620  * set the ->expmask bits on the leaf rcu_node structures to tell phase 2
621  * that work is needed here.
622  *
623  * Caller must hold sync_rcu_preempt_exp_mutex.
624  */
625 static void
626 sync_rcu_preempt_exp_init1(struct rcu_state *rsp, struct rcu_node *rnp)
627 {
628         unsigned long flags;
629         unsigned long mask;
630         struct rcu_node *rnp_up;
631 
632         raw_spin_lock_irqsave(&rnp->lock, flags);
633         smp_mb__after_unlock_lock();
634         WARN_ON_ONCE(rnp->expmask);
635         WARN_ON_ONCE(rnp->exp_tasks);
636         if (!rcu_preempt_has_tasks(rnp)) {
637                 /* No blocked tasks, nothing to do. */
638                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
639                 return;
640         }
641         /* Call for Phase 2 and propagate ->expmask bits up the tree. */
642         rnp->expmask = 1;
643         rnp_up = rnp;
644         while (rnp_up->parent) {
645                 mask = rnp_up->grpmask;
646                 rnp_up = rnp_up->parent;
647                 if (rnp_up->expmask & mask)
648                         break;
649                 raw_spin_lock(&rnp_up->lock); /* irqs already off */
650                 smp_mb__after_unlock_lock();
651                 rnp_up->expmask |= mask;
652                 raw_spin_unlock(&rnp_up->lock); /* irqs still off */
653         }
654         raw_spin_unlock_irqrestore(&rnp->lock, flags);
655 }
656 
657 /*
658  * Snapshot the tasks blocking the newly started preemptible-RCU expedited
659  * grace period for the specified rcu_node structure, phase 2.  If the
660  * leaf rcu_node structure has its ->expmask field set, check for tasks.
661  * If there are some, clear ->expmask and set ->exp_tasks accordingly,
662  * then initiate RCU priority boosting.  Otherwise, clear ->expmask and
663  * invoke rcu_report_exp_rnp() to clear out the upper-level ->expmask bits,
664  * enabling rcu_read_unlock_special() to do the bit-clearing.
665  *
666  * Caller must hold sync_rcu_preempt_exp_mutex.
667  */
668 static void
669 sync_rcu_preempt_exp_init2(struct rcu_state *rsp, struct rcu_node *rnp)
670 {
671         unsigned long flags;
672 
673         raw_spin_lock_irqsave(&rnp->lock, flags);
674         smp_mb__after_unlock_lock();
675         if (!rnp->expmask) {
676                 /* Phase 1 didn't do anything, so Phase 2 doesn't either. */
677                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
678                 return;
679         }
680 
681         /* Phase 1 is over. */
682         rnp->expmask = 0;
683 
684         /*
685          * If there are still blocked tasks, set up ->exp_tasks so that
686          * rcu_read_unlock_special() will wake us and then boost them.
687          */
688         if (rcu_preempt_has_tasks(rnp)) {
689                 rnp->exp_tasks = rnp->blkd_tasks.next;
690                 rcu_initiate_boost(rnp, flags);  /* releases rnp->lock */
691                 return;
692         }
693 
694         /* No longer any blocked tasks, so undo bit setting. */
695         raw_spin_unlock_irqrestore(&rnp->lock, flags);
696         rcu_report_exp_rnp(rsp, rnp, false);
697 }
698 
699 /**
700  * synchronize_rcu_expedited - Brute-force RCU grace period
701  *
702  * Wait for an RCU-preempt grace period, but expedite it.  The basic
703  * idea is to invoke synchronize_sched_expedited() to push all the tasks to
704  * the ->blkd_tasks lists and wait for this list to drain.  This consumes
705  * significant time on all CPUs and is unfriendly to real-time workloads,
706  * so is thus not recommended for any sort of common-case code.
707  * In fact, if you are using synchronize_rcu_expedited() in a loop,
708  * please restructure your code to batch your updates, and then Use a
709  * single synchronize_rcu() instead.
710  */
711 void synchronize_rcu_expedited(void)
712 {
713         struct rcu_node *rnp;
714         struct rcu_state *rsp = &rcu_preempt_state;
715         unsigned long snap;
716         int trycount = 0;
717 
718         smp_mb(); /* Caller's modifications seen first by other CPUs. */
719         snap = ACCESS_ONCE(sync_rcu_preempt_exp_count) + 1;
720         smp_mb(); /* Above access cannot bleed into critical section. */
721 
722         /*
723          * Block CPU-hotplug operations.  This means that any CPU-hotplug
724          * operation that finds an rcu_node structure with tasks in the
725          * process of being boosted will know that all tasks blocking
726          * this expedited grace period will already be in the process of
727          * being boosted.  This simplifies the process of moving tasks
728          * from leaf to root rcu_node structures.
729          */
730         if (!try_get_online_cpus()) {
731                 /* CPU-hotplug operation in flight, fall back to normal GP. */
732                 wait_rcu_gp(call_rcu);
733                 return;
734         }
735 
736         /*
737          * Acquire lock, falling back to synchronize_rcu() if too many
738          * lock-acquisition failures.  Of course, if someone does the
739          * expedited grace period for us, just leave.
740          */
741         while (!mutex_trylock(&sync_rcu_preempt_exp_mutex)) {
742                 if (ULONG_CMP_LT(snap,
743                     ACCESS_ONCE(sync_rcu_preempt_exp_count))) {
744                         put_online_cpus();
745                         goto mb_ret; /* Others did our work for us. */
746                 }
747                 if (trycount++ < 10) {
748                         udelay(trycount * num_online_cpus());
749                 } else {
750                         put_online_cpus();
751                         wait_rcu_gp(call_rcu);
752                         return;
753                 }
754         }
755         if (ULONG_CMP_LT(snap, ACCESS_ONCE(sync_rcu_preempt_exp_count))) {
756                 put_online_cpus();
757                 goto unlock_mb_ret; /* Others did our work for us. */
758         }
759 
760         /* force all RCU readers onto ->blkd_tasks lists. */
761         synchronize_sched_expedited();
762 
763         /*
764          * Snapshot current state of ->blkd_tasks lists into ->expmask.
765          * Phase 1 sets bits and phase 2 permits rcu_read_unlock_special()
766          * to start clearing them.  Doing this in one phase leads to
767          * strange races between setting and clearing bits, so just say "no"!
768          */
769         rcu_for_each_leaf_node(rsp, rnp)
770                 sync_rcu_preempt_exp_init1(rsp, rnp);
771         rcu_for_each_leaf_node(rsp, rnp)
772                 sync_rcu_preempt_exp_init2(rsp, rnp);
773 
774         put_online_cpus();
775 
776         /* Wait for snapshotted ->blkd_tasks lists to drain. */
777         rnp = rcu_get_root(rsp);
778         wait_event(sync_rcu_preempt_exp_wq,
779                    sync_rcu_preempt_exp_done(rnp));
780 
781         /* Clean up and exit. */
782         smp_mb(); /* ensure expedited GP seen before counter increment. */
783         ACCESS_ONCE(sync_rcu_preempt_exp_count) =
784                                         sync_rcu_preempt_exp_count + 1;
785 unlock_mb_ret:
786         mutex_unlock(&sync_rcu_preempt_exp_mutex);
787 mb_ret:
788         smp_mb(); /* ensure subsequent action seen after grace period. */
789 }
790 EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
791 
792 /**
793  * rcu_barrier - Wait until all in-flight call_rcu() callbacks complete.
794  *
795  * Note that this primitive does not necessarily wait for an RCU grace period
796  * to complete.  For example, if there are no RCU callbacks queued anywhere
797  * in the system, then rcu_barrier() is within its rights to return
798  * immediately, without waiting for anything, much less an RCU grace period.
799  */
800 void rcu_barrier(void)
801 {
802         _rcu_barrier(&rcu_preempt_state);
803 }
804 EXPORT_SYMBOL_GPL(rcu_barrier);
805 
806 /*
807  * Initialize preemptible RCU's state structures.
808  */
809 static void __init __rcu_init_preempt(void)
810 {
811         rcu_init_one(&rcu_preempt_state, &rcu_preempt_data);
812 }
813 
814 /*
815  * Check for a task exiting while in a preemptible-RCU read-side
816  * critical section, clean up if so.  No need to issue warnings,
817  * as debug_check_no_locks_held() already does this if lockdep
818  * is enabled.
819  */
820 void exit_rcu(void)
821 {
822         struct task_struct *t = current;
823 
824         if (likely(list_empty(&current->rcu_node_entry)))
825                 return;
826         t->rcu_read_lock_nesting = 1;
827         barrier();
828         t->rcu_read_unlock_special.b.blocked = true;
829         __rcu_read_unlock();
830 }
831 
832 #else /* #ifdef CONFIG_PREEMPT_RCU */
833 
834 static struct rcu_state *rcu_state_p = &rcu_sched_state;
835 
836 /*
837  * Tell them what RCU they are running.
838  */
839 static void __init rcu_bootup_announce(void)
840 {
841         pr_info("Hierarchical RCU implementation.\n");
842         rcu_bootup_announce_oddness();
843 }
844 
845 /*
846  * Because preemptible RCU does not exist, we never have to check for
847  * CPUs being in quiescent states.
848  */
849 static void rcu_preempt_note_context_switch(void)
850 {
851 }
852 
853 /*
854  * Because preemptible RCU does not exist, there are never any preempted
855  * RCU readers.
856  */
857 static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
858 {
859         return 0;
860 }
861 
862 /*
863  * Because there is no preemptible RCU, there can be no readers blocked.
864  */
865 static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
866 {
867         return false;
868 }
869 
870 /*
871  * Because preemptible RCU does not exist, we never have to check for
872  * tasks blocked within RCU read-side critical sections.
873  */
874 static void rcu_print_detail_task_stall(struct rcu_state *rsp)
875 {
876 }
877 
878 /*
879  * Because preemptible RCU does not exist, we never have to check for
880  * tasks blocked within RCU read-side critical sections.
881  */
882 static int rcu_print_task_stall(struct rcu_node *rnp)
883 {
884         return 0;
885 }
886 
887 /*
888  * Because there is no preemptible RCU, there can be no readers blocked,
889  * so there is no need to check for blocked tasks.  So check only for
890  * bogus qsmask values.
891  */
892 static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
893 {
894         WARN_ON_ONCE(rnp->qsmask);
895 }
896 
897 /*
898  * Because preemptible RCU does not exist, it never has any callbacks
899  * to check.
900  */
901 static void rcu_preempt_check_callbacks(void)
902 {
903 }
904 
905 /*
906  * Wait for an rcu-preempt grace period, but make it happen quickly.
907  * But because preemptible RCU does not exist, map to rcu-sched.
908  */
909 void synchronize_rcu_expedited(void)
910 {
911         synchronize_sched_expedited();
912 }
913 EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
914 
915 /*
916  * Because preemptible RCU does not exist, rcu_barrier() is just
917  * another name for rcu_barrier_sched().
918  */
919 void rcu_barrier(void)
920 {
921         rcu_barrier_sched();
922 }
923 EXPORT_SYMBOL_GPL(rcu_barrier);
924 
925 /*
926  * Because preemptible RCU does not exist, it need not be initialized.
927  */
928 static void __init __rcu_init_preempt(void)
929 {
930 }
931 
932 /*
933  * Because preemptible RCU does not exist, tasks cannot possibly exit
934  * while in preemptible RCU read-side critical sections.
935  */
936 void exit_rcu(void)
937 {
938 }
939 
940 #endif /* #else #ifdef CONFIG_PREEMPT_RCU */
941 
942 #ifdef CONFIG_RCU_BOOST
943 
944 #include "../locking/rtmutex_common.h"
945 
946 #ifdef CONFIG_RCU_TRACE
947 
948 static void rcu_initiate_boost_trace(struct rcu_node *rnp)
949 {
950         if (!rcu_preempt_has_tasks(rnp))
951                 rnp->n_balk_blkd_tasks++;
952         else if (rnp->exp_tasks == NULL && rnp->gp_tasks == NULL)
953                 rnp->n_balk_exp_gp_tasks++;
954         else if (rnp->gp_tasks != NULL && rnp->boost_tasks != NULL)
955                 rnp->n_balk_boost_tasks++;
956         else if (rnp->gp_tasks != NULL && rnp->qsmask != 0)
957                 rnp->n_balk_notblocked++;
958         else if (rnp->gp_tasks != NULL &&
959                  ULONG_CMP_LT(jiffies, rnp->boost_time))
960                 rnp->n_balk_notyet++;
961         else
962                 rnp->n_balk_nos++;
963 }
964 
965 #else /* #ifdef CONFIG_RCU_TRACE */
966 
967 static void rcu_initiate_boost_trace(struct rcu_node *rnp)
968 {
969 }
970 
971 #endif /* #else #ifdef CONFIG_RCU_TRACE */
972 
973 static void rcu_wake_cond(struct task_struct *t, int status)
974 {
975         /*
976          * If the thread is yielding, only wake it when this
977          * is invoked from idle
978          */
979         if (status != RCU_KTHREAD_YIELDING || is_idle_task(current))
980                 wake_up_process(t);
981 }
982 
983 /*
984  * Carry out RCU priority boosting on the task indicated by ->exp_tasks
985  * or ->boost_tasks, advancing the pointer to the next task in the
986  * ->blkd_tasks list.
987  *
988  * Note that irqs must be enabled: boosting the task can block.
989  * Returns 1 if there are more tasks needing to be boosted.
990  */
991 static int rcu_boost(struct rcu_node *rnp)
992 {
993         unsigned long flags;
994         struct task_struct *t;
995         struct list_head *tb;
996 
997         if (ACCESS_ONCE(rnp->exp_tasks) == NULL &&
998             ACCESS_ONCE(rnp->boost_tasks) == NULL)
999                 return 0;  /* Nothing left to boost. */
1000 
1001         raw_spin_lock_irqsave(&rnp->lock, flags);
1002         smp_mb__after_unlock_lock();
1003 
1004         /*
1005          * Recheck under the lock: all tasks in need of boosting
1006          * might exit their RCU read-side critical sections on their own.
1007          */
1008         if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL) {
1009                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1010                 return 0;
1011         }
1012 
1013         /*
1014          * Preferentially boost tasks blocking expedited grace periods.
1015          * This cannot starve the normal grace periods because a second
1016          * expedited grace period must boost all blocked tasks, including
1017          * those blocking the pre-existing normal grace period.
1018          */
1019         if (rnp->exp_tasks != NULL) {
1020                 tb = rnp->exp_tasks;
1021                 rnp->n_exp_boosts++;
1022         } else {
1023                 tb = rnp->boost_tasks;
1024                 rnp->n_normal_boosts++;
1025         }
1026         rnp->n_tasks_boosted++;
1027 
1028         /*
1029          * We boost task t by manufacturing an rt_mutex that appears to
1030          * be held by task t.  We leave a pointer to that rt_mutex where
1031          * task t can find it, and task t will release the mutex when it
1032          * exits its outermost RCU read-side critical section.  Then
1033          * simply acquiring this artificial rt_mutex will boost task
1034          * t's priority.  (Thanks to tglx for suggesting this approach!)
1035          *
1036          * Note that task t must acquire rnp->lock to remove itself from
1037          * the ->blkd_tasks list, which it will do from exit() if from
1038          * nowhere else.  We therefore are guaranteed that task t will
1039          * stay around at least until we drop rnp->lock.  Note that
1040          * rnp->lock also resolves races between our priority boosting
1041          * and task t's exiting its outermost RCU read-side critical
1042          * section.
1043          */
1044         t = container_of(tb, struct task_struct, rcu_node_entry);
1045         rt_mutex_init_proxy_locked(&rnp->boost_mtx, t);
1046         raw_spin_unlock_irqrestore(&rnp->lock, flags);
1047         /* Lock only for side effect: boosts task t's priority. */
1048         rt_mutex_lock(&rnp->boost_mtx);
1049         rt_mutex_unlock(&rnp->boost_mtx);  /* Then keep lockdep happy. */
1050 
1051         return ACCESS_ONCE(rnp->exp_tasks) != NULL ||
1052                ACCESS_ONCE(rnp->boost_tasks) != NULL;
1053 }
1054 
1055 /*
1056  * Priority-boosting kthread.  One per leaf rcu_node and one for the
1057  * root rcu_node.
1058  */
1059 static int rcu_boost_kthread(void *arg)
1060 {
1061         struct rcu_node *rnp = (struct rcu_node *)arg;
1062         int spincnt = 0;
1063         int more2boost;
1064 
1065         trace_rcu_utilization(TPS("Start boost kthread@init"));
1066         for (;;) {
1067                 rnp->boost_kthread_status = RCU_KTHREAD_WAITING;
1068                 trace_rcu_utilization(TPS("End boost kthread@rcu_wait"));
1069                 rcu_wait(rnp->boost_tasks || rnp->exp_tasks);
1070                 trace_rcu_utilization(TPS("Start boost kthread@rcu_wait"));
1071                 rnp->boost_kthread_status = RCU_KTHREAD_RUNNING;
1072                 more2boost = rcu_boost(rnp);
1073                 if (more2boost)
1074                         spincnt++;
1075                 else
1076                         spincnt = 0;
1077                 if (spincnt > 10) {
1078                         rnp->boost_kthread_status = RCU_KTHREAD_YIELDING;
1079                         trace_rcu_utilization(TPS("End boost kthread@rcu_yield"));
1080                         schedule_timeout_interruptible(2);
1081                         trace_rcu_utilization(TPS("Start boost kthread@rcu_yield"));
1082                         spincnt = 0;
1083                 }
1084         }
1085         /* NOTREACHED */
1086         trace_rcu_utilization(TPS("End boost kthread@notreached"));
1087         return 0;
1088 }
1089 
1090 /*
1091  * Check to see if it is time to start boosting RCU readers that are
1092  * blocking the current grace period, and, if so, tell the per-rcu_node
1093  * kthread to start boosting them.  If there is an expedited grace
1094  * period in progress, it is always time to boost.
1095  *
1096  * The caller must hold rnp->lock, which this function releases.
1097  * The ->boost_kthread_task is immortal, so we don't need to worry
1098  * about it going away.
1099  */
1100 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
1101         __releases(rnp->lock)
1102 {
1103         struct task_struct *t;
1104 
1105         if (!rcu_preempt_blocked_readers_cgp(rnp) && rnp->exp_tasks == NULL) {
1106                 rnp->n_balk_exp_gp_tasks++;
1107                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1108                 return;
1109         }
1110         if (rnp->exp_tasks != NULL ||
1111             (rnp->gp_tasks != NULL &&
1112              rnp->boost_tasks == NULL &&
1113              rnp->qsmask == 0 &&
1114              ULONG_CMP_GE(jiffies, rnp->boost_time))) {
1115                 if (rnp->exp_tasks == NULL)
1116                         rnp->boost_tasks = rnp->gp_tasks;
1117                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1118                 t = rnp->boost_kthread_task;
1119                 if (t)
1120                         rcu_wake_cond(t, rnp->boost_kthread_status);
1121         } else {
1122                 rcu_initiate_boost_trace(rnp);
1123                 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1124         }
1125 }
1126 
1127 /*
1128  * Wake up the per-CPU kthread to invoke RCU callbacks.
1129  */
1130 static void invoke_rcu_callbacks_kthread(void)
1131 {
1132         unsigned long flags;
1133 
1134         local_irq_save(flags);
1135         __this_cpu_write(rcu_cpu_has_work, 1);
1136         if (__this_cpu_read(rcu_cpu_kthread_task) != NULL &&
1137             current != __this_cpu_read(rcu_cpu_kthread_task)) {
1138                 rcu_wake_cond(__this_cpu_read(rcu_cpu_kthread_task),
1139                               __this_cpu_read(rcu_cpu_kthread_status));
1140         }
1141         local_irq_restore(flags);
1142 }
1143 
1144 /*
1145  * Is the current CPU running the RCU-callbacks kthread?
1146  * Caller must have preemption disabled.
1147  */
1148 static bool rcu_is_callbacks_kthread(void)
1149 {
1150         return __this_cpu_read(rcu_cpu_kthread_task) == current;
1151 }
1152 
1153 #define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)
1154 
1155 /*
1156  * Do priority-boost accounting for the start of a new grace period.
1157  */
1158 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1159 {
1160         rnp->boost_time = jiffies + RCU_BOOST_DELAY_JIFFIES;
1161 }
1162 
1163 /*
1164  * Create an RCU-boost kthread for the specified node if one does not
1165  * already exist.  We only create this kthread for preemptible RCU.
1166  * Returns zero if all is well, a negated errno otherwise.
1167  */
1168 static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
1169                                        struct rcu_node *rnp)
1170 {
1171         int rnp_index = rnp - &rsp->node[0];
1172         unsigned long flags;
1173         struct sched_param sp;
1174         struct task_struct *t;
1175 
1176         if (&rcu_preempt_state != rsp)
1177                 return 0;
1178 
1179         if (!rcu_scheduler_fully_active || rcu_rnp_online_cpus(rnp) == 0)
1180                 return 0;
1181 
1182         rsp->boost = 1;
1183         if (rnp->boost_kthread_task != NULL)
1184                 return 0;
1185         t = kthread_create(rcu_boost_kthread, (void *)rnp,
1186                            "rcub/%d", rnp_index);
1187         if (IS_ERR(t))
1188                 return PTR_ERR(t);
1189         raw_spin_lock_irqsave(&rnp->lock, flags);
1190         smp_mb__after_unlock_lock();
1191         rnp->boost_kthread_task = t;
1192         raw_spin_unlock_irqrestore(&rnp->lock, flags);
1193         sp.sched_priority = kthread_prio;
1194         sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
1195         wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */
1196         return 0;
1197 }
1198 
1199 static void rcu_kthread_do_work(void)
1200 {
1201         rcu_do_batch(&rcu_sched_state, this_cpu_ptr(&rcu_sched_data));
1202         rcu_do_batch(&rcu_bh_state, this_cpu_ptr(&rcu_bh_data));
1203         rcu_preempt_do_callbacks();
1204 }
1205 
1206 static void rcu_cpu_kthread_setup(unsigned int cpu)
1207 {
1208         struct sched_param sp;
1209 
1210         sp.sched_priority = kthread_prio;
1211         sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1212 }
1213 
1214 static void rcu_cpu_kthread_park(unsigned int cpu)
1215 {
1216         per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
1217 }
1218 
1219 static int rcu_cpu_kthread_should_run(unsigned int cpu)
1220 {
1221         return __this_cpu_read(rcu_cpu_has_work);
1222 }
1223 
1224 /*
1225  * Per-CPU kernel thread that invokes RCU callbacks.  This replaces the
1226  * RCU softirq used in flavors and configurations of RCU that do not
1227  * support RCU priority boosting.
1228  */
1229 static void rcu_cpu_kthread(unsigned int cpu)
1230 {
1231         unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status);
1232         char work, *workp = this_cpu_ptr(&rcu_cpu_has_work);
1233         int spincnt;
1234 
1235         for (spincnt = 0; spincnt < 10; spincnt++) {
1236                 trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait"));
1237                 local_bh_disable();
1238                 *statusp = RCU_KTHREAD_RUNNING;
1239                 this_cpu_inc(rcu_cpu_kthread_loops);
1240                 local_irq_disable();
1241                 work = *workp;
1242                 *workp = 0;
1243                 local_irq_enable();
1244                 if (work)
1245                         rcu_kthread_do_work();
1246                 local_bh_enable();
1247                 if (*workp == 0) {
1248                         trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
1249                         *statusp = RCU_KTHREAD_WAITING;
1250                         return;
1251                 }
1252         }
1253         *statusp = RCU_KTHREAD_YIELDING;
1254         trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
1255         schedule_timeout_interruptible(2);
1256         trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
1257         *statusp = RCU_KTHREAD_WAITING;
1258 }
1259 
1260 /*
1261  * Set the per-rcu_node kthread's affinity to cover all CPUs that are
1262  * served by the rcu_node in question.  The CPU hotplug lock is still
1263  * held, so the value of rnp->qsmaskinit will be stable.
1264  *
1265  * We don't include outgoingcpu in the affinity set, use -1 if there is
1266  * no outgoing CPU.  If there are no CPUs left in the affinity set,
1267  * this function allows the kthread to execute on any CPU.
1268  */
1269 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1270 {
1271         struct task_struct *t = rnp->boost_kthread_task;
1272         unsigned long mask = rcu_rnp_online_cpus(rnp);
1273         cpumask_var_t cm;
1274         int cpu;
1275 
1276         if (!t)
1277                 return;
1278         if (!zalloc_cpumask_var(&cm, GFP_KERNEL))
1279                 return;
1280         for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask >>= 1)
1281                 if ((mask & 0x1) && cpu != outgoingcpu)
1282                         cpumask_set_cpu(cpu, cm);
1283         if (cpumask_weight(cm) == 0)
1284                 cpumask_setall(cm);
1285         set_cpus_allowed_ptr(t, cm);
1286         free_cpumask_var(cm);
1287 }
1288 
1289 static struct smp_hotplug_thread rcu_cpu_thread_spec = {
1290         .store                  = &rcu_cpu_kthread_task,
1291         .thread_should_run      = rcu_cpu_kthread_should_run,
1292         .thread_fn              = rcu_cpu_kthread,
1293         .thread_comm            = "rcuc/%u",
1294         .setup                  = rcu_cpu_kthread_setup,
1295         .park                   = rcu_cpu_kthread_park,
1296 };
1297 
1298 /*
1299  * Spawn boost kthreads -- called as soon as the scheduler is running.
1300  */
1301 static void __init rcu_spawn_boost_kthreads(void)
1302 {
1303         struct rcu_node *rnp;
1304         int cpu;
1305 
1306         for_each_possible_cpu(cpu)
1307                 per_cpu(rcu_cpu_has_work, cpu) = 0;
1308         BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec));
1309         rcu_for_each_leaf_node(rcu_state_p, rnp)
1310                 (void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp);
1311 }
1312 
1313 static void rcu_prepare_kthreads(int cpu)
1314 {
1315         struct rcu_data *rdp = per_cpu_ptr(rcu_state_p->rda, cpu);
1316         struct rcu_node *rnp = rdp->mynode;
1317 
1318         /* Fire up the incoming CPU's kthread and leaf rcu_node kthread. */
1319         if (rcu_scheduler_fully_active)
1320                 (void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp);
1321 }
1322 
1323 #else /* #ifdef CONFIG_RCU_BOOST */
1324 
1325 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
1326         __releases(rnp->lock)
1327 {
1328         raw_spin_unlock_irqrestore(&rnp->lock, flags);
1329 }
1330 
1331 static void invoke_rcu_callbacks_kthread(void)
1332 {
1333         WARN_ON_ONCE(1);
1334 }
1335 
1336 static bool rcu_is_callbacks_kthread(void)
1337 {
1338         return false;
1339 }
1340 
1341 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1342 {
1343 }
1344 
1345 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1346 {
1347 }
1348 
1349 static void __init rcu_spawn_boost_kthreads(void)
1350 {
1351 }
1352 
1353 static void rcu_prepare_kthreads(int cpu)
1354 {
1355 }
1356 
1357 #endif /* #else #ifdef CONFIG_RCU_BOOST */
1358 
1359 #if !defined(CONFIG_RCU_FAST_NO_HZ)
1360 
1361 /*
1362  * Check to see if any future RCU-related work will need to be done
1363  * by the current CPU, even if none need be done immediately, returning
1364  * 1 if so.  This function is part of the RCU implementation; it is -not-
1365  * an exported member of the RCU API.
1366  *
1367  * Because we not have RCU_FAST_NO_HZ, just check whether this CPU needs
1368  * any flavor of RCU.
1369  */
1370 #ifndef CONFIG_RCU_NOCB_CPU_ALL
1371 int rcu_needs_cpu(unsigned long *delta_jiffies)
1372 {
1373         *delta_jiffies = ULONG_MAX;
1374         return rcu_cpu_has_callbacks(NULL);
1375 }
1376 #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
1377 
1378 /*
1379  * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
1380  * after it.
1381  */
1382 static void rcu_cleanup_after_idle(void)
1383 {
1384 }
1385 
1386 /*
1387  * Do the idle-entry grace-period work, which, because CONFIG_RCU_FAST_NO_HZ=n,
1388  * is nothing.
1389  */
1390 static void rcu_prepare_for_idle(void)
1391 {
1392 }
1393 
1394 /*
1395  * Don't bother keeping a running count of the number of RCU callbacks
1396  * posted because CONFIG_RCU_FAST_NO_HZ=n.
1397  */
1398 static void rcu_idle_count_callbacks_posted(void)
1399 {
1400 }
1401 
1402 #else /* #if !defined(CONFIG_RCU_FAST_NO_HZ) */
1403 
1404 /*
1405  * This code is invoked when a CPU goes idle, at which point we want
1406  * to have the CPU do everything required for RCU so that it can enter
1407  * the energy-efficient dyntick-idle mode.  This is handled by a
1408  * state machine implemented by rcu_prepare_for_idle() below.
1409  *
1410  * The following three proprocessor symbols control this state machine:
1411  *
1412  * RCU_IDLE_GP_DELAY gives the number of jiffies that a CPU is permitted
1413  *      to sleep in dyntick-idle mode with RCU callbacks pending.  This
1414  *      is sized to be roughly one RCU grace period.  Those energy-efficiency
1415  *      benchmarkers who might otherwise be tempted to set this to a large
1416  *      number, be warned: Setting RCU_IDLE_GP_DELAY too high can hang your
1417  *      system.  And if you are -that- concerned about energy efficiency,
1418  *      just power the system down and be done with it!
1419  * RCU_IDLE_LAZY_GP_DELAY gives the number of jiffies that a CPU is
1420  *      permitted to sleep in dyntick-idle mode with only lazy RCU
1421  *      callbacks pending.  Setting this too high can OOM your system.
1422  *
1423  * The values below work well in practice.  If future workloads require
1424  * adjustment, they can be converted into kernel config parameters, though
1425  * making the state machine smarter might be a better option.
1426  */
1427 #define RCU_IDLE_GP_DELAY 4             /* Roughly one grace period. */
1428 #define RCU_IDLE_LAZY_GP_DELAY (6 * HZ) /* Roughly six seconds. */
1429 
1430 static int rcu_idle_gp_delay = RCU_IDLE_GP_DELAY;
1431 module_param(rcu_idle_gp_delay, int, 0644);
1432 static int rcu_idle_lazy_gp_delay = RCU_IDLE_LAZY_GP_DELAY;
1433 module_param(rcu_idle_lazy_gp_delay, int, 0644);
1434 
1435 extern int tick_nohz_active;
1436 
1437 /*
1438  * Try to advance callbacks for all flavors of RCU on the current CPU, but
1439  * only if it has been awhile since the last time we did so.  Afterwards,
1440  * if there are any callbacks ready for immediate invocation, return true.
1441  */
1442 static bool __maybe_unused rcu_try_advance_all_cbs(void)
1443 {
1444         bool cbs_ready = false;
1445         struct rcu_data *rdp;
1446         struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1447         struct rcu_node *rnp;
1448         struct rcu_state *rsp;
1449 
1450         /* Exit early if we advanced recently. */
1451         if (jiffies == rdtp->last_advance_all)
1452                 return false;
1453         rdtp->last_advance_all = jiffies;
1454 
1455         for_each_rcu_flavor(rsp) {
1456                 rdp = this_cpu_ptr(rsp->rda);
1457                 rnp = rdp->mynode;
1458 
1459                 /*
1460                  * Don't bother checking unless a grace period has
1461                  * completed since we last checked and there are
1462                  * callbacks not yet ready to invoke.
1463                  */
1464                 if ((rdp->completed != rnp->completed ||
1465                      unlikely(ACCESS_ONCE(rdp->gpwrap))) &&
1466                     rdp->nxttail[RCU_DONE_TAIL] != rdp->nxttail[RCU_NEXT_TAIL])
1467                         note_gp_changes(rsp, rdp);
1468 
1469                 if (cpu_has_callbacks_ready_to_invoke(rdp))
1470                         cbs_ready = true;
1471         }
1472         return cbs_ready;
1473 }
1474 
1475 /*
1476  * Allow the CPU to enter dyntick-idle mode unless it has callbacks ready
1477  * to invoke.  If the CPU has callbacks, try to advance them.  Tell the
1478  * caller to set the timeout based on whether or not there are non-lazy
1479  * callbacks.
1480  *
1481  * The caller must have disabled interrupts.
1482  */
1483 #ifndef CONFIG_RCU_NOCB_CPU_ALL
1484 int rcu_needs_cpu(unsigned long *dj)
1485 {
1486         struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1487 
1488         /* Snapshot to detect later posting of non-lazy callback. */
1489         rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
1490 
1491         /* If no callbacks, RCU doesn't need the CPU. */
1492         if (!rcu_cpu_has_callbacks(&rdtp->all_lazy)) {
1493                 *dj = ULONG_MAX;
1494                 return 0;
1495         }
1496 
1497         /* Attempt to advance callbacks. */
1498         if (rcu_try_advance_all_cbs()) {
1499                 /* Some ready to invoke, so initiate later invocation. */
1500                 invoke_rcu_core();
1501                 return 1;
1502         }
1503         rdtp->last_accelerate = jiffies;
1504 
1505         /* Request timer delay depending on laziness, and round. */
1506         if (!rdtp->all_lazy) {
1507                 *dj = round_up(rcu_idle_gp_delay + jiffies,
1508                                rcu_idle_gp_delay) - jiffies;
1509         } else {
1510                 *dj = round_jiffies(rcu_idle_lazy_gp_delay + jiffies) - jiffies;
1511         }
1512         return 0;
1513 }
1514 #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
1515 
1516 /*
1517  * Prepare a CPU for idle from an RCU perspective.  The first major task
1518  * is to sense whether nohz mode has been enabled or disabled via sysfs.
1519  * The second major task is to check to see if a non-lazy callback has
1520  * arrived at a CPU that previously had only lazy callbacks.  The third
1521  * major task is to accelerate (that is, assign grace-period numbers to)
1522  * any recently arrived callbacks.
1523  *
1524  * The caller must have disabled interrupts.
1525  */
1526 static void rcu_prepare_for_idle(void)
1527 {
1528 #ifndef CONFIG_RCU_NOCB_CPU_ALL
1529         bool needwake;
1530         struct rcu_data *rdp;
1531         struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1532         struct rcu_node *rnp;
1533         struct rcu_state *rsp;
1534         int tne;
1535 
1536         /* Handle nohz enablement switches conservatively. */
1537         tne = ACCESS_ONCE(tick_nohz_active);
1538         if (tne != rdtp->tick_nohz_enabled_snap) {
1539                 if (rcu_cpu_has_callbacks(NULL))
1540                         invoke_rcu_core(); /* force nohz to see update. */
1541                 rdtp->tick_nohz_enabled_snap = tne;
1542                 return;
1543         }
1544         if (!tne)
1545                 return;
1546 
1547         /* If this is a no-CBs CPU, no callbacks, just return. */
1548         if (rcu_is_nocb_cpu(smp_processor_id()))
1549                 return;
1550 
1551         /*
1552          * If a non-lazy callback arrived at a CPU having only lazy
1553          * callbacks, invoke RCU core for the side-effect of recalculating
1554          * idle duration on re-entry to idle.
1555          */
1556         if (rdtp->all_lazy &&
1557             rdtp->nonlazy_posted != rdtp->nonlazy_posted_snap) {
1558                 rdtp->all_lazy = false;
1559                 rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
1560                 invoke_rcu_core();
1561                 return;
1562         }
1563 
1564         /*
1565          * If we have not yet accelerated this jiffy, accelerate all
1566          * callbacks on this CPU.
1567          */
1568         if (rdtp->last_accelerate == jiffies)
1569                 return;
1570         rdtp->last_accelerate = jiffies;
1571         for_each_rcu_flavor(rsp) {
1572                 rdp = this_cpu_ptr(rsp->rda);
1573                 if (!*rdp->nxttail[RCU_DONE_TAIL])
1574                         continue;
1575                 rnp = rdp->mynode;
1576                 raw_spin_lock(&rnp->lock); /* irqs already disabled. */
1577                 smp_mb__after_unlock_lock();
1578                 needwake = rcu_accelerate_cbs(rsp, rnp, rdp);
1579                 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
1580                 if (needwake)
1581                         rcu_gp_kthread_wake(rsp);
1582         }
1583 #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
1584 }
1585 
1586 /*
1587  * Clean up for exit from idle.  Attempt to advance callbacks based on
1588  * any grace periods that elapsed while the CPU was idle, and if any
1589  * callbacks are now ready to invoke, initiate invocation.
1590  */
1591 static void rcu_cleanup_after_idle(void)
1592 {
1593 #ifndef CONFIG_RCU_NOCB_CPU_ALL
1594         if (rcu_is_nocb_cpu(smp_processor_id()))
1595                 return;
1596         if (rcu_try_advance_all_cbs())
1597                 invoke_rcu_core();
1598 #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
1599 }
1600 
1601 /*
1602  * Keep a running count of the number of non-lazy callbacks posted
1603  * on this CPU.  This running counter (which is never decremented) allows
1604  * rcu_prepare_for_idle() to detect when something out of the idle loop
1605  * posts a callback, even if an equal number of callbacks are invoked.
1606  * Of course, callbacks should only be posted from within a trace event
1607  * designed to be called from idle or from within RCU_NONIDLE().
1608  */
1609 static void rcu_idle_count_callbacks_posted(void)
1610 {
1611         __this_cpu_add(rcu_dynticks.nonlazy_posted, 1);
1612 }
1613 
1614 /*
1615  * Data for flushing lazy RCU callbacks at OOM time.
1616  */
1617 static atomic_t oom_callback_count;
1618 static DECLARE_WAIT_QUEUE_HEAD(oom_callback_wq);
1619 
1620 /*
1621  * RCU OOM callback -- decrement the outstanding count and deliver the
1622  * wake-up if we are the last one.
1623  */
1624 static void rcu_oom_callback(struct rcu_head *rhp)
1625 {
1626         if (atomic_dec_and_test(&oom_callback_count))
1627                 wake_up(&oom_callback_wq);
1628 }
1629 
1630 /*
1631  * Post an rcu_oom_notify callback on the current CPU if it has at
1632  * least one lazy callback.  This will unnecessarily post callbacks
1633  * to CPUs that already have a non-lazy callback at the end of their
1634  * callback list, but this is an infrequent operation, so accept some
1635  * extra overhead to keep things simple.
1636  */
1637 static void rcu_oom_notify_cpu(void *unused)
1638 {
1639         struct rcu_state *rsp;
1640         struct rcu_data *rdp;
1641 
1642         for_each_rcu_flavor(rsp) {
1643                 rdp = raw_cpu_ptr(rsp->rda);
1644                 if (rdp->qlen_lazy != 0) {
1645                         atomic_inc(&oom_callback_count);
1646                         rsp->call(&rdp->oom_head, rcu_oom_callback);
1647                 }
1648         }
1649 }
1650 
1651 /*
1652  * If low on memory, ensure that each CPU has a non-lazy callback.
1653  * This will wake up CPUs that have only lazy callbacks, in turn
1654  * ensuring that they free up the corresponding memory in a timely manner.
1655  * Because an uncertain amount of memory will be freed in some uncertain
1656  * timeframe, we do not claim to have freed anything.
1657  */
1658 static int rcu_oom_notify(struct notifier_block *self,
1659                           unsigned long notused, void *nfreed)
1660 {
1661         int cpu;
1662 
1663         /* Wait for callbacks from earlier instance to complete. */
1664         wait_event(oom_callback_wq, atomic_read(&oom_callback_count) == 0);
1665         smp_mb(); /* Ensure callback reuse happens after callback invocation. */
1666 
1667         /*
1668          * Prevent premature wakeup: ensure that all increments happen
1669          * before there is a chance of the counter reaching zero.
1670          */
1671         atomic_set(&oom_callback_count, 1);
1672 
1673         get_online_cpus();
1674         for_each_online_cpu(cpu) {
1675                 smp_call_function_single(cpu, rcu_oom_notify_cpu, NULL, 1);
1676                 cond_resched_rcu_qs();
1677         }
1678         put_online_cpus();
1679 
1680         /* Unconditionally decrement: no need to wake ourselves up. */
1681         atomic_dec(&oom_callback_count);
1682 
1683         return NOTIFY_OK;
1684 }
1685 
1686 static struct notifier_block rcu_oom_nb = {
1687         .notifier_call = rcu_oom_notify
1688 };
1689 
1690 static int __init rcu_register_oom_notifier(void)
1691 {
1692         register_oom_notifier(&rcu_oom_nb);
1693         return 0;
1694 }
1695 early_initcall(rcu_register_oom_notifier);
1696 
1697 #endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */
1698 
1699 #ifdef CONFIG_RCU_CPU_STALL_INFO
1700 
1701 #ifdef CONFIG_RCU_FAST_NO_HZ
1702 
1703 static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
1704 {
1705         struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
1706         unsigned long nlpd = rdtp->nonlazy_posted - rdtp->nonlazy_posted_snap;
1707 
1708         sprintf(cp, "last_accelerate: %04lx/%04lx, nonlazy_posted: %ld, %c%c",
1709                 rdtp->last_accelerate & 0xffff, jiffies & 0xffff,
1710                 ulong2long(nlpd),
1711                 rdtp->all_lazy ? 'L' : '.',
1712                 rdtp->tick_nohz_enabled_snap ? '.' : 'D');
1713 }
1714 
1715 #else /* #ifdef CONFIG_RCU_FAST_NO_HZ */
1716 
1717 static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
1718 {
1719         *cp = '\0';
1720 }
1721 
1722 #endif /* #else #ifdef CONFIG_RCU_FAST_NO_HZ */
1723 
1724 /* Initiate the stall-info list. */
1725 static void print_cpu_stall_info_begin(void)
1726 {
1727         pr_cont("\n");
1728 }
1729 
1730 /*
1731  * Print out diagnostic information for the specified stalled CPU.
1732  *
1733  * If the specified CPU is aware of the current RCU grace period
1734  * (flavor specified by rsp), then print the number of scheduling
1735  * clock interrupts the CPU has taken during the time that it has
1736  * been aware.  Otherwise, print the number of RCU grace periods
1737  * that this CPU is ignorant of, for example, "1" if the CPU was
1738  * aware of the previous grace period.
1739  *
1740  * Also print out idle and (if CONFIG_RCU_FAST_NO_HZ) idle-entry info.
1741  */
1742 static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
1743 {
1744         char fast_no_hz[72];
1745         struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
1746         struct rcu_dynticks *rdtp = rdp->dynticks;
1747         char *ticks_title;
1748         unsigned long ticks_value;
1749 
1750         if (rsp->gpnum == rdp->gpnum) {
1751                 ticks_title = "ticks this GP";
1752                 ticks_value = rdp->ticks_this_gp;
1753         } else {
1754                 ticks_title = "GPs behind";
1755                 ticks_value = rsp->gpnum - rdp->gpnum;
1756         }
1757         print_cpu_stall_fast_no_hz(fast_no_hz, cpu);
1758         pr_err("\t%d: (%lu %s) idle=%03x/%llx/%d softirq=%u/%u fqs=%ld %s\n",
1759                cpu, ticks_value, ticks_title,
1760                atomic_read(&rdtp->dynticks) & 0xfff,
1761                rdtp->dynticks_nesting, rdtp->dynticks_nmi_nesting,
1762                rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
1763                ACCESS_ONCE(rsp->n_force_qs) - rsp->n_force_qs_gpstart,
1764                fast_no_hz);
1765 }
1766 
1767 /* Terminate the stall-info list. */
1768 static void print_cpu_stall_info_end(void)
1769 {
1770         pr_err("\t");
1771 }
1772 
1773 /* Zero ->ticks_this_gp for all flavors of RCU. */
1774 static void zero_cpu_stall_ticks(struct rcu_data *rdp)
1775 {
1776         rdp->ticks_this_gp = 0;
1777         rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id());
1778 }
1779 
1780 /* Increment ->ticks_this_gp for all flavors of RCU. */
1781 static void increment_cpu_stall_ticks(void)
1782 {
1783         struct rcu_state *rsp;
1784 
1785         for_each_rcu_flavor(rsp)
1786                 raw_cpu_inc(rsp->rda->ticks_this_gp);
1787 }
1788 
1789 #else /* #ifdef CONFIG_RCU_CPU_STALL_INFO */
1790 
1791 static void print_cpu_stall_info_begin(void)
1792 {
1793         pr_cont(" {");
1794 }
1795 
1796 static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
1797 {
1798         pr_cont(" %d", cpu);
1799 }
1800 
1801 static void print_cpu_stall_info_end(void)
1802 {
1803         pr_cont("} ");
1804 }
1805 
1806 static void zero_cpu_stall_ticks(struct rcu_data *rdp)
1807 {
1808 }
1809 
1810 static void increment_cpu_stall_ticks(void)
1811 {
1812 }
1813 
1814 #endif /* #else #ifdef CONFIG_RCU_CPU_STALL_INFO */
1815 
1816 #ifdef CONFIG_RCU_NOCB_CPU
1817 
1818 /*
1819  * Offload callback processing from the boot-time-specified set of CPUs
1820  * specified by rcu_nocb_mask.  For each CPU in the set, there is a
1821  * kthread created that pulls the callbacks from the corresponding CPU,
1822  * waits for a grace period to elapse, and invokes the callbacks.
1823  * The no-CBs CPUs do a wake_up() on their kthread when they insert
1824  * a callback into any empty list, unless the rcu_nocb_poll boot parameter
1825  * has been specified, in which case each kthread actively polls its
1826  * CPU.  (Which isn't so great for energy efficiency, but which does
1827  * reduce RCU's overhead on that CPU.)
1828  *
1829  * This is intended to be used in conjunction with Frederic Weisbecker's
1830  * adaptive-idle work, which would seriously reduce OS jitter on CPUs
1831  * running CPU-bound user-mode computations.
1832  *
1833  * Offloading of callback processing could also in theory be used as
1834  * an energy-efficiency measure because CPUs with no RCU callbacks
1835  * queued are more aggressive about entering dyntick-idle mode.
1836  */
1837 
1838 
1839 /* Parse the boot-time rcu_nocb_mask CPU list from the kernel parameters. */
1840 static int __init rcu_nocb_setup(char *str)
1841 {
1842         alloc_bootmem_cpumask_var(&rcu_nocb_mask);
1843         have_rcu_nocb_mask = true;
1844         cpulist_parse(str, rcu_nocb_mask);
1845         return 1;
1846 }
1847 __setup("rcu_nocbs=", rcu_nocb_setup);
1848 
1849 static int __init parse_rcu_nocb_poll(char *arg)
1850 {
1851         rcu_nocb_poll = 1;
1852         return 0;
1853 }
1854 early_param("rcu_nocb_poll", parse_rcu_nocb_poll);
1855 
1856 /*
1857  * Wake up any no-CBs CPUs' kthreads that were waiting on the just-ended
1858  * grace period.
1859  */
1860 static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)
1861 {
1862         wake_up_all(&rnp->nocb_gp_wq[rnp->completed & 0x1]);
1863 }
1864 
1865 /*
1866  * Set the root rcu_node structure's ->need_future_gp field
1867  * based on the sum of those of all rcu_node structures.  This does
1868  * double-count the root rcu_node structure's requests, but this
1869  * is necessary to handle the possibility of a rcu_nocb_kthread()
1870  * having awakened during the time that the rcu_node structures
1871  * were being updated for the end of the previous grace period.
1872  */
1873 static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)
1874 {
1875         rnp->need_future_gp[(rnp->completed + 1) & 0x1] += nrq;
1876 }
1877 
1878 static void rcu_init_one_nocb(struct rcu_node *rnp)
1879 {
1880         init_waitqueue_head(&rnp->nocb_gp_wq[0]);
1881         init_waitqueue_head(&rnp->nocb_gp_wq[1]);
1882 }
1883 
1884 #ifndef CONFIG_RCU_NOCB_CPU_ALL
1885 /* Is the specified CPU a no-CBs CPU? */
1886 bool rcu_is_nocb_cpu(int cpu)
1887 {
1888         if (have_rcu_nocb_mask)
1889                 return cpumask_test_cpu(cpu, rcu_nocb_mask);
1890         return false;
1891 }
1892 #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
1893 
1894 /*
1895  * Kick the leader kthread for this NOCB group.
1896  */
1897 static void wake_nocb_leader(struct rcu_data *rdp, bool force)
1898 {
1899         struct rcu_data *rdp_leader = rdp->nocb_leader;
1900 
1901         if (!ACCESS_ONCE(rdp_leader->nocb_kthread))
1902                 return;
1903         if (ACCESS_ONCE(rdp_leader->nocb_leader_sleep) || force) {
1904                 /* Prior smp_mb__after_atomic() orders against prior enqueue. */
1905                 ACCESS_ONCE(rdp_leader->nocb_leader_sleep) = false;
1906                 wake_up(&rdp_leader->nocb_wq);
1907         }
1908 }
1909 
1910 /*
1911  * Does the specified CPU need an RCU callback for the specified flavor
1912  * of rcu_barrier()?
1913  */
1914 static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)
1915 {
1916         struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
1917         unsigned long ret;
1918 #ifdef CONFIG_PROVE_RCU
1919         struct rcu_head *rhp;
1920 #endif /* #ifdef CONFIG_PROVE_RCU */
1921 
1922         /*
1923          * Check count of all no-CBs callbacks awaiting invocation.
1924          * There needs to be a barrier before this function is called,
1925          * but associated with a prior determination that no more
1926          * callbacks would be posted.  In the worst case, the first
1927          * barrier in _rcu_barrier() suffices (but the caller cannot
1928          * necessarily rely on this, not a substitute for the caller
1929          * getting the concurrency design right!).  There must also be
1930          * a barrier between the following load an posting of a callback
1931          * (if a callback is in fact needed).  This is associated with an
1932          * atomic_inc() in the caller.
1933          */
1934         ret = atomic_long_read(&rdp->nocb_q_count);
1935 
1936 #ifdef CONFIG_PROVE_RCU
1937         rhp = ACCESS_ONCE(rdp->nocb_head);
1938         if (!rhp)
1939                 rhp = ACCESS_ONCE(rdp->nocb_gp_head);
1940         if (!rhp)
1941                 rhp = ACCESS_ONCE(rdp->nocb_follower_head);
1942 
1943         /* Having no rcuo kthread but CBs after scheduler starts is bad! */
1944         if (!ACCESS_ONCE(rdp->nocb_kthread) && rhp &&
1945             rcu_scheduler_fully_active) {
1946                 /* RCU callback enqueued before CPU first came online??? */
1947                 pr_err("RCU: Never-onlined no-CBs CPU %d has CB %p\n",
1948                        cpu, rhp->func);
1949                 WARN_ON_ONCE(1);
1950         }
1951 #endif /* #ifdef CONFIG_PROVE_RCU */
1952 
1953         return !!ret;
1954 }
1955 
1956 /*
1957  * Enqueue the specified string of rcu_head structures onto the specified
1958  * CPU's no-CBs lists.  The CPU is specified by rdp, the head of the
1959  * string by rhp, and the tail of the string by rhtp.  The non-lazy/lazy
1960  * counts are supplied by rhcount and rhcount_lazy.
1961  *
1962  * If warranted, also wake up the kthread servicing this CPUs queues.
1963  */
1964 static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,
1965                                     struct rcu_head *rhp,
1966                                     struct rcu_head **rhtp,
1967                                     int rhcount, int rhcount_lazy,
1968                                     unsigned long flags)
1969 {
1970         int len;
1971         struct rcu_head **old_rhpp;
1972         struct task_struct *t;
1973 
1974         /* Enqueue the callback on the nocb list and update counts. */
1975         atomic_long_add(rhcount, &rdp->nocb_q_count);
1976         /* rcu_barrier() relies on ->nocb_q_count add before xchg. */
1977         old_rhpp = xchg(&rdp->nocb_tail, rhtp);
1978         ACCESS_ONCE(*old_rhpp) = rhp;
1979         atomic_long_add(rhcount_lazy, &rdp->nocb_q_count_lazy);
1980         smp_mb__after_atomic(); /* Store *old_rhpp before _wake test. */
1981 
1982         /* If we are not being polled and there is a kthread, awaken it ... */
1983         t = ACCESS_ONCE(rdp->nocb_kthread);
1984         if (rcu_nocb_poll || !t) {
1985                 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1986                                     TPS("WakeNotPoll"));
1987                 return;
1988         }
1989         len = atomic_long_read(&rdp->nocb_q_count);
1990         if (old_rhpp == &rdp->nocb_head) {
1991                 if (!irqs_disabled_flags(flags)) {
1992                         /* ... if queue was empty ... */
1993                         wake_nocb_leader(rdp, false);
1994                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1995                                             TPS("WakeEmpty"));
1996                 } else {
1997                         rdp->nocb_defer_wakeup = RCU_NOGP_WAKE;
1998                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
1999                                             TPS("WakeEmptyIsDeferred"));
2000                 }
2001                 rdp->qlen_last_fqs_check = 0;
2002         } else if (len > rdp->qlen_last_fqs_check + qhimark) {
2003                 /* ... or if many callbacks queued. */
2004                 if (!irqs_disabled_flags(flags)) {
2005                         wake_nocb_leader(rdp, true);
2006                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2007                                             TPS("WakeOvf"));
2008                 } else {
2009                         rdp->nocb_defer_wakeup = RCU_NOGP_WAKE_FORCE;
2010                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2011                                             TPS("WakeOvfIsDeferred"));
2012                 }
2013                 rdp->qlen_last_fqs_check = LONG_MAX / 2;
2014         } else {
2015                 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WakeNot"));
2016         }
2017         return;
2018 }
2019 
2020 /*
2021  * This is a helper for __call_rcu(), which invokes this when the normal
2022  * callback queue is inoperable.  If this is not a no-CBs CPU, this
2023  * function returns failure back to __call_rcu(), which can complain
2024  * appropriately.
2025  *
2026  * Otherwise, this function queues the callback where the corresponding
2027  * "rcuo" kthread can find it.
2028  */
2029 static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
2030                             bool lazy, unsigned long flags)
2031 {
2032 
2033         if (!rcu_is_nocb_cpu(rdp->cpu))
2034                 return false;
2035         __call_rcu_nocb_enqueue(rdp, rhp, &rhp->next, 1, lazy, flags);
2036         if (__is_kfree_rcu_offset((unsigned long)rhp->func))
2037                 trace_rcu_kfree_callback(rdp->rsp->name, rhp,
2038                                          (unsigned long)rhp->func,
2039                                          -atomic_long_read(&rdp->nocb_q_count_lazy),
2040                                          -atomic_long_read(&rdp->nocb_q_count));
2041         else
2042                 trace_rcu_callback(rdp->rsp->name, rhp,
2043                                    -atomic_long_read(&rdp->nocb_q_count_lazy),
2044                                    -atomic_long_read(&rdp->nocb_q_count));
2045 
2046         /*
2047          * If called from an extended quiescent state with interrupts
2048          * disabled, invoke the RCU core in order to allow the idle-entry
2049          * deferred-wakeup check to function.
2050          */
2051         if (irqs_disabled_flags(flags) &&
2052             !rcu_is_watching() &&
2053             cpu_online(smp_processor_id()))
2054                 invoke_rcu_core();
2055 
2056         return true;
2057 }
2058 
2059 /*
2060  * Adopt orphaned callbacks on a no-CBs CPU, or return 0 if this is
2061  * not a no-CBs CPU.
2062  */
2063 static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
2064                                                      struct rcu_data *rdp,
2065                                                      unsigned long flags)
2066 {
2067         long ql = rsp->qlen;
2068         long qll = rsp->qlen_lazy;
2069 
2070         /* If this is not a no-CBs CPU, tell the caller to do it the old way. */
2071         if (!rcu_is_nocb_cpu(smp_processor_id()))
2072                 return false;
2073         rsp->qlen = 0;
2074         rsp->qlen_lazy = 0;
2075 
2076         /* First, enqueue the donelist, if any.  This preserves CB ordering. */
2077         if (rsp->orphan_donelist != NULL) {
2078                 __call_rcu_nocb_enqueue(rdp, rsp->orphan_donelist,
2079                                         rsp->orphan_donetail, ql, qll, flags);
2080                 ql = qll = 0;
2081                 rsp->orphan_donelist = NULL;
2082                 rsp->orphan_donetail = &rsp->orphan_donelist;
2083         }
2084         if (rsp->orphan_nxtlist != NULL) {
2085                 __call_rcu_nocb_enqueue(rdp, rsp->orphan_nxtlist,
2086                                         rsp->orphan_nxttail, ql, qll, flags);
2087                 ql = qll = 0;
2088                 rsp->orphan_nxtlist = NULL;
2089                 rsp->orphan_nxttail = &rsp->orphan_nxtlist;
2090         }
2091         return true;
2092 }
2093 
2094 /*
2095  * If necessary, kick off a new grace period, and either way wait
2096  * for a subsequent grace period to complete.
2097  */
2098 static void rcu_nocb_wait_gp(struct rcu_data *rdp)
2099 {
2100         unsigned long c;
2101         bool d;
2102         unsigned long flags;
2103         bool needwake;
2104         struct rcu_node *rnp = rdp->mynode;
2105 
2106         raw_spin_lock_irqsave(&rnp->lock, flags);
2107         smp_mb__after_unlock_lock();
2108         needwake = rcu_start_future_gp(rnp, rdp, &c);
2109         raw_spin_unlock_irqrestore(&rnp->lock, flags);
2110         if (needwake)
2111                 rcu_gp_kthread_wake(rdp->rsp);
2112 
2113         /*
2114          * Wait for the grace period.  Do so interruptibly to avoid messing
2115          * up the load average.
2116          */
2117         trace_rcu_future_gp(rnp, rdp, c, TPS("StartWait"));
2118         for (;;) {
2119                 wait_event_interruptible(
2120                         rnp->nocb_gp_wq[c & 0x1],
2121                         (d = ULONG_CMP_GE(ACCESS_ONCE(rnp->completed), c)));
2122                 if (likely(d))
2123                         break;
2124                 WARN_ON(signal_pending(current));
2125                 trace_rcu_future_gp(rnp, rdp, c, TPS("ResumeWait"));
2126         }
2127         trace_rcu_future_gp(rnp, rdp, c, TPS("EndWait"));
2128         smp_mb(); /* Ensure that CB invocation happens after GP end. */
2129 }
2130 
2131 /*
2132  * Leaders come here to wait for additional callbacks to show up.
2133  * This function does not return until callbacks appear.
2134  */
2135 static void nocb_leader_wait(struct rcu_data *my_rdp)
2136 {
2137         bool firsttime = true;
2138         bool gotcbs;
2139         struct rcu_data *rdp;
2140         struct rcu_head **tail;
2141 
2142 wait_again:
2143 
2144         /* Wait for callbacks to appear. */
2145         if (!rcu_nocb_poll) {
2146                 trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep");
2147                 wait_event_interruptible(my_rdp->nocb_wq,
2148                                 !ACCESS_ONCE(my_rdp->nocb_leader_sleep));
2149                 /* Memory barrier handled by smp_mb() calls below and repoll. */
2150         } else if (firsttime) {
2151                 firsttime = false; /* Don't drown trace log with "Poll"! */
2152                 trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Poll");
2153         }
2154 
2155         /*
2156          * Each pass through the following loop checks a follower for CBs.
2157          * We are our own first follower.  Any CBs found are moved to
2158          * nocb_gp_head, where they await a grace period.
2159          */
2160         gotcbs = false;
2161         for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {
2162                 rdp->nocb_gp_head = ACCESS_ONCE(rdp->nocb_head);
2163                 if (!rdp->nocb_gp_head)
2164                         continue;  /* No CBs here, try next follower. */
2165 
2166                 /* Move callbacks to wait-for-GP list, which is empty. */
2167                 ACCESS_ONCE(rdp->nocb_head) = NULL;
2168                 rdp->nocb_gp_tail = xchg(&rdp->nocb_tail, &rdp->nocb_head);
2169                 gotcbs = true;
2170         }
2171 
2172         /*
2173          * If there were no callbacks, sleep a bit, rescan after a
2174          * memory barrier, and go retry.
2175          */
2176         if (unlikely(!gotcbs)) {
2177                 if (!rcu_nocb_poll)
2178                         trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu,
2179                                             "WokeEmpty");
2180                 WARN_ON(signal_pending(current));
2181                 schedule_timeout_interruptible(1);
2182 
2183                 /* Rescan in case we were a victim of memory ordering. */
2184                 my_rdp->nocb_leader_sleep = true;
2185                 smp_mb();  /* Ensure _sleep true before scan. */
2186                 for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower)
2187                         if (ACCESS_ONCE(rdp->nocb_head)) {
2188                                 /* Found CB, so short-circuit next wait. */
2189                                 my_rdp->nocb_leader_sleep = false;
2190                                 break;
2191                         }
2192                 goto wait_again;
2193         }
2194 
2195         /* Wait for one grace period. */
2196         rcu_nocb_wait_gp(my_rdp);
2197 
2198         /*
2199          * We left ->nocb_leader_sleep unset to reduce cache thrashing.
2200          * We set it now, but recheck for new callbacks while
2201          * traversing our follower list.
2202          */
2203         my_rdp->nocb_leader_sleep = true;
2204         smp_mb(); /* Ensure _sleep true before scan of ->nocb_head. */
2205 
2206         /* Each pass through the following loop wakes a follower, if needed. */
2207         for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {
2208                 if (ACCESS_ONCE(rdp->nocb_head))
2209                         my_rdp->nocb_leader_sleep = false;/* No need to sleep.*/
2210                 if (!rdp->nocb_gp_head)
2211                         continue; /* No CBs, so no need to wake follower. */
2212 
2213                 /* Append callbacks to follower's "done" list. */
2214                 tail = xchg(&rdp->nocb_follower_tail, rdp->nocb_gp_tail);
2215                 *tail = rdp->nocb_gp_head;
2216                 smp_mb__after_atomic(); /* Store *tail before wakeup. */
2217                 if (rdp != my_rdp && tail == &rdp->nocb_follower_head) {
2218                         /*
2219                          * List was empty, wake up the follower.
2220                          * Memory barriers supplied by atomic_long_add().
2221                          */
2222                         wake_up(&rdp->nocb_wq);
2223                 }
2224         }
2225 
2226         /* If we (the leader) don't have CBs, go wait some more. */
2227         if (!my_rdp->nocb_follower_head)
2228                 goto wait_again;
2229 }
2230 
2231 /*
2232  * Followers come here to wait for additional callbacks to show up.
2233  * This function does not return until callbacks appear.
2234  */
2235 static void nocb_follower_wait(struct rcu_data *rdp)
2236 {
2237         bool firsttime = true;
2238 
2239         for (;;) {
2240                 if (!rcu_nocb_poll) {
2241                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2242                                             "FollowerSleep");
2243                         wait_event_interruptible(rdp->nocb_wq,
2244                                                  ACCESS_ONCE(rdp->nocb_follower_head));
2245                 } else if (firsttime) {
2246                         /* Don't drown trace log with "Poll"! */
2247                         firsttime = false;
2248                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "Poll");
2249                 }
2250                 if (smp_load_acquire(&rdp->nocb_follower_head)) {
2251                         /* ^^^ Ensure CB invocation follows _head test. */
2252                         return;
2253                 }
2254                 if (!rcu_nocb_poll)
2255                         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2256                                             "WokeEmpty");
2257                 WARN_ON(signal_pending(current));
2258                 schedule_timeout_interruptible(1);
2259         }
2260 }
2261 
2262 /*
2263  * Per-rcu_data kthread, but only for no-CBs CPUs.  Each kthread invokes
2264  * callbacks queued by the corresponding no-CBs CPU, however, there is
2265  * an optional leader-follower relationship so that the grace-period
2266  * kthreads don't have to do quite so many wakeups.
2267  */
2268 static int rcu_nocb_kthread(void *arg)
2269 {
2270         int c, cl;
2271         struct rcu_head *list;
2272         struct rcu_head *next;
2273         struct rcu_head **tail;
2274         struct rcu_data *rdp = arg;
2275 
2276         /* Each pass through this loop invokes one batch of callbacks */
2277         for (;;) {
2278                 /* Wait for callbacks. */
2279                 if (rdp->nocb_leader == rdp)
2280                         nocb_leader_wait(rdp);
2281                 else
2282                         nocb_follower_wait(rdp);
2283 
2284                 /* Pull the ready-to-invoke callbacks onto local list. */
2285                 list = ACCESS_ONCE(rdp->nocb_follower_head);
2286                 BUG_ON(!list);
2287                 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "WokeNonEmpty");
2288                 ACCESS_ONCE(rdp->nocb_follower_head) = NULL;
2289                 tail = xchg(&rdp->nocb_follower_tail, &rdp->nocb_follower_head);
2290 
2291                 /* Each pass through the following loop invokes a callback. */
2292                 trace_rcu_batch_start(rdp->rsp->name,
2293                                       atomic_long_read(&rdp->nocb_q_count_lazy),
2294                                       atomic_long_read(&rdp->nocb_q_count), -1);
2295                 c = cl = 0;
2296                 while (list) {
2297                         next = list->next;
2298                         /* Wait for enqueuing to complete, if needed. */
2299                         while (next == NULL && &list->next != tail) {
2300                                 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2301                                                     TPS("WaitQueue"));
2302                                 schedule_timeout_interruptible(1);
2303                                 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
2304                                                     TPS("WokeQueue"));
2305                                 next = list->next;
2306                         }
2307                         debug_rcu_head_unqueue(list);
2308                         local_bh_disable();
2309                         if (__rcu_reclaim(rdp->rsp->name, list))
2310                                 cl++;
2311                         c++;
2312                         local_bh_enable();
2313                         list = next;
2314                 }
2315                 trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1);
2316                 smp_mb__before_atomic();  /* _add after CB invocation. */
2317                 atomic_long_add(-c, &rdp->nocb_q_count);
2318                 atomic_long_add(-cl, &rdp->nocb_q_count_lazy);
2319                 rdp->n_nocbs_invoked += c;
2320         }
2321         return 0;
2322 }
2323 
2324 /* Is a deferred wakeup of rcu_nocb_kthread() required? */
2325 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
2326 {
2327         return ACCESS_ONCE(rdp->nocb_defer_wakeup);
2328 }
2329 
2330 /* Do a deferred wakeup of rcu_nocb_kthread(). */
2331 static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
2332 {
2333         int ndw;
2334 
2335         if (!rcu_nocb_need_deferred_wakeup(rdp))
2336                 return;
2337         ndw = ACCESS_ONCE(rdp->nocb_defer_wakeup);
2338         ACCESS_ONCE(rdp->nocb_defer_wakeup) = RCU_NOGP_WAKE_NOT;
2339         wake_nocb_leader(rdp, ndw == RCU_NOGP_WAKE_FORCE);
2340         trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("DeferredWake"));
2341 }
2342 
2343 void __init rcu_init_nohz(void)
2344 {
2345         int cpu;
2346         bool need_rcu_nocb_mask = true;
2347         struct rcu_state *rsp;
2348 
2349 #ifdef CONFIG_RCU_NOCB_CPU_NONE
2350         need_rcu_nocb_mask = false;
2351 #endif /* #ifndef CONFIG_RCU_NOCB_CPU_NONE */
2352 
2353 #if defined(CONFIG_NO_HZ_FULL)
2354         if (tick_nohz_full_running && cpumask_weight(tick_nohz_full_mask))
2355                 need_rcu_nocb_mask = true;
2356 #endif /* #if defined(CONFIG_NO_HZ_FULL) */
2357 
2358         if (!have_rcu_nocb_mask && need_rcu_nocb_mask) {
2359                 if (!zalloc_cpumask_var(&rcu_nocb_mask, GFP_KERNEL)) {
2360                         pr_info("rcu_nocb_mask allocation failed, callback offloading disabled.\n");
2361                         return;
2362                 }
2363                 have_rcu_nocb_mask = true;
2364         }
2365         if (!have_rcu_nocb_mask)
2366                 return;
2367 
2368 #ifdef CONFIG_RCU_NOCB_CPU_ZERO
2369         pr_info("\tOffload RCU callbacks from CPU 0\n");
2370         cpumask_set_cpu(0, rcu_nocb_mask);
2371 #endif /* #ifdef CONFIG_RCU_NOCB_CPU_ZERO */
2372 #ifdef CONFIG_RCU_NOCB_CPU_ALL
2373         pr_info("\tOffload RCU callbacks from all CPUs\n");
2374         cpumask_copy(rcu_nocb_mask, cpu_possible_mask);
2375 #endif /* #ifdef CONFIG_RCU_NOCB_CPU_ALL */
2376 #if defined(CONFIG_NO_HZ_FULL)
2377         if (tick_nohz_full_running)
2378                 cpumask_or(rcu_nocb_mask, rcu_nocb_mask, tick_nohz_full_mask);
2379 #endif /* #if defined(CONFIG_NO_HZ_FULL) */
2380 
2381         if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) {
2382                 pr_info("\tNote: kernel parameter 'rcu_nocbs=' contains nonexistent CPUs.\n");
2383                 cpumask_and(rcu_nocb_mask, cpu_possible_mask,
2384                             rcu_nocb_mask);
2385         }
2386         pr_info("\tOffload RCU callbacks from CPUs: %*pbl.\n",
2387                 cpumask_pr_args(rcu_nocb_mask));
2388         if (rcu_nocb_poll)
2389                 pr_info("\tPoll for callbacks from no-CBs CPUs.\n");
2390 
2391         for_each_rcu_flavor(rsp) {
2392                 for_each_cpu(cpu, rcu_nocb_mask)
2393                         init_nocb_callback_list(per_cpu_ptr(rsp->rda, cpu));
2394                 rcu_organize_nocb_kthreads(rsp);
2395         }
2396 }
2397 
2398 /* Initialize per-rcu_data variables for no-CBs CPUs. */
2399 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
2400 {
2401         rdp->nocb_tail = &rdp->nocb_head;
2402         init_waitqueue_head(&rdp->nocb_wq);
2403         rdp->nocb_follower_tail = &rdp->nocb_follower_head;
2404 }
2405 
2406 /*
2407  * If the specified CPU is a no-CBs CPU that does not already have its
2408  * rcuo kthread for the specified RCU flavor, spawn it.  If the CPUs are
2409  * brought online out of order, this can require re-organizing the
2410  * leader-follower relationships.
2411  */
2412 static void rcu_spawn_one_nocb_kthread(struct rcu_state *rsp, int cpu)
2413 {
2414         struct rcu_data *rdp;
2415         struct rcu_data *rdp_last;
2416         struct rcu_data *rdp_old_leader;
2417         struct rcu_data *rdp_spawn = per_cpu_ptr(rsp->rda, cpu);
2418         struct task_struct *t;
2419 
2420         /*
2421          * If this isn't a no-CBs CPU or if it already has an rcuo kthread,
2422          * then nothing to do.
2423          */
2424         if (!rcu_is_nocb_cpu(cpu) || rdp_spawn->nocb_kthread)
2425                 return;
2426 
2427         /* If we didn't spawn the leader first, reorganize! */
2428         rdp_old_leader = rdp_spawn->nocb_leader;
2429         if (rdp_old_leader != rdp_spawn && !rdp_old_leader->nocb_kthread) {
2430                 rdp_last = NULL;
2431                 rdp = rdp_old_leader;
2432                 do {
2433                         rdp->nocb_leader = rdp_spawn;
2434                         if (rdp_last && rdp != rdp_spawn)
2435                                 rdp_last->nocb_next_follower = rdp;
2436                         if (rdp == rdp_spawn) {
2437                                 rdp = rdp->nocb_next_follower;
2438                         } else {
2439                                 rdp_last = rdp;
2440                                 rdp = rdp->nocb_next_follower;
2441                                 rdp_last->nocb_next_follower = NULL;
2442                         }
2443                 } while (rdp);
2444                 rdp_spawn->nocb_next_follower = rdp_old_leader;
2445         }
2446 
2447         /* Spawn the kthread for this CPU and RCU flavor. */
2448         t = kthread_run(rcu_nocb_kthread, rdp_spawn,
2449                         "rcuo%c/%d", rsp->abbr, cpu);
2450         BUG_ON(IS_ERR(t));
2451         ACCESS_ONCE(rdp_spawn->nocb_kthread) = t;
2452 }
2453 
2454 /*
2455  * If the specified CPU is a no-CBs CPU that does not already have its
2456  * rcuo kthreads, spawn them.
2457  */
2458 static void rcu_spawn_all_nocb_kthreads(int cpu)
2459 {
2460         struct rcu_state *rsp;
2461 
2462         if (rcu_scheduler_fully_active)
2463                 for_each_rcu_flavor(rsp)
2464                         rcu_spawn_one_nocb_kthread(rsp, cpu);
2465 }
2466 
2467 /*
2468  * Once the scheduler is running, spawn rcuo kthreads for all online
2469  * no-CBs CPUs.  This assumes that the early_initcall()s happen before
2470  * non-boot CPUs come online -- if this changes, we will need to add
2471  * some mutual exclusion.
2472  */
2473 static void __init rcu_spawn_nocb_kthreads(void)
2474 {
2475         int cpu;
2476 
2477         for_each_online_cpu(cpu)
2478                 rcu_spawn_all_nocb_kthreads(cpu);
2479 }
2480 
2481 /* How many follower CPU IDs per leader?  Default of -1 for sqrt(nr_cpu_ids). */
2482 static int rcu_nocb_leader_stride = -1;
2483 module_param(rcu_nocb_leader_stride, int, 0444);
2484 
2485 /*
2486  * Initialize leader-follower relationships for all no-CBs CPU.
2487  */
2488 static void __init rcu_organize_nocb_kthreads(struct rcu_state *rsp)
2489 {
2490         int cpu;
2491         int ls = rcu_nocb_leader_stride;
2492         int nl = 0;  /* Next leader. */
2493         struct rcu_data *rdp;
2494         struct rcu_data *rdp_leader = NULL;  /* Suppress misguided gcc warn. */
2495         struct rcu_data *rdp_prev = NULL;
2496 
2497         if (!have_rcu_nocb_mask)
2498                 return;
2499         if (ls == -1) {
2500                 ls = int_sqrt(nr_cpu_ids);
2501                 rcu_nocb_leader_stride = ls;
2502         }
2503 
2504         /*
2505          * Each pass through this loop sets up one rcu_data structure and
2506          * spawns one rcu_nocb_kthread().
2507          */
2508         for_each_cpu(cpu, rcu_nocb_mask) {
2509                 rdp = per_cpu_ptr(rsp->rda, cpu);
2510                 if (rdp->cpu >= nl) {
2511                         /* New leader, set up for followers & next leader. */
2512                         nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls;
2513                         rdp->nocb_leader = rdp;
2514                         rdp_leader = rdp;
2515                 } else {
2516                         /* Another follower, link to previous leader. */
2517                         rdp->nocb_leader = rdp_leader;
2518                         rdp_prev->nocb_next_follower = rdp;
2519                 }
2520                 rdp_prev = rdp;
2521         }
2522 }
2523 
2524 /* Prevent __call_rcu() from enqueuing callbacks on no-CBs CPUs */
2525 static bool init_nocb_callback_list(struct rcu_data *rdp)
2526 {
2527         if (!rcu_is_nocb_cpu(rdp->cpu))
2528                 return false;
2529 
2530         /* If there are early-boot callbacks, move them to nocb lists. */
2531         if (rdp->nxtlist) {
2532                 rdp->nocb_head = rdp->nxtlist;
2533                 rdp->nocb_tail = rdp->nxttail[RCU_NEXT_TAIL];
2534                 atomic_long_set(&rdp->nocb_q_count, rdp->qlen);
2535                 atomic_long_set(&rdp->nocb_q_count_lazy, rdp->qlen_lazy);
2536                 rdp->nxtlist = NULL;
2537                 rdp->qlen = 0;
2538                 rdp->qlen_lazy = 0;
2539         }
2540         rdp->nxttail[RCU_NEXT_TAIL] = NULL;
2541         return true;
2542 }
2543 
2544 #else /* #ifdef CONFIG_RCU_NOCB_CPU */
2545 
2546 static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)
2547 {
2548         WARN_ON_ONCE(1); /* Should be dead code. */
2549         return false;
2550 }
2551 
2552 static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)
2553 {
2554 }
2555 
2556 static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)
2557 {
2558 }
2559 
2560 static void rcu_init_one_nocb(struct rcu_node *rnp)
2561 {
2562 }
2563 
2564 static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
2565                             bool lazy, unsigned long flags)
2566 {
2567         return false;
2568 }
2569 
2570 static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
2571                                                      struct rcu_data *rdp,
2572                                                      unsigned long flags)
2573 {
2574         return false;
2575 }
2576 
2577 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
2578 {
2579 }
2580 
2581 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
2582 {
2583         return false;
2584 }
2585 
2586 static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
2587 {
2588 }
2589 
2590 static void rcu_spawn_all_nocb_kthreads(int cpu)
2591 {
2592 }
2593 
2594 static void __init rcu_spawn_nocb_kthreads(void)
2595 {
2596 }
2597 
2598 static bool init_nocb_callback_list(struct rcu_data *rdp)
2599 {
2600         return false;
2601 }
2602 
2603 #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
2604 
2605 /*
2606  * An adaptive-ticks CPU can potentially execute in kernel mode for an
2607  * arbitrarily long period of time with the scheduling-clock tick turned
2608  * off.  RCU will be paying attention to this CPU because it is in the
2609  * kernel, but the CPU cannot be guaranteed to be executing the RCU state
2610  * machine because the scheduling-clock tick has been disabled.  Therefore,
2611  * if an adaptive-ticks CPU is failing to respond to the current grace
2612  * period and has not be idle from an RCU perspective, kick it.
2613  */
2614 static void __maybe_unused rcu_kick_nohz_cpu(int cpu)
2615 {
2616 #ifdef CONFIG_NO_HZ_FULL
2617         if (tick_nohz_full_cpu(cpu))
2618                 smp_send_reschedule(cpu);
2619 #endif /* #ifdef CONFIG_NO_HZ_FULL */
2620 }
2621 
2622 
2623 #ifdef CONFIG_NO_HZ_FULL_SYSIDLE
2624 
2625 static int full_sysidle_state;          /* Current system-idle state. */
2626 #define RCU_SYSIDLE_NOT         0       /* Some CPU is not idle. */
2627 #define RCU_SYSIDLE_SHORT       1       /* All CPUs idle for brief period. */
2628 #define RCU_SYSIDLE_LONG        2       /* All CPUs idle for long enough. */
2629 #define RCU_SYSIDLE_FULL        3       /* All CPUs idle, ready for sysidle. */
2630 #define RCU_SYSIDLE_FULL_NOTED  4       /* Actually entered sysidle state. */
2631 
2632 /*
2633  * Invoked to note exit from irq or task transition to idle.  Note that
2634  * usermode execution does -not- count as idle here!  After all, we want
2635  * to detect full-system idle states, not RCU quiescent states and grace
2636  * periods.  The caller must have disabled interrupts.
2637  */
2638 static void rcu_sysidle_enter(int irq)
2639 {
2640         unsigned long j;
2641         struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
2642 
2643         /* If there are no nohz_full= CPUs, no need to track this. */
2644         if (!tick_nohz_full_enabled())
2645                 return;
2646 
2647         /* Adjust nesting, check for fully idle. */
2648         if (irq) {
2649                 rdtp->dynticks_idle_nesting--;
2650                 WARN_ON_ONCE(rdtp->dynticks_idle_nesting < 0);
2651                 if (rdtp->dynticks_idle_nesting != 0)
2652                         return;  /* Still not fully idle. */
2653         } else {
2654                 if ((rdtp->dynticks_idle_nesting & DYNTICK_TASK_NEST_MASK) ==
2655                     DYNTICK_TASK_NEST_VALUE) {
2656                         rdtp->dynticks_idle_nesting = 0;
2657                 } else {
2658                         rdtp->dynticks_idle_nesting -= DYNTICK_TASK_NEST_VALUE;
2659                         WARN_ON_ONCE(rdtp->dynticks_idle_nesting < 0);
2660                         return;  /* Still not fully idle. */
2661                 }
2662         }
2663 
2664         /* Record start of fully idle period. */
2665         j = jiffies;
2666         ACCESS_ONCE(rdtp->dynticks_idle_jiffies) = j;
2667         smp_mb__before_atomic();
2668         atomic_inc(&rdtp->dynticks_idle);
2669         smp_mb__after_atomic();
2670         WARN_ON_ONCE(atomic_read(&rdtp->dynticks_idle) & 0x1);
2671 }
2672 
2673 /*
2674  * Unconditionally force exit from full system-idle state.  This is
2675  * invoked when a normal CPU exits idle, but must be called separately
2676  * for the timekeeping CPU (tick_do_timer_cpu).  The reason for this
2677  * is that the timekeeping CPU is permitted to take scheduling-clock
2678  * interrupts while the system is in system-idle state, and of course
2679  * rcu_sysidle_exit() has no way of distinguishing a scheduling-clock
2680  * interrupt from any other type of interrupt.
2681  */
2682 void rcu_sysidle_force_exit(void)
2683 {
2684         int oldstate = ACCESS_ONCE(full_sysidle_state);
2685         int newoldstate;
2686 
2687         /*
2688          * Each pass through the following loop attempts to exit full
2689          * system-idle state.  If contention proves to be a problem,
2690          * a trylock-based contention tree could be used here.
2691          */
2692         while (oldstate > RCU_SYSIDLE_SHORT) {
2693                 newoldstate = cmpxchg(&full_sysidle_state,
2694                                       oldstate, RCU_SYSIDLE_NOT);
2695                 if (oldstate == newoldstate &&
2696                     oldstate == RCU_SYSIDLE_FULL_NOTED) {
2697                         rcu_kick_nohz_cpu(tick_do_timer_cpu);
2698                         return; /* We cleared it, done! */
2699                 }
2700                 oldstate = newoldstate;
2701         }
2702         smp_mb(); /* Order initial oldstate fetch vs. later non-idle work. */
2703 }
2704 
2705 /*
2706  * Invoked to note entry to irq or task transition from idle.  Note that
2707  * usermode execution does -not- count as idle here!  The caller must
2708  * have disabled interrupts.
2709  */
2710 static void rcu_sysidle_exit(int irq)
2711 {
2712         struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
2713 
2714         /* If there are no nohz_full= CPUs, no need to track this. */
2715         if (!tick_nohz_full_enabled())
2716                 return;
2717 
2718         /* Adjust nesting, check for already non-idle. */
2719         if (irq) {
2720                 rdtp->dynticks_idle_nesting++;
2721                 WARN_ON_ONCE(rdtp->dynticks_idle_nesting <= 0);
2722                 if (rdtp->dynticks_idle_nesting != 1)
2723                         return; /* Already non-idle. */
2724         } else {
2725                 /*
2726                  * Allow for irq misnesting.  Yes, it really is possible
2727                  * to enter an irq handler then never leave it, and maybe
2728                  * also vice versa.  Handle both possibilities.
2729                  */
2730                 if (rdtp->dynticks_idle_nesting & DYNTICK_TASK_NEST_MASK) {
2731                         rdtp->dynticks_idle_nesting += DYNTICK_TASK_NEST_VALUE;
2732                         WARN_ON_ONCE(rdtp->dynticks_idle_nesting <= 0);
2733                         return; /* Already non-idle. */
2734                 } else {
2735                         rdtp->dynticks_idle_nesting = DYNTICK_TASK_EXIT_IDLE;
2736                 }
2737         }
2738 
2739         /* Record end of idle period. */
2740         smp_mb__before_atomic();
2741         atomic_inc(&rdtp->dynticks_idle);
2742         smp_mb__after_atomic();
2743         WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks_idle) & 0x1));
2744 
2745         /*
2746          * If we are the timekeeping CPU, we are permitted to be non-idle
2747          * during a system-idle state.  This must be the case, because
2748          * the timekeeping CPU has to take scheduling-clock interrupts
2749          * during the time that the system is transitioning to full
2750          * system-idle state.  This means that the timekeeping CPU must
2751          * invoke rcu_sysidle_force_exit() directly if it does anything
2752          * more than take a scheduling-clock interrupt.
2753          */
2754         if (smp_processor_id() == tick_do_timer_cpu)
2755                 return;
2756 
2757         /* Update system-idle state: We are clearly no longer fully idle! */
2758         rcu_sysidle_force_exit();
2759 }
2760 
2761 /*
2762  * Check to see if the current CPU is idle.  Note that usermode execution
2763  * does not count as idle.  The caller must have disabled interrupts,
2764  * and must be running on tick_do_timer_cpu.
2765  */
2766 static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
2767                                   unsigned long *maxj)
2768 {
2769         int cur;
2770         unsigned long j;
2771         struct rcu_dynticks *rdtp = rdp->dynticks;
2772 
2773         /* If there are no nohz_full= CPUs, don't check system-wide idleness. */
2774         if (!tick_nohz_full_enabled())
2775                 return;
2776 
2777         /*
2778          * If some other CPU has already reported non-idle, if this is
2779          * not the flavor of RCU that tracks sysidle state, or if this
2780          * is an offline or the timekeeping CPU, nothing to do.
2781          */
2782         if (!*isidle || rdp->rsp != rcu_state_p ||
2783             cpu_is_offline(rdp->cpu) || rdp->cpu == tick_do_timer_cpu)
2784                 return;
2785         /* Verify affinity of current kthread. */
2786         WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu);
2787 
2788         /* Pick up current idle and NMI-nesting counter and check. */
2789         cur = atomic_read(&rdtp->dynticks_idle);
2790         if (cur & 0x1) {
2791                 *isidle = false; /* We are not idle! */
2792                 return;
2793         }
2794         smp_mb(); /* Read counters before timestamps. */
2795 
2796         /* Pick up timestamps. */
2797         j = ACCESS_ONCE(rdtp->dynticks_idle_jiffies);
2798         /* If this CPU entered idle more recently, update maxj timestamp. */
2799         if (ULONG_CMP_LT(*maxj, j))
2800                 *maxj = j;
2801 }
2802 
2803 /*
2804  * Is this the flavor of RCU that is handling full-system idle?
2805  */
2806 static bool is_sysidle_rcu_state(struct rcu_state *rsp)
2807 {
2808         return rsp == rcu_state_p;
2809 }
2810 
2811 /*
2812  * Return a delay in jiffies based on the number of CPUs, rcu_node
2813  * leaf fanout, and jiffies tick rate.  The idea is to allow larger
2814  * systems more time to transition to full-idle state in order to
2815  * avoid the cache thrashing that otherwise occur on the state variable.
2816  * Really small systems (less than a couple of tens of CPUs) should
2817  * instead use a single global atomically incremented counter, and later
2818  * versions of this will automatically reconfigure themselves accordingly.
2819  */
2820 static unsigned long rcu_sysidle_delay(void)
2821 {
2822         if (nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL)
2823                 return 0;
2824         return DIV_ROUND_UP(nr_cpu_ids * HZ, rcu_fanout_leaf * 1000);
2825 }
2826 
2827 /*
2828  * Advance the full-system-idle state.  This is invoked when all of
2829  * the non-timekeeping CPUs are idle.
2830  */
2831 static void rcu_sysidle(unsigned long j)
2832 {
2833         /* Check the current state. */
2834         switch (ACCESS_ONCE(full_sysidle_state)) {
2835         case RCU_SYSIDLE_NOT:
2836 
2837                 /* First time all are idle, so note a short idle period. */
2838                 ACCESS_ONCE(full_sysidle_state) = RCU_SYSIDLE_SHORT;
2839                 break;
2840 
2841         case RCU_SYSIDLE_SHORT:
2842 
2843                 /*
2844                  * Idle for a bit, time to advance to next state?
2845                  * cmpxchg failure means race with non-idle, let them win.
2846                  */
2847                 if (ULONG_CMP_GE(jiffies, j + rcu_sysidle_delay()))
2848                         (void)cmpxchg(&full_sysidle_state,
2849                                       RCU_SYSIDLE_SHORT, RCU_SYSIDLE_LONG);
2850                 break;
2851 
2852         case RCU_SYSIDLE_LONG:
2853 
2854                 /*
2855                  * Do an additional check pass before advancing to full.
2856                  * cmpxchg failure means race with non-idle, let them win.
2857                  */
2858                 if (ULONG_CMP_GE(jiffies, j + rcu_sysidle_delay()))
2859                         (void)cmpxchg(&full_sysidle_state,
2860                                       RCU_SYSIDLE_LONG, RCU_SYSIDLE_FULL);
2861                 break;
2862 
2863         default:
2864                 break;
2865         }
2866 }
2867 
2868 /*
2869  * Found a non-idle non-timekeeping CPU, so kick the system-idle state
2870  * back to the beginning.
2871  */
2872 static void rcu_sysidle_cancel(void)
2873 {
2874         smp_mb();
2875         if (full_sysidle_state > RCU_SYSIDLE_SHORT)
2876                 ACCESS_ONCE(full_sysidle_state) = RCU_SYSIDLE_NOT;
2877 }
2878 
2879 /*
2880  * Update the sysidle state based on the results of a force-quiescent-state
2881  * scan of the CPUs' dyntick-idle state.
2882  */
2883 static void rcu_sysidle_report(struct rcu_state *rsp, int isidle,
2884                                unsigned long maxj, bool gpkt)
2885 {
2886         if (rsp != rcu_state_p)
2887                 return;  /* Wrong flavor, ignore. */
2888         if (gpkt && nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL)
2889                 return;  /* Running state machine from timekeeping CPU. */
2890         if (isidle)
2891                 rcu_sysidle(maxj);    /* More idle! */
2892         else
2893                 rcu_sysidle_cancel(); /* Idle is over. */
2894 }
2895 
2896 /*
2897  * Wrapper for rcu_sysidle_report() when called from the grace-period
2898  * kthread's context.
2899  */
2900 static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
2901                                   unsigned long maxj)
2902 {
2903         /* If there are no nohz_full= CPUs, no need to track this. */
2904         if (!tick_nohz_full_enabled())
2905                 return;
2906 
2907         rcu_sysidle_report(rsp, isidle, maxj, true);
2908 }
2909 
2910 /* Callback and function for forcing an RCU grace period. */
2911 struct rcu_sysidle_head {
2912         struct rcu_head rh;
2913         int inuse;
2914 };
2915 
2916 static void rcu_sysidle_cb(struct rcu_head *rhp)
2917 {
2918         struct rcu_sysidle_head *rshp;
2919 
2920         /*
2921          * The following memory barrier is needed to replace the
2922          * memory barriers that would normally be in the memory
2923          * allocator.
2924          */
2925         smp_mb();  /* grace period precedes setting inuse. */
2926 
2927         rshp = container_of(rhp, struct rcu_sysidle_head, rh);
2928         ACCESS_ONCE(rshp->inuse) = 0;
2929 }
2930 
2931 /*
2932  * Check to see if the system is fully idle, other than the timekeeping CPU.
2933  * The caller must have disabled interrupts.  This is not intended to be
2934  * called unless tick_nohz_full_enabled().
2935  */
2936 bool rcu_sys_is_idle(void)
2937 {
2938         static struct rcu_sysidle_head rsh;
2939         int rss = ACCESS_ONCE(full_sysidle_state);
2940 
2941         if (WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu))
2942                 return false;
2943 
2944         /* Handle small-system case by doing a full scan of CPUs. */
2945         if (nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL) {
2946                 int oldrss = rss - 1;
2947 
2948                 /*
2949                  * One pass to advance to each state up to _FULL.
2950                  * Give up if any pass fails to advance the state.
2951                  */
2952                 while (rss < RCU_SYSIDLE_FULL && oldrss < rss) {
2953                         int cpu;
2954                         bool isidle = true;
2955                         unsigned long maxj = jiffies - ULONG_MAX / 4;
2956                         struct rcu_data *rdp;
2957 
2958                         /* Scan all the CPUs looking for nonidle CPUs. */
2959                         for_each_possible_cpu(cpu) {
2960                                 rdp = per_cpu_ptr(rcu_state_p->rda, cpu);
2961                                 rcu_sysidle_check_cpu(rdp, &isidle, &maxj);
2962                                 if (!isidle)
2963                                         break;
2964                         }
2965                         rcu_sysidle_report(rcu_state_p, isidle, maxj, false);
2966                         oldrss = rss;
2967                         rss = ACCESS_ONCE(full_sysidle_state);
2968                 }
2969         }
2970 
2971         /* If this is the first observation of an idle period, record it. */
2972         if (rss == RCU_SYSIDLE_FULL) {
2973                 rss = cmpxchg(&full_sysidle_state,
2974                               RCU_SYSIDLE_FULL, RCU_SYSIDLE_FULL_NOTED);
2975                 return rss == RCU_SYSIDLE_FULL;
2976         }
2977 
2978         smp_mb(); /* ensure rss load happens before later caller actions. */
2979 
2980         /* If already fully idle, tell the caller (in case of races). */
2981         if (rss == RCU_SYSIDLE_FULL_NOTED)
2982                 return true;
2983 
2984         /*
2985          * If we aren't there yet, and a grace period is not in flight,
2986          * initiate a grace period.  Either way, tell the caller that
2987          * we are not there yet.  We use an xchg() rather than an assignment
2988          * to make up for the memory barriers that would otherwise be
2989          * provided by the memory allocator.
2990          */
2991         if (nr_cpu_ids > CONFIG_NO_HZ_FULL_SYSIDLE_SMALL &&
2992             !rcu_gp_in_progress(rcu_state_p) &&
2993             !rsh.inuse && xchg(&rsh.inuse, 1) == 0)
2994                 call_rcu(&rsh.rh, rcu_sysidle_cb);
2995         return false;
2996 }
2997 
2998 /*
2999  * Initialize dynticks sysidle state for CPUs coming online.
3000  */
3001 static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp)
3002 {
3003         rdtp->dynticks_idle_nesting = DYNTICK_TASK_NEST_VALUE;
3004 }
3005 
3006 #else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
3007 
3008 static void rcu_sysidle_enter(int irq)
3009 {
3010 }
3011 
3012 static void rcu_sysidle_exit(int irq)
3013 {
3014 }
3015 
3016 static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
3017                                   unsigned long *maxj)
3018 {
3019 }
3020 
3021 static bool is_sysidle_rcu_state(struct rcu_state *rsp)
3022 {
3023         return false;
3024 }
3025 
3026 static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
3027                                   unsigned long maxj)
3028 {
3029 }
3030 
3031 static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp)
3032 {
3033 }
3034 
3035 #endif /* #else #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
3036 
3037 /*
3038  * Is this CPU a NO_HZ_FULL CPU that should ignore RCU so that the
3039  * grace-period kthread will do force_quiescent_state() processing?
3040  * The idea is to avoid waking up RCU core processing on such a
3041  * CPU unless the grace period has extended for too long.
3042  *
3043  * This code relies on the fact that all NO_HZ_FULL CPUs are also
3044  * CONFIG_RCU_NOCB_CPU CPUs.
3045  */
3046 static bool rcu_nohz_full_cpu(struct rcu_state *rsp)
3047 {
3048 #ifdef CONFIG_NO_HZ_FULL
3049         if (tick_nohz_full_cpu(smp_processor_id()) &&
3050             (!rcu_gp_in_progress(rsp) ||
3051              ULONG_CMP_LT(jiffies, ACCESS_ONCE(rsp->gp_start) + HZ)))
3052                 return 1;
3053 #endif /* #ifdef CONFIG_NO_HZ_FULL */
3054         return 0;
3055 }
3056 
3057 /*
3058  * Bind the grace-period kthread for the sysidle flavor of RCU to the
3059  * timekeeping CPU.
3060  */
3061 static void rcu_bind_gp_kthread(void)
3062 {
3063         int __maybe_unused cpu;
3064 
3065         if (!tick_nohz_full_enabled())
3066                 return;
3067 #ifdef CONFIG_NO_HZ_FULL_SYSIDLE
3068         cpu = tick_do_timer_cpu;
3069         if (cpu >= 0 && cpu < nr_cpu_ids)
3070                 set_cpus_allowed_ptr(current, cpumask_of(cpu));
3071 #else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
3072         housekeeping_affine(current);
3073 #endif /* #else #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
3074 }
3075 
3076 /* Record the current task on dyntick-idle entry. */
3077 static void rcu_dynticks_task_enter(void)
3078 {
3079 #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
3080         ACCESS_ONCE(current->rcu_tasks_idle_cpu) = smp_processor_id();
3081 #endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */
3082 }
3083 
3084 /* Record no current task on dyntick-idle exit. */
3085 static void rcu_dynticks_task_exit(void)
3086 {
3087 #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
3088         ACCESS_ONCE(current->rcu_tasks_idle_cpu) = -1;
3089 #endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */
3090 }
3091 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp