~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/sched/core.c

Version: ~ [ linux-5.16-rc3 ] ~ [ linux-5.15.5 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.82 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.162 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.218 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.256 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.291 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.293 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-only
  2 /*
  3  *  kernel/sched/core.c
  4  *
  5  *  Core kernel scheduler code and related syscalls
  6  *
  7  *  Copyright (C) 1991-2002  Linus Torvalds
  8  */
  9 #define CREATE_TRACE_POINTS
 10 #include <trace/events/sched.h>
 11 #undef CREATE_TRACE_POINTS
 12 
 13 #include "sched.h"
 14 
 15 #include <linux/nospec.h>
 16 
 17 #include <linux/kcov.h>
 18 #include <linux/scs.h>
 19 
 20 #include <asm/switch_to.h>
 21 #include <asm/tlb.h>
 22 
 23 #include "../workqueue_internal.h"
 24 #include "../../fs/io-wq.h"
 25 #include "../smpboot.h"
 26 
 27 #include "pelt.h"
 28 #include "smp.h"
 29 
 30 /*
 31  * Export tracepoints that act as a bare tracehook (ie: have no trace event
 32  * associated with them) to allow external modules to probe them.
 33  */
 34 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_cfs_tp);
 35 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_rt_tp);
 36 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_dl_tp);
 37 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_irq_tp);
 38 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_se_tp);
 39 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_cpu_capacity_tp);
 40 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_overutilized_tp);
 41 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_cfs_tp);
 42 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_se_tp);
 43 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_update_nr_running_tp);
 44 
 45 DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
 46 
 47 #ifdef CONFIG_SCHED_DEBUG
 48 /*
 49  * Debugging: various feature bits
 50  *
 51  * If SCHED_DEBUG is disabled, each compilation unit has its own copy of
 52  * sysctl_sched_features, defined in sched.h, to allow constants propagation
 53  * at compile time and compiler optimization based on features default.
 54  */
 55 #define SCHED_FEAT(name, enabled)       \
 56         (1UL << __SCHED_FEAT_##name) * enabled |
 57 const_debug unsigned int sysctl_sched_features =
 58 #include "features.h"
 59         0;
 60 #undef SCHED_FEAT
 61 #endif
 62 
 63 /*
 64  * Number of tasks to iterate in a single balance run.
 65  * Limited because this is done with IRQs disabled.
 66  */
 67 const_debug unsigned int sysctl_sched_nr_migrate = 32;
 68 
 69 /*
 70  * period over which we measure -rt task CPU usage in us.
 71  * default: 1s
 72  */
 73 unsigned int sysctl_sched_rt_period = 1000000;
 74 
 75 __read_mostly int scheduler_running;
 76 
 77 /*
 78  * part of the period that we allow rt tasks to run in us.
 79  * default: 0.95s
 80  */
 81 int sysctl_sched_rt_runtime = 950000;
 82 
 83 
 84 /*
 85  * Serialization rules:
 86  *
 87  * Lock order:
 88  *
 89  *   p->pi_lock
 90  *     rq->lock
 91  *       hrtimer_cpu_base->lock (hrtimer_start() for bandwidth controls)
 92  *
 93  *  rq1->lock
 94  *    rq2->lock  where: rq1 < rq2
 95  *
 96  * Regular state:
 97  *
 98  * Normal scheduling state is serialized by rq->lock. __schedule() takes the
 99  * local CPU's rq->lock, it optionally removes the task from the runqueue and
100  * always looks at the local rq data structures to find the most eligible task
101  * to run next.
102  *
103  * Task enqueue is also under rq->lock, possibly taken from another CPU.
104  * Wakeups from another LLC domain might use an IPI to transfer the enqueue to
105  * the local CPU to avoid bouncing the runqueue state around [ see
106  * ttwu_queue_wakelist() ]
107  *
108  * Task wakeup, specifically wakeups that involve migration, are horribly
109  * complicated to avoid having to take two rq->locks.
110  *
111  * Special state:
112  *
113  * System-calls and anything external will use task_rq_lock() which acquires
114  * both p->pi_lock and rq->lock. As a consequence the state they change is
115  * stable while holding either lock:
116  *
117  *  - sched_setaffinity()/
118  *    set_cpus_allowed_ptr():   p->cpus_ptr, p->nr_cpus_allowed
119  *  - set_user_nice():          p->se.load, p->*prio
120  *  - __sched_setscheduler():   p->sched_class, p->policy, p->*prio,
121  *                              p->se.load, p->rt_priority,
122  *                              p->dl.dl_{runtime, deadline, period, flags, bw, density}
123  *  - sched_setnuma():          p->numa_preferred_nid
124  *  - sched_move_task()/
125  *    cpu_cgroup_fork():        p->sched_task_group
126  *  - uclamp_update_active()    p->uclamp*
127  *
128  * p->state <- TASK_*:
129  *
130  *   is changed locklessly using set_current_state(), __set_current_state() or
131  *   set_special_state(), see their respective comments, or by
132  *   try_to_wake_up(). This latter uses p->pi_lock to serialize against
133  *   concurrent self.
134  *
135  * p->on_rq <- { 0, 1 = TASK_ON_RQ_QUEUED, 2 = TASK_ON_RQ_MIGRATING }:
136  *
137  *   is set by activate_task() and cleared by deactivate_task(), under
138  *   rq->lock. Non-zero indicates the task is runnable, the special
139  *   ON_RQ_MIGRATING state is used for migration without holding both
140  *   rq->locks. It indicates task_cpu() is not stable, see task_rq_lock().
141  *
142  * p->on_cpu <- { 0, 1 }:
143  *
144  *   is set by prepare_task() and cleared by finish_task() such that it will be
145  *   set before p is scheduled-in and cleared after p is scheduled-out, both
146  *   under rq->lock. Non-zero indicates the task is running on its CPU.
147  *
148  *   [ The astute reader will observe that it is possible for two tasks on one
149  *     CPU to have ->on_cpu = 1 at the same time. ]
150  *
151  * task_cpu(p): is changed by set_task_cpu(), the rules are:
152  *
153  *  - Don't call set_task_cpu() on a blocked task:
154  *
155  *    We don't care what CPU we're not running on, this simplifies hotplug,
156  *    the CPU assignment of blocked tasks isn't required to be valid.
157  *
158  *  - for try_to_wake_up(), called under p->pi_lock:
159  *
160  *    This allows try_to_wake_up() to only take one rq->lock, see its comment.
161  *
162  *  - for migration called under rq->lock:
163  *    [ see task_on_rq_migrating() in task_rq_lock() ]
164  *
165  *    o move_queued_task()
166  *    o detach_task()
167  *
168  *  - for migration called under double_rq_lock():
169  *
170  *    o __migrate_swap_task()
171  *    o push_rt_task() / pull_rt_task()
172  *    o push_dl_task() / pull_dl_task()
173  *    o dl_task_offline_migration()
174  *
175  */
176 
177 /*
178  * __task_rq_lock - lock the rq @p resides on.
179  */
180 struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf)
181         __acquires(rq->lock)
182 {
183         struct rq *rq;
184 
185         lockdep_assert_held(&p->pi_lock);
186 
187         for (;;) {
188                 rq = task_rq(p);
189                 raw_spin_lock(&rq->lock);
190                 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
191                         rq_pin_lock(rq, rf);
192                         return rq;
193                 }
194                 raw_spin_unlock(&rq->lock);
195 
196                 while (unlikely(task_on_rq_migrating(p)))
197                         cpu_relax();
198         }
199 }
200 
201 /*
202  * task_rq_lock - lock p->pi_lock and lock the rq @p resides on.
203  */
204 struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf)
205         __acquires(p->pi_lock)
206         __acquires(rq->lock)
207 {
208         struct rq *rq;
209 
210         for (;;) {
211                 raw_spin_lock_irqsave(&p->pi_lock, rf->flags);
212                 rq = task_rq(p);
213                 raw_spin_lock(&rq->lock);
214                 /*
215                  *      move_queued_task()              task_rq_lock()
216                  *
217                  *      ACQUIRE (rq->lock)
218                  *      [S] ->on_rq = MIGRATING         [L] rq = task_rq()
219                  *      WMB (__set_task_cpu())          ACQUIRE (rq->lock);
220                  *      [S] ->cpu = new_cpu             [L] task_rq()
221                  *                                      [L] ->on_rq
222                  *      RELEASE (rq->lock)
223                  *
224                  * If we observe the old CPU in task_rq_lock(), the acquire of
225                  * the old rq->lock will fully serialize against the stores.
226                  *
227                  * If we observe the new CPU in task_rq_lock(), the address
228                  * dependency headed by '[L] rq = task_rq()' and the acquire
229                  * will pair with the WMB to ensure we then also see migrating.
230                  */
231                 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
232                         rq_pin_lock(rq, rf);
233                         return rq;
234                 }
235                 raw_spin_unlock(&rq->lock);
236                 raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);
237 
238                 while (unlikely(task_on_rq_migrating(p)))
239                         cpu_relax();
240         }
241 }
242 
243 /*
244  * RQ-clock updating methods:
245  */
246 
247 static void update_rq_clock_task(struct rq *rq, s64 delta)
248 {
249 /*
250  * In theory, the compile should just see 0 here, and optimize out the call
251  * to sched_rt_avg_update. But I don't trust it...
252  */
253         s64 __maybe_unused steal = 0, irq_delta = 0;
254 
255 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
256         irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;
257 
258         /*
259          * Since irq_time is only updated on {soft,}irq_exit, we might run into
260          * this case when a previous update_rq_clock() happened inside a
261          * {soft,}irq region.
262          *
263          * When this happens, we stop ->clock_task and only update the
264          * prev_irq_time stamp to account for the part that fit, so that a next
265          * update will consume the rest. This ensures ->clock_task is
266          * monotonic.
267          *
268          * It does however cause some slight miss-attribution of {soft,}irq
269          * time, a more accurate solution would be to update the irq_time using
270          * the current rq->clock timestamp, except that would require using
271          * atomic ops.
272          */
273         if (irq_delta > delta)
274                 irq_delta = delta;
275 
276         rq->prev_irq_time += irq_delta;
277         delta -= irq_delta;
278 #endif
279 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
280         if (static_key_false((&paravirt_steal_rq_enabled))) {
281                 steal = paravirt_steal_clock(cpu_of(rq));
282                 steal -= rq->prev_steal_time_rq;
283 
284                 if (unlikely(steal > delta))
285                         steal = delta;
286 
287                 rq->prev_steal_time_rq += steal;
288                 delta -= steal;
289         }
290 #endif
291 
292         rq->clock_task += delta;
293 
294 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ
295         if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY))
296                 update_irq_load_avg(rq, irq_delta + steal);
297 #endif
298         update_rq_clock_pelt(rq, delta);
299 }
300 
301 void update_rq_clock(struct rq *rq)
302 {
303         s64 delta;
304 
305         lockdep_assert_held(&rq->lock);
306 
307         if (rq->clock_update_flags & RQCF_ACT_SKIP)
308                 return;
309 
310 #ifdef CONFIG_SCHED_DEBUG
311         if (sched_feat(WARN_DOUBLE_CLOCK))
312                 SCHED_WARN_ON(rq->clock_update_flags & RQCF_UPDATED);
313         rq->clock_update_flags |= RQCF_UPDATED;
314 #endif
315 
316         delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
317         if (delta < 0)
318                 return;
319         rq->clock += delta;
320         update_rq_clock_task(rq, delta);
321 }
322 
323 #ifdef CONFIG_SCHED_HRTICK
324 /*
325  * Use HR-timers to deliver accurate preemption points.
326  */
327 
328 static void hrtick_clear(struct rq *rq)
329 {
330         if (hrtimer_active(&rq->hrtick_timer))
331                 hrtimer_cancel(&rq->hrtick_timer);
332 }
333 
334 /*
335  * High-resolution timer tick.
336  * Runs from hardirq context with interrupts disabled.
337  */
338 static enum hrtimer_restart hrtick(struct hrtimer *timer)
339 {
340         struct rq *rq = container_of(timer, struct rq, hrtick_timer);
341         struct rq_flags rf;
342 
343         WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
344 
345         rq_lock(rq, &rf);
346         update_rq_clock(rq);
347         rq->curr->sched_class->task_tick(rq, rq->curr, 1);
348         rq_unlock(rq, &rf);
349 
350         return HRTIMER_NORESTART;
351 }
352 
353 #ifdef CONFIG_SMP
354 
355 static void __hrtick_restart(struct rq *rq)
356 {
357         struct hrtimer *timer = &rq->hrtick_timer;
358         ktime_t time = rq->hrtick_time;
359 
360         hrtimer_start(timer, time, HRTIMER_MODE_ABS_PINNED_HARD);
361 }
362 
363 /*
364  * called from hardirq (IPI) context
365  */
366 static void __hrtick_start(void *arg)
367 {
368         struct rq *rq = arg;
369         struct rq_flags rf;
370 
371         rq_lock(rq, &rf);
372         __hrtick_restart(rq);
373         rq_unlock(rq, &rf);
374 }
375 
376 /*
377  * Called to set the hrtick timer state.
378  *
379  * called with rq->lock held and irqs disabled
380  */
381 void hrtick_start(struct rq *rq, u64 delay)
382 {
383         struct hrtimer *timer = &rq->hrtick_timer;
384         s64 delta;
385 
386         /*
387          * Don't schedule slices shorter than 10000ns, that just
388          * doesn't make sense and can cause timer DoS.
389          */
390         delta = max_t(s64, delay, 10000LL);
391         rq->hrtick_time = ktime_add_ns(timer->base->get_time(), delta);
392 
393         if (rq == this_rq())
394                 __hrtick_restart(rq);
395         else
396                 smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd);
397 }
398 
399 #else
400 /*
401  * Called to set the hrtick timer state.
402  *
403  * called with rq->lock held and irqs disabled
404  */
405 void hrtick_start(struct rq *rq, u64 delay)
406 {
407         /*
408          * Don't schedule slices shorter than 10000ns, that just
409          * doesn't make sense. Rely on vruntime for fairness.
410          */
411         delay = max_t(u64, delay, 10000LL);
412         hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay),
413                       HRTIMER_MODE_REL_PINNED_HARD);
414 }
415 
416 #endif /* CONFIG_SMP */
417 
418 static void hrtick_rq_init(struct rq *rq)
419 {
420 #ifdef CONFIG_SMP
421         INIT_CSD(&rq->hrtick_csd, __hrtick_start, rq);
422 #endif
423         hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
424         rq->hrtick_timer.function = hrtick;
425 }
426 #else   /* CONFIG_SCHED_HRTICK */
427 static inline void hrtick_clear(struct rq *rq)
428 {
429 }
430 
431 static inline void hrtick_rq_init(struct rq *rq)
432 {
433 }
434 #endif  /* CONFIG_SCHED_HRTICK */
435 
436 /*
437  * cmpxchg based fetch_or, macro so it works for different integer types
438  */
439 #define fetch_or(ptr, mask)                                             \
440         ({                                                              \
441                 typeof(ptr) _ptr = (ptr);                               \
442                 typeof(mask) _mask = (mask);                            \
443                 typeof(*_ptr) _old, _val = *_ptr;                       \
444                                                                         \
445                 for (;;) {                                              \
446                         _old = cmpxchg(_ptr, _val, _val | _mask);       \
447                         if (_old == _val)                               \
448                                 break;                                  \
449                         _val = _old;                                    \
450                 }                                                       \
451         _old;                                                           \
452 })
453 
454 #if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG)
455 /*
456  * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG,
457  * this avoids any races wrt polling state changes and thereby avoids
458  * spurious IPIs.
459  */
460 static bool set_nr_and_not_polling(struct task_struct *p)
461 {
462         struct thread_info *ti = task_thread_info(p);
463         return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG);
464 }
465 
466 /*
467  * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set.
468  *
469  * If this returns true, then the idle task promises to call
470  * sched_ttwu_pending() and reschedule soon.
471  */
472 static bool set_nr_if_polling(struct task_struct *p)
473 {
474         struct thread_info *ti = task_thread_info(p);
475         typeof(ti->flags) old, val = READ_ONCE(ti->flags);
476 
477         for (;;) {
478                 if (!(val & _TIF_POLLING_NRFLAG))
479                         return false;
480                 if (val & _TIF_NEED_RESCHED)
481                         return true;
482                 old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED);
483                 if (old == val)
484                         break;
485                 val = old;
486         }
487         return true;
488 }
489 
490 #else
491 static bool set_nr_and_not_polling(struct task_struct *p)
492 {
493         set_tsk_need_resched(p);
494         return true;
495 }
496 
497 #ifdef CONFIG_SMP
498 static bool set_nr_if_polling(struct task_struct *p)
499 {
500         return false;
501 }
502 #endif
503 #endif
504 
505 static bool __wake_q_add(struct wake_q_head *head, struct task_struct *task)
506 {
507         struct wake_q_node *node = &task->wake_q;
508 
509         /*
510          * Atomically grab the task, if ->wake_q is !nil already it means
511          * it's already queued (either by us or someone else) and will get the
512          * wakeup due to that.
513          *
514          * In order to ensure that a pending wakeup will observe our pending
515          * state, even in the failed case, an explicit smp_mb() must be used.
516          */
517         smp_mb__before_atomic();
518         if (unlikely(cmpxchg_relaxed(&node->next, NULL, WAKE_Q_TAIL)))
519                 return false;
520 
521         /*
522          * The head is context local, there can be no concurrency.
523          */
524         *head->lastp = node;
525         head->lastp = &node->next;
526         return true;
527 }
528 
529 /**
530  * wake_q_add() - queue a wakeup for 'later' waking.
531  * @head: the wake_q_head to add @task to
532  * @task: the task to queue for 'later' wakeup
533  *
534  * Queue a task for later wakeup, most likely by the wake_up_q() call in the
535  * same context, _HOWEVER_ this is not guaranteed, the wakeup can come
536  * instantly.
537  *
538  * This function must be used as-if it were wake_up_process(); IOW the task
539  * must be ready to be woken at this location.
540  */
541 void wake_q_add(struct wake_q_head *head, struct task_struct *task)
542 {
543         if (__wake_q_add(head, task))
544                 get_task_struct(task);
545 }
546 
547 /**
548  * wake_q_add_safe() - safely queue a wakeup for 'later' waking.
549  * @head: the wake_q_head to add @task to
550  * @task: the task to queue for 'later' wakeup
551  *
552  * Queue a task for later wakeup, most likely by the wake_up_q() call in the
553  * same context, _HOWEVER_ this is not guaranteed, the wakeup can come
554  * instantly.
555  *
556  * This function must be used as-if it were wake_up_process(); IOW the task
557  * must be ready to be woken at this location.
558  *
559  * This function is essentially a task-safe equivalent to wake_q_add(). Callers
560  * that already hold reference to @task can call the 'safe' version and trust
561  * wake_q to do the right thing depending whether or not the @task is already
562  * queued for wakeup.
563  */
564 void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task)
565 {
566         if (!__wake_q_add(head, task))
567                 put_task_struct(task);
568 }
569 
570 void wake_up_q(struct wake_q_head *head)
571 {
572         struct wake_q_node *node = head->first;
573 
574         while (node != WAKE_Q_TAIL) {
575                 struct task_struct *task;
576 
577                 task = container_of(node, struct task_struct, wake_q);
578                 BUG_ON(!task);
579                 /* Task can safely be re-inserted now: */
580                 node = node->next;
581                 task->wake_q.next = NULL;
582 
583                 /*
584                  * wake_up_process() executes a full barrier, which pairs with
585                  * the queueing in wake_q_add() so as not to miss wakeups.
586                  */
587                 wake_up_process(task);
588                 put_task_struct(task);
589         }
590 }
591 
592 /*
593  * resched_curr - mark rq's current task 'to be rescheduled now'.
594  *
595  * On UP this means the setting of the need_resched flag, on SMP it
596  * might also involve a cross-CPU call to trigger the scheduler on
597  * the target CPU.
598  */
599 void resched_curr(struct rq *rq)
600 {
601         struct task_struct *curr = rq->curr;
602         int cpu;
603 
604         lockdep_assert_held(&rq->lock);
605 
606         if (test_tsk_need_resched(curr))
607                 return;
608 
609         cpu = cpu_of(rq);
610 
611         if (cpu == smp_processor_id()) {
612                 set_tsk_need_resched(curr);
613                 set_preempt_need_resched();
614                 return;
615         }
616 
617         if (set_nr_and_not_polling(curr))
618                 smp_send_reschedule(cpu);
619         else
620                 trace_sched_wake_idle_without_ipi(cpu);
621 }
622 
623 void resched_cpu(int cpu)
624 {
625         struct rq *rq = cpu_rq(cpu);
626         unsigned long flags;
627 
628         raw_spin_lock_irqsave(&rq->lock, flags);
629         if (cpu_online(cpu) || cpu == smp_processor_id())
630                 resched_curr(rq);
631         raw_spin_unlock_irqrestore(&rq->lock, flags);
632 }
633 
634 #ifdef CONFIG_SMP
635 #ifdef CONFIG_NO_HZ_COMMON
636 /*
637  * In the semi idle case, use the nearest busy CPU for migrating timers
638  * from an idle CPU.  This is good for power-savings.
639  *
640  * We don't do similar optimization for completely idle system, as
641  * selecting an idle CPU will add more delays to the timers than intended
642  * (as that CPU's timer base may not be uptodate wrt jiffies etc).
643  */
644 int get_nohz_timer_target(void)
645 {
646         int i, cpu = smp_processor_id(), default_cpu = -1;
647         struct sched_domain *sd;
648 
649         if (housekeeping_cpu(cpu, HK_FLAG_TIMER)) {
650                 if (!idle_cpu(cpu))
651                         return cpu;
652                 default_cpu = cpu;
653         }
654 
655         rcu_read_lock();
656         for_each_domain(cpu, sd) {
657                 for_each_cpu_and(i, sched_domain_span(sd),
658                         housekeeping_cpumask(HK_FLAG_TIMER)) {
659                         if (cpu == i)
660                                 continue;
661 
662                         if (!idle_cpu(i)) {
663                                 cpu = i;
664                                 goto unlock;
665                         }
666                 }
667         }
668 
669         if (default_cpu == -1)
670                 default_cpu = housekeeping_any_cpu(HK_FLAG_TIMER);
671         cpu = default_cpu;
672 unlock:
673         rcu_read_unlock();
674         return cpu;
675 }
676 
677 /*
678  * When add_timer_on() enqueues a timer into the timer wheel of an
679  * idle CPU then this timer might expire before the next timer event
680  * which is scheduled to wake up that CPU. In case of a completely
681  * idle system the next event might even be infinite time into the
682  * future. wake_up_idle_cpu() ensures that the CPU is woken up and
683  * leaves the inner idle loop so the newly added timer is taken into
684  * account when the CPU goes back to idle and evaluates the timer
685  * wheel for the next timer event.
686  */
687 static void wake_up_idle_cpu(int cpu)
688 {
689         struct rq *rq = cpu_rq(cpu);
690 
691         if (cpu == smp_processor_id())
692                 return;
693 
694         if (set_nr_and_not_polling(rq->idle))
695                 smp_send_reschedule(cpu);
696         else
697                 trace_sched_wake_idle_without_ipi(cpu);
698 }
699 
700 static bool wake_up_full_nohz_cpu(int cpu)
701 {
702         /*
703          * We just need the target to call irq_exit() and re-evaluate
704          * the next tick. The nohz full kick at least implies that.
705          * If needed we can still optimize that later with an
706          * empty IRQ.
707          */
708         if (cpu_is_offline(cpu))
709                 return true;  /* Don't try to wake offline CPUs. */
710         if (tick_nohz_full_cpu(cpu)) {
711                 if (cpu != smp_processor_id() ||
712                     tick_nohz_tick_stopped())
713                         tick_nohz_full_kick_cpu(cpu);
714                 return true;
715         }
716 
717         return false;
718 }
719 
720 /*
721  * Wake up the specified CPU.  If the CPU is going offline, it is the
722  * caller's responsibility to deal with the lost wakeup, for example,
723  * by hooking into the CPU_DEAD notifier like timers and hrtimers do.
724  */
725 void wake_up_nohz_cpu(int cpu)
726 {
727         if (!wake_up_full_nohz_cpu(cpu))
728                 wake_up_idle_cpu(cpu);
729 }
730 
731 static void nohz_csd_func(void *info)
732 {
733         struct rq *rq = info;
734         int cpu = cpu_of(rq);
735         unsigned int flags;
736 
737         /*
738          * Release the rq::nohz_csd.
739          */
740         flags = atomic_fetch_andnot(NOHZ_KICK_MASK, nohz_flags(cpu));
741         WARN_ON(!(flags & NOHZ_KICK_MASK));
742 
743         rq->idle_balance = idle_cpu(cpu);
744         if (rq->idle_balance && !need_resched()) {
745                 rq->nohz_idle_balance = flags;
746                 raise_softirq_irqoff(SCHED_SOFTIRQ);
747         }
748 }
749 
750 #endif /* CONFIG_NO_HZ_COMMON */
751 
752 #ifdef CONFIG_NO_HZ_FULL
753 bool sched_can_stop_tick(struct rq *rq)
754 {
755         int fifo_nr_running;
756 
757         /* Deadline tasks, even if single, need the tick */
758         if (rq->dl.dl_nr_running)
759                 return false;
760 
761         /*
762          * If there are more than one RR tasks, we need the tick to affect the
763          * actual RR behaviour.
764          */
765         if (rq->rt.rr_nr_running) {
766                 if (rq->rt.rr_nr_running == 1)
767                         return true;
768                 else
769                         return false;
770         }
771 
772         /*
773          * If there's no RR tasks, but FIFO tasks, we can skip the tick, no
774          * forced preemption between FIFO tasks.
775          */
776         fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running;
777         if (fifo_nr_running)
778                 return true;
779 
780         /*
781          * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left;
782          * if there's more than one we need the tick for involuntary
783          * preemption.
784          */
785         if (rq->nr_running > 1)
786                 return false;
787 
788         return true;
789 }
790 #endif /* CONFIG_NO_HZ_FULL */
791 #endif /* CONFIG_SMP */
792 
793 #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
794                         (defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH)))
795 /*
796  * Iterate task_group tree rooted at *from, calling @down when first entering a
797  * node and @up when leaving it for the final time.
798  *
799  * Caller must hold rcu_lock or sufficient equivalent.
800  */
801 int walk_tg_tree_from(struct task_group *from,
802                              tg_visitor down, tg_visitor up, void *data)
803 {
804         struct task_group *parent, *child;
805         int ret;
806 
807         parent = from;
808 
809 down:
810         ret = (*down)(parent, data);
811         if (ret)
812                 goto out;
813         list_for_each_entry_rcu(child, &parent->children, siblings) {
814                 parent = child;
815                 goto down;
816 
817 up:
818                 continue;
819         }
820         ret = (*up)(parent, data);
821         if (ret || parent == from)
822                 goto out;
823 
824         child = parent;
825         parent = parent->parent;
826         if (parent)
827                 goto up;
828 out:
829         return ret;
830 }
831 
832 int tg_nop(struct task_group *tg, void *data)
833 {
834         return 0;
835 }
836 #endif
837 
838 static void set_load_weight(struct task_struct *p, bool update_load)
839 {
840         int prio = p->static_prio - MAX_RT_PRIO;
841         struct load_weight *load = &p->se.load;
842 
843         /*
844          * SCHED_IDLE tasks get minimal weight:
845          */
846         if (task_has_idle_policy(p)) {
847                 load->weight = scale_load(WEIGHT_IDLEPRIO);
848                 load->inv_weight = WMULT_IDLEPRIO;
849                 return;
850         }
851 
852         /*
853          * SCHED_OTHER tasks have to update their load when changing their
854          * weight
855          */
856         if (update_load && p->sched_class == &fair_sched_class) {
857                 reweight_task(p, prio);
858         } else {
859                 load->weight = scale_load(sched_prio_to_weight[prio]);
860                 load->inv_weight = sched_prio_to_wmult[prio];
861         }
862 }
863 
864 #ifdef CONFIG_UCLAMP_TASK
865 /*
866  * Serializes updates of utilization clamp values
867  *
868  * The (slow-path) user-space triggers utilization clamp value updates which
869  * can require updates on (fast-path) scheduler's data structures used to
870  * support enqueue/dequeue operations.
871  * While the per-CPU rq lock protects fast-path update operations, user-space
872  * requests are serialized using a mutex to reduce the risk of conflicting
873  * updates or API abuses.
874  */
875 static DEFINE_MUTEX(uclamp_mutex);
876 
877 /* Max allowed minimum utilization */
878 unsigned int sysctl_sched_uclamp_util_min = SCHED_CAPACITY_SCALE;
879 
880 /* Max allowed maximum utilization */
881 unsigned int sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE;
882 
883 /*
884  * By default RT tasks run at the maximum performance point/capacity of the
885  * system. Uclamp enforces this by always setting UCLAMP_MIN of RT tasks to
886  * SCHED_CAPACITY_SCALE.
887  *
888  * This knob allows admins to change the default behavior when uclamp is being
889  * used. In battery powered devices, particularly, running at the maximum
890  * capacity and frequency will increase energy consumption and shorten the
891  * battery life.
892  *
893  * This knob only affects RT tasks that their uclamp_se->user_defined == false.
894  *
895  * This knob will not override the system default sched_util_clamp_min defined
896  * above.
897  */
898 unsigned int sysctl_sched_uclamp_util_min_rt_default = SCHED_CAPACITY_SCALE;
899 
900 /* All clamps are required to be less or equal than these values */
901 static struct uclamp_se uclamp_default[UCLAMP_CNT];
902 
903 /*
904  * This static key is used to reduce the uclamp overhead in the fast path. It
905  * primarily disables the call to uclamp_rq_{inc, dec}() in
906  * enqueue/dequeue_task().
907  *
908  * This allows users to continue to enable uclamp in their kernel config with
909  * minimum uclamp overhead in the fast path.
910  *
911  * As soon as userspace modifies any of the uclamp knobs, the static key is
912  * enabled, since we have an actual users that make use of uclamp
913  * functionality.
914  *
915  * The knobs that would enable this static key are:
916  *
917  *   * A task modifying its uclamp value with sched_setattr().
918  *   * An admin modifying the sysctl_sched_uclamp_{min, max} via procfs.
919  *   * An admin modifying the cgroup cpu.uclamp.{min, max}
920  */
921 DEFINE_STATIC_KEY_FALSE(sched_uclamp_used);
922 
923 /* Integer rounded range for each bucket */
924 #define UCLAMP_BUCKET_DELTA DIV_ROUND_CLOSEST(SCHED_CAPACITY_SCALE, UCLAMP_BUCKETS)
925 
926 #define for_each_clamp_id(clamp_id) \
927         for ((clamp_id) = 0; (clamp_id) < UCLAMP_CNT; (clamp_id)++)
928 
929 static inline unsigned int uclamp_bucket_id(unsigned int clamp_value)
930 {
931         return min_t(unsigned int, clamp_value / UCLAMP_BUCKET_DELTA, UCLAMP_BUCKETS - 1);
932 }
933 
934 static inline unsigned int uclamp_none(enum uclamp_id clamp_id)
935 {
936         if (clamp_id == UCLAMP_MIN)
937                 return 0;
938         return SCHED_CAPACITY_SCALE;
939 }
940 
941 static inline void uclamp_se_set(struct uclamp_se *uc_se,
942                                  unsigned int value, bool user_defined)
943 {
944         uc_se->value = value;
945         uc_se->bucket_id = uclamp_bucket_id(value);
946         uc_se->user_defined = user_defined;
947 }
948 
949 static inline unsigned int
950 uclamp_idle_value(struct rq *rq, enum uclamp_id clamp_id,
951                   unsigned int clamp_value)
952 {
953         /*
954          * Avoid blocked utilization pushing up the frequency when we go
955          * idle (which drops the max-clamp) by retaining the last known
956          * max-clamp.
957          */
958         if (clamp_id == UCLAMP_MAX) {
959                 rq->uclamp_flags |= UCLAMP_FLAG_IDLE;
960                 return clamp_value;
961         }
962 
963         return uclamp_none(UCLAMP_MIN);
964 }
965 
966 static inline void uclamp_idle_reset(struct rq *rq, enum uclamp_id clamp_id,
967                                      unsigned int clamp_value)
968 {
969         /* Reset max-clamp retention only on idle exit */
970         if (!(rq->uclamp_flags & UCLAMP_FLAG_IDLE))
971                 return;
972 
973         WRITE_ONCE(rq->uclamp[clamp_id].value, clamp_value);
974 }
975 
976 static inline
977 unsigned int uclamp_rq_max_value(struct rq *rq, enum uclamp_id clamp_id,
978                                    unsigned int clamp_value)
979 {
980         struct uclamp_bucket *bucket = rq->uclamp[clamp_id].bucket;
981         int bucket_id = UCLAMP_BUCKETS - 1;
982 
983         /*
984          * Since both min and max clamps are max aggregated, find the
985          * top most bucket with tasks in.
986          */
987         for ( ; bucket_id >= 0; bucket_id--) {
988                 if (!bucket[bucket_id].tasks)
989                         continue;
990                 return bucket[bucket_id].value;
991         }
992 
993         /* No tasks -- default clamp values */
994         return uclamp_idle_value(rq, clamp_id, clamp_value);
995 }
996 
997 static void __uclamp_update_util_min_rt_default(struct task_struct *p)
998 {
999         unsigned int default_util_min;
1000         struct uclamp_se *uc_se;
1001 
1002         lockdep_assert_held(&p->pi_lock);
1003 
1004         uc_se = &p->uclamp_req[UCLAMP_MIN];
1005 
1006         /* Only sync if user didn't override the default */
1007         if (uc_se->user_defined)
1008                 return;
1009 
1010         default_util_min = sysctl_sched_uclamp_util_min_rt_default;
1011         uclamp_se_set(uc_se, default_util_min, false);
1012 }
1013 
1014 static void uclamp_update_util_min_rt_default(struct task_struct *p)
1015 {
1016         struct rq_flags rf;
1017         struct rq *rq;
1018 
1019         if (!rt_task(p))
1020                 return;
1021 
1022         /* Protect updates to p->uclamp_* */
1023         rq = task_rq_lock(p, &rf);
1024         __uclamp_update_util_min_rt_default(p);
1025         task_rq_unlock(rq, p, &rf);
1026 }
1027 
1028 static void uclamp_sync_util_min_rt_default(void)
1029 {
1030         struct task_struct *g, *p;
1031 
1032         /*
1033          * copy_process()                       sysctl_uclamp
1034          *                                        uclamp_min_rt = X;
1035          *   write_lock(&tasklist_lock)           read_lock(&tasklist_lock)
1036          *   // link thread                       smp_mb__after_spinlock()
1037          *   write_unlock(&tasklist_lock)         read_unlock(&tasklist_lock);
1038          *   sched_post_fork()                    for_each_process_thread()
1039          *     __uclamp_sync_rt()                   __uclamp_sync_rt()
1040          *
1041          * Ensures that either sched_post_fork() will observe the new
1042          * uclamp_min_rt or for_each_process_thread() will observe the new
1043          * task.
1044          */
1045         read_lock(&tasklist_lock);
1046         smp_mb__after_spinlock();
1047         read_unlock(&tasklist_lock);
1048 
1049         rcu_read_lock();
1050         for_each_process_thread(g, p)
1051                 uclamp_update_util_min_rt_default(p);
1052         rcu_read_unlock();
1053 }
1054 
1055 static inline struct uclamp_se
1056 uclamp_tg_restrict(struct task_struct *p, enum uclamp_id clamp_id)
1057 {
1058         /* Copy by value as we could modify it */
1059         struct uclamp_se uc_req = p->uclamp_req[clamp_id];
1060 #ifdef CONFIG_UCLAMP_TASK_GROUP
1061         unsigned int tg_min, tg_max, value;
1062 
1063         /*
1064          * Tasks in autogroups or root task group will be
1065          * restricted by system defaults.
1066          */
1067         if (task_group_is_autogroup(task_group(p)))
1068                 return uc_req;
1069         if (task_group(p) == &root_task_group)
1070                 return uc_req;
1071 
1072         tg_min = task_group(p)->uclamp[UCLAMP_MIN].value;
1073         tg_max = task_group(p)->uclamp[UCLAMP_MAX].value;
1074         value = uc_req.value;
1075         value = clamp(value, tg_min, tg_max);
1076         uclamp_se_set(&uc_req, value, false);
1077 #endif
1078 
1079         return uc_req;
1080 }
1081 
1082 /*
1083  * The effective clamp bucket index of a task depends on, by increasing
1084  * priority:
1085  * - the task specific clamp value, when explicitly requested from userspace
1086  * - the task group effective clamp value, for tasks not either in the root
1087  *   group or in an autogroup
1088  * - the system default clamp value, defined by the sysadmin
1089  */
1090 static inline struct uclamp_se
1091 uclamp_eff_get(struct task_struct *p, enum uclamp_id clamp_id)
1092 {
1093         struct uclamp_se uc_req = uclamp_tg_restrict(p, clamp_id);
1094         struct uclamp_se uc_max = uclamp_default[clamp_id];
1095 
1096         /* System default restrictions always apply */
1097         if (unlikely(uc_req.value > uc_max.value))
1098                 return uc_max;
1099 
1100         return uc_req;
1101 }
1102 
1103 unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id)
1104 {
1105         struct uclamp_se uc_eff;
1106 
1107         /* Task currently refcounted: use back-annotated (effective) value */
1108         if (p->uclamp[clamp_id].active)
1109                 return (unsigned long)p->uclamp[clamp_id].value;
1110 
1111         uc_eff = uclamp_eff_get(p, clamp_id);
1112 
1113         return (unsigned long)uc_eff.value;
1114 }
1115 
1116 /*
1117  * When a task is enqueued on a rq, the clamp bucket currently defined by the
1118  * task's uclamp::bucket_id is refcounted on that rq. This also immediately
1119  * updates the rq's clamp value if required.
1120  *
1121  * Tasks can have a task-specific value requested from user-space, track
1122  * within each bucket the maximum value for tasks refcounted in it.
1123  * This "local max aggregation" allows to track the exact "requested" value
1124  * for each bucket when all its RUNNABLE tasks require the same clamp.
1125  */
1126 static inline void uclamp_rq_inc_id(struct rq *rq, struct task_struct *p,
1127                                     enum uclamp_id clamp_id)
1128 {
1129         struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id];
1130         struct uclamp_se *uc_se = &p->uclamp[clamp_id];
1131         struct uclamp_bucket *bucket;
1132 
1133         lockdep_assert_held(&rq->lock);
1134 
1135         /* Update task effective clamp */
1136         p->uclamp[clamp_id] = uclamp_eff_get(p, clamp_id);
1137 
1138         bucket = &uc_rq->bucket[uc_se->bucket_id];
1139         bucket->tasks++;
1140         uc_se->active = true;
1141 
1142         uclamp_idle_reset(rq, clamp_id, uc_se->value);
1143 
1144         /*
1145          * Local max aggregation: rq buckets always track the max
1146          * "requested" clamp value of its RUNNABLE tasks.
1147          */
1148         if (bucket->tasks == 1 || uc_se->value > bucket->value)
1149                 bucket->value = uc_se->value;
1150 
1151         if (uc_se->value > READ_ONCE(uc_rq->value))
1152                 WRITE_ONCE(uc_rq->value, uc_se->value);
1153 }
1154 
1155 /*
1156  * When a task is dequeued from a rq, the clamp bucket refcounted by the task
1157  * is released. If this is the last task reference counting the rq's max
1158  * active clamp value, then the rq's clamp value is updated.
1159  *
1160  * Both refcounted tasks and rq's cached clamp values are expected to be
1161  * always valid. If it's detected they are not, as defensive programming,
1162  * enforce the expected state and warn.
1163  */
1164 static inline void uclamp_rq_dec_id(struct rq *rq, struct task_struct *p,
1165                                     enum uclamp_id clamp_id)
1166 {
1167         struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id];
1168         struct uclamp_se *uc_se = &p->uclamp[clamp_id];
1169         struct uclamp_bucket *bucket;
1170         unsigned int bkt_clamp;
1171         unsigned int rq_clamp;
1172 
1173         lockdep_assert_held(&rq->lock);
1174 
1175         /*
1176          * If sched_uclamp_used was enabled after task @p was enqueued,
1177          * we could end up with unbalanced call to uclamp_rq_dec_id().
1178          *
1179          * In this case the uc_se->active flag should be false since no uclamp
1180          * accounting was performed at enqueue time and we can just return
1181          * here.
1182          *
1183          * Need to be careful of the following enqueue/dequeue ordering
1184          * problem too
1185          *
1186          *      enqueue(taskA)
1187          *      // sched_uclamp_used gets enabled
1188          *      enqueue(taskB)
1189          *      dequeue(taskA)
1190          *      // Must not decrement bucket->tasks here
1191          *      dequeue(taskB)
1192          *
1193          * where we could end up with stale data in uc_se and
1194          * bucket[uc_se->bucket_id].
1195          *
1196          * The following check here eliminates the possibility of such race.
1197          */
1198         if (unlikely(!uc_se->active))
1199                 return;
1200 
1201         bucket = &uc_rq->bucket[uc_se->bucket_id];
1202 
1203         SCHED_WARN_ON(!bucket->tasks);
1204         if (likely(bucket->tasks))
1205                 bucket->tasks--;
1206 
1207         uc_se->active = false;
1208 
1209         /*
1210          * Keep "local max aggregation" simple and accept to (possibly)
1211          * overboost some RUNNABLE tasks in the same bucket.
1212          * The rq clamp bucket value is reset to its base value whenever
1213          * there are no more RUNNABLE tasks refcounting it.
1214          */
1215         if (likely(bucket->tasks))
1216                 return;
1217 
1218         rq_clamp = READ_ONCE(uc_rq->value);
1219         /*
1220          * Defensive programming: this should never happen. If it happens,
1221          * e.g. due to future modification, warn and fixup the expected value.
1222          */
1223         SCHED_WARN_ON(bucket->value > rq_clamp);
1224         if (bucket->value >= rq_clamp) {
1225                 bkt_clamp = uclamp_rq_max_value(rq, clamp_id, uc_se->value);
1226                 WRITE_ONCE(uc_rq->value, bkt_clamp);
1227         }
1228 }
1229 
1230 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p)
1231 {
1232         enum uclamp_id clamp_id;
1233 
1234         /*
1235          * Avoid any overhead until uclamp is actually used by the userspace.
1236          *
1237          * The condition is constructed such that a NOP is generated when
1238          * sched_uclamp_used is disabled.
1239          */
1240         if (!static_branch_unlikely(&sched_uclamp_used))
1241                 return;
1242 
1243         if (unlikely(!p->sched_class->uclamp_enabled))
1244                 return;
1245 
1246         for_each_clamp_id(clamp_id)
1247                 uclamp_rq_inc_id(rq, p, clamp_id);
1248 
1249         /* Reset clamp idle holding when there is one RUNNABLE task */
1250         if (rq->uclamp_flags & UCLAMP_FLAG_IDLE)
1251                 rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE;
1252 }
1253 
1254 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p)
1255 {
1256         enum uclamp_id clamp_id;
1257 
1258         /*
1259          * Avoid any overhead until uclamp is actually used by the userspace.
1260          *
1261          * The condition is constructed such that a NOP is generated when
1262          * sched_uclamp_used is disabled.
1263          */
1264         if (!static_branch_unlikely(&sched_uclamp_used))
1265                 return;
1266 
1267         if (unlikely(!p->sched_class->uclamp_enabled))
1268                 return;
1269 
1270         for_each_clamp_id(clamp_id)
1271                 uclamp_rq_dec_id(rq, p, clamp_id);
1272 }
1273 
1274 static inline void
1275 uclamp_update_active(struct task_struct *p)
1276 {
1277         enum uclamp_id clamp_id;
1278         struct rq_flags rf;
1279         struct rq *rq;
1280 
1281         /*
1282          * Lock the task and the rq where the task is (or was) queued.
1283          *
1284          * We might lock the (previous) rq of a !RUNNABLE task, but that's the
1285          * price to pay to safely serialize util_{min,max} updates with
1286          * enqueues, dequeues and migration operations.
1287          * This is the same locking schema used by __set_cpus_allowed_ptr().
1288          */
1289         rq = task_rq_lock(p, &rf);
1290 
1291         /*
1292          * Setting the clamp bucket is serialized by task_rq_lock().
1293          * If the task is not yet RUNNABLE and its task_struct is not
1294          * affecting a valid clamp bucket, the next time it's enqueued,
1295          * it will already see the updated clamp bucket value.
1296          */
1297         for_each_clamp_id(clamp_id) {
1298                 if (p->uclamp[clamp_id].active) {
1299                         uclamp_rq_dec_id(rq, p, clamp_id);
1300                         uclamp_rq_inc_id(rq, p, clamp_id);
1301                 }
1302         }
1303 
1304         task_rq_unlock(rq, p, &rf);
1305 }
1306 
1307 #ifdef CONFIG_UCLAMP_TASK_GROUP
1308 static inline void
1309 uclamp_update_active_tasks(struct cgroup_subsys_state *css)
1310 {
1311         struct css_task_iter it;
1312         struct task_struct *p;
1313 
1314         css_task_iter_start(css, 0, &it);
1315         while ((p = css_task_iter_next(&it)))
1316                 uclamp_update_active(p);
1317         css_task_iter_end(&it);
1318 }
1319 
1320 static void cpu_util_update_eff(struct cgroup_subsys_state *css);
1321 static void uclamp_update_root_tg(void)
1322 {
1323         struct task_group *tg = &root_task_group;
1324 
1325         uclamp_se_set(&tg->uclamp_req[UCLAMP_MIN],
1326                       sysctl_sched_uclamp_util_min, false);
1327         uclamp_se_set(&tg->uclamp_req[UCLAMP_MAX],
1328                       sysctl_sched_uclamp_util_max, false);
1329 
1330         rcu_read_lock();
1331         cpu_util_update_eff(&root_task_group.css);
1332         rcu_read_unlock();
1333 }
1334 #else
1335 static void uclamp_update_root_tg(void) { }
1336 #endif
1337 
1338 int sysctl_sched_uclamp_handler(struct ctl_table *table, int write,
1339                                 void *buffer, size_t *lenp, loff_t *ppos)
1340 {
1341         bool update_root_tg = false;
1342         int old_min, old_max, old_min_rt;
1343         int result;
1344 
1345         mutex_lock(&uclamp_mutex);
1346         old_min = sysctl_sched_uclamp_util_min;
1347         old_max = sysctl_sched_uclamp_util_max;
1348         old_min_rt = sysctl_sched_uclamp_util_min_rt_default;
1349 
1350         result = proc_dointvec(table, write, buffer, lenp, ppos);
1351         if (result)
1352                 goto undo;
1353         if (!write)
1354                 goto done;
1355 
1356         if (sysctl_sched_uclamp_util_min > sysctl_sched_uclamp_util_max ||
1357             sysctl_sched_uclamp_util_max > SCHED_CAPACITY_SCALE ||
1358             sysctl_sched_uclamp_util_min_rt_default > SCHED_CAPACITY_SCALE) {
1359 
1360                 result = -EINVAL;
1361                 goto undo;
1362         }
1363 
1364         if (old_min != sysctl_sched_uclamp_util_min) {
1365                 uclamp_se_set(&uclamp_default[UCLAMP_MIN],
1366                               sysctl_sched_uclamp_util_min, false);
1367                 update_root_tg = true;
1368         }
1369         if (old_max != sysctl_sched_uclamp_util_max) {
1370                 uclamp_se_set(&uclamp_default[UCLAMP_MAX],
1371                               sysctl_sched_uclamp_util_max, false);
1372                 update_root_tg = true;
1373         }
1374 
1375         if (update_root_tg) {
1376                 static_branch_enable(&sched_uclamp_used);
1377                 uclamp_update_root_tg();
1378         }
1379 
1380         if (old_min_rt != sysctl_sched_uclamp_util_min_rt_default) {
1381                 static_branch_enable(&sched_uclamp_used);
1382                 uclamp_sync_util_min_rt_default();
1383         }
1384 
1385         /*
1386          * We update all RUNNABLE tasks only when task groups are in use.
1387          * Otherwise, keep it simple and do just a lazy update at each next
1388          * task enqueue time.
1389          */
1390 
1391         goto done;
1392 
1393 undo:
1394         sysctl_sched_uclamp_util_min = old_min;
1395         sysctl_sched_uclamp_util_max = old_max;
1396         sysctl_sched_uclamp_util_min_rt_default = old_min_rt;
1397 done:
1398         mutex_unlock(&uclamp_mutex);
1399 
1400         return result;
1401 }
1402 
1403 static int uclamp_validate(struct task_struct *p,
1404                            const struct sched_attr *attr)
1405 {
1406         int util_min = p->uclamp_req[UCLAMP_MIN].value;
1407         int util_max = p->uclamp_req[UCLAMP_MAX].value;
1408 
1409         if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN) {
1410                 util_min = attr->sched_util_min;
1411 
1412                 if (util_min + 1 > SCHED_CAPACITY_SCALE + 1)
1413                         return -EINVAL;
1414         }
1415 
1416         if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX) {
1417                 util_max = attr->sched_util_max;
1418 
1419                 if (util_max + 1 > SCHED_CAPACITY_SCALE + 1)
1420                         return -EINVAL;
1421         }
1422 
1423         if (util_min != -1 && util_max != -1 && util_min > util_max)
1424                 return -EINVAL;
1425 
1426         /*
1427          * We have valid uclamp attributes; make sure uclamp is enabled.
1428          *
1429          * We need to do that here, because enabling static branches is a
1430          * blocking operation which obviously cannot be done while holding
1431          * scheduler locks.
1432          */
1433         static_branch_enable(&sched_uclamp_used);
1434 
1435         return 0;
1436 }
1437 
1438 static bool uclamp_reset(const struct sched_attr *attr,
1439                          enum uclamp_id clamp_id,
1440                          struct uclamp_se *uc_se)
1441 {
1442         /* Reset on sched class change for a non user-defined clamp value. */
1443         if (likely(!(attr->sched_flags & SCHED_FLAG_UTIL_CLAMP)) &&
1444             !uc_se->user_defined)
1445                 return true;
1446 
1447         /* Reset on sched_util_{min,max} == -1. */
1448         if (clamp_id == UCLAMP_MIN &&
1449             attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN &&
1450             attr->sched_util_min == -1) {
1451                 return true;
1452         }
1453 
1454         if (clamp_id == UCLAMP_MAX &&
1455             attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX &&
1456             attr->sched_util_max == -1) {
1457                 return true;
1458         }
1459 
1460         return false;
1461 }
1462 
1463 static void __setscheduler_uclamp(struct task_struct *p,
1464                                   const struct sched_attr *attr)
1465 {
1466         enum uclamp_id clamp_id;
1467 
1468         for_each_clamp_id(clamp_id) {
1469                 struct uclamp_se *uc_se = &p->uclamp_req[clamp_id];
1470                 unsigned int value;
1471 
1472                 if (!uclamp_reset(attr, clamp_id, uc_se))
1473                         continue;
1474 
1475                 /*
1476                  * RT by default have a 100% boost value that could be modified
1477                  * at runtime.
1478                  */
1479                 if (unlikely(rt_task(p) && clamp_id == UCLAMP_MIN))
1480                         value = sysctl_sched_uclamp_util_min_rt_default;
1481                 else
1482                         value = uclamp_none(clamp_id);
1483 
1484                 uclamp_se_set(uc_se, value, false);
1485 
1486         }
1487 
1488         if (likely(!(attr->sched_flags & SCHED_FLAG_UTIL_CLAMP)))
1489                 return;
1490 
1491         if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN &&
1492             attr->sched_util_min != -1) {
1493                 uclamp_se_set(&p->uclamp_req[UCLAMP_MIN],
1494                               attr->sched_util_min, true);
1495         }
1496 
1497         if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX &&
1498             attr->sched_util_max != -1) {
1499                 uclamp_se_set(&p->uclamp_req[UCLAMP_MAX],
1500                               attr->sched_util_max, true);
1501         }
1502 }
1503 
1504 static void uclamp_fork(struct task_struct *p)
1505 {
1506         enum uclamp_id clamp_id;
1507 
1508         /*
1509          * We don't need to hold task_rq_lock() when updating p->uclamp_* here
1510          * as the task is still at its early fork stages.
1511          */
1512         for_each_clamp_id(clamp_id)
1513                 p->uclamp[clamp_id].active = false;
1514 
1515         if (likely(!p->sched_reset_on_fork))
1516                 return;
1517 
1518         for_each_clamp_id(clamp_id) {
1519                 uclamp_se_set(&p->uclamp_req[clamp_id],
1520                               uclamp_none(clamp_id), false);
1521         }
1522 }
1523 
1524 static void uclamp_post_fork(struct task_struct *p)
1525 {
1526         uclamp_update_util_min_rt_default(p);
1527 }
1528 
1529 static void __init init_uclamp_rq(struct rq *rq)
1530 {
1531         enum uclamp_id clamp_id;
1532         struct uclamp_rq *uc_rq = rq->uclamp;
1533 
1534         for_each_clamp_id(clamp_id) {
1535                 uc_rq[clamp_id] = (struct uclamp_rq) {
1536                         .value = uclamp_none(clamp_id)
1537                 };
1538         }
1539 
1540         rq->uclamp_flags = 0;
1541 }
1542 
1543 static void __init init_uclamp(void)
1544 {
1545         struct uclamp_se uc_max = {};
1546         enum uclamp_id clamp_id;
1547         int cpu;
1548 
1549         for_each_possible_cpu(cpu)
1550                 init_uclamp_rq(cpu_rq(cpu));
1551 
1552         for_each_clamp_id(clamp_id) {
1553                 uclamp_se_set(&init_task.uclamp_req[clamp_id],
1554                               uclamp_none(clamp_id), false);
1555         }
1556 
1557         /* System defaults allow max clamp values for both indexes */
1558         uclamp_se_set(&uc_max, uclamp_none(UCLAMP_MAX), false);
1559         for_each_clamp_id(clamp_id) {
1560                 uclamp_default[clamp_id] = uc_max;
1561 #ifdef CONFIG_UCLAMP_TASK_GROUP
1562                 root_task_group.uclamp_req[clamp_id] = uc_max;
1563                 root_task_group.uclamp[clamp_id] = uc_max;
1564 #endif
1565         }
1566 }
1567 
1568 #else /* CONFIG_UCLAMP_TASK */
1569 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) { }
1570 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) { }
1571 static inline int uclamp_validate(struct task_struct *p,
1572                                   const struct sched_attr *attr)
1573 {
1574         return -EOPNOTSUPP;
1575 }
1576 static void __setscheduler_uclamp(struct task_struct *p,
1577                                   const struct sched_attr *attr) { }
1578 static inline void uclamp_fork(struct task_struct *p) { }
1579 static inline void uclamp_post_fork(struct task_struct *p) { }
1580 static inline void init_uclamp(void) { }
1581 #endif /* CONFIG_UCLAMP_TASK */
1582 
1583 static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
1584 {
1585         if (!(flags & ENQUEUE_NOCLOCK))
1586                 update_rq_clock(rq);
1587 
1588         if (!(flags & ENQUEUE_RESTORE)) {
1589                 sched_info_queued(rq, p);
1590                 psi_enqueue(p, flags & ENQUEUE_WAKEUP);
1591         }
1592 
1593         uclamp_rq_inc(rq, p);
1594         p->sched_class->enqueue_task(rq, p, flags);
1595 }
1596 
1597 static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
1598 {
1599         if (!(flags & DEQUEUE_NOCLOCK))
1600                 update_rq_clock(rq);
1601 
1602         if (!(flags & DEQUEUE_SAVE)) {
1603                 sched_info_dequeued(rq, p);
1604                 psi_dequeue(p, flags & DEQUEUE_SLEEP);
1605         }
1606 
1607         uclamp_rq_dec(rq, p);
1608         p->sched_class->dequeue_task(rq, p, flags);
1609 }
1610 
1611 void activate_task(struct rq *rq, struct task_struct *p, int flags)
1612 {
1613         enqueue_task(rq, p, flags);
1614 
1615         p->on_rq = TASK_ON_RQ_QUEUED;
1616 }
1617 
1618 void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
1619 {
1620         p->on_rq = (flags & DEQUEUE_SLEEP) ? 0 : TASK_ON_RQ_MIGRATING;
1621 
1622         dequeue_task(rq, p, flags);
1623 }
1624 
1625 /*
1626  * __normal_prio - return the priority that is based on the static prio
1627  */
1628 static inline int __normal_prio(struct task_struct *p)
1629 {
1630         return p->static_prio;
1631 }
1632 
1633 /*
1634  * Calculate the expected normal priority: i.e. priority
1635  * without taking RT-inheritance into account. Might be
1636  * boosted by interactivity modifiers. Changes upon fork,
1637  * setprio syscalls, and whenever the interactivity
1638  * estimator recalculates.
1639  */
1640 static inline int normal_prio(struct task_struct *p)
1641 {
1642         int prio;
1643 
1644         if (task_has_dl_policy(p))
1645                 prio = MAX_DL_PRIO-1;
1646         else if (task_has_rt_policy(p))
1647                 prio = MAX_RT_PRIO-1 - p->rt_priority;
1648         else
1649                 prio = __normal_prio(p);
1650         return prio;
1651 }
1652 
1653 /*
1654  * Calculate the current priority, i.e. the priority
1655  * taken into account by the scheduler. This value might
1656  * be boosted by RT tasks, or might be boosted by
1657  * interactivity modifiers. Will be RT if the task got
1658  * RT-boosted. If not then it returns p->normal_prio.
1659  */
1660 static int effective_prio(struct task_struct *p)
1661 {
1662         p->normal_prio = normal_prio(p);
1663         /*
1664          * If we are RT tasks or we were boosted to RT priority,
1665          * keep the priority unchanged. Otherwise, update priority
1666          * to the normal priority:
1667          */
1668         if (!rt_prio(p->prio))
1669                 return p->normal_prio;
1670         return p->prio;
1671 }
1672 
1673 /**
1674  * task_curr - is this task currently executing on a CPU?
1675  * @p: the task in question.
1676  *
1677  * Return: 1 if the task is currently executing. 0 otherwise.
1678  */
1679 inline int task_curr(const struct task_struct *p)
1680 {
1681         return cpu_curr(task_cpu(p)) == p;
1682 }
1683 
1684 /*
1685  * switched_from, switched_to and prio_changed must _NOT_ drop rq->lock,
1686  * use the balance_callback list if you want balancing.
1687  *
1688  * this means any call to check_class_changed() must be followed by a call to
1689  * balance_callback().
1690  */
1691 static inline void check_class_changed(struct rq *rq, struct task_struct *p,
1692                                        const struct sched_class *prev_class,
1693                                        int oldprio)
1694 {
1695         if (prev_class != p->sched_class) {
1696                 if (prev_class->switched_from)
1697                         prev_class->switched_from(rq, p);
1698 
1699                 p->sched_class->switched_to(rq, p);
1700         } else if (oldprio != p->prio || dl_task(p))
1701                 p->sched_class->prio_changed(rq, p, oldprio);
1702 }
1703 
1704 void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
1705 {
1706         if (p->sched_class == rq->curr->sched_class)
1707                 rq->curr->sched_class->check_preempt_curr(rq, p, flags);
1708         else if (p->sched_class > rq->curr->sched_class)
1709                 resched_curr(rq);
1710 
1711         /*
1712          * A queue event has occurred, and we're going to schedule.  In
1713          * this case, we can save a useless back to back clock update.
1714          */
1715         if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr))
1716                 rq_clock_skip_update(rq);
1717 }
1718 
1719 #ifdef CONFIG_SMP
1720 
1721 static void
1722 __do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32 flags);
1723 
1724 static int __set_cpus_allowed_ptr(struct task_struct *p,
1725                                   const struct cpumask *new_mask,
1726                                   u32 flags);
1727 
1728 static void migrate_disable_switch(struct rq *rq, struct task_struct *p)
1729 {
1730         if (likely(!p->migration_disabled))
1731                 return;
1732 
1733         if (p->cpus_ptr != &p->cpus_mask)
1734                 return;
1735 
1736         /*
1737          * Violates locking rules! see comment in __do_set_cpus_allowed().
1738          */
1739         __do_set_cpus_allowed(p, cpumask_of(rq->cpu), SCA_MIGRATE_DISABLE);
1740 }
1741 
1742 void migrate_disable(void)
1743 {
1744         struct task_struct *p = current;
1745 
1746         if (p->migration_disabled) {
1747                 p->migration_disabled++;
1748                 return;
1749         }
1750 
1751         preempt_disable();
1752         this_rq()->nr_pinned++;
1753         p->migration_disabled = 1;
1754         preempt_enable();
1755 }
1756 EXPORT_SYMBOL_GPL(migrate_disable);
1757 
1758 void migrate_enable(void)
1759 {
1760         struct task_struct *p = current;
1761 
1762         if (p->migration_disabled > 1) {
1763                 p->migration_disabled--;
1764                 return;
1765         }
1766 
1767         /*
1768          * Ensure stop_task runs either before or after this, and that
1769          * __set_cpus_allowed_ptr(SCA_MIGRATE_ENABLE) doesn't schedule().
1770          */
1771         preempt_disable();
1772         if (p->cpus_ptr != &p->cpus_mask)
1773                 __set_cpus_allowed_ptr(p, &p->cpus_mask, SCA_MIGRATE_ENABLE);
1774         /*
1775          * Mustn't clear migration_disabled() until cpus_ptr points back at the
1776          * regular cpus_mask, otherwise things that race (eg.
1777          * select_fallback_rq) get confused.
1778          */
1779         barrier();
1780         p->migration_disabled = 0;
1781         this_rq()->nr_pinned--;
1782         preempt_enable();
1783 }
1784 EXPORT_SYMBOL_GPL(migrate_enable);
1785 
1786 static inline bool rq_has_pinned_tasks(struct rq *rq)
1787 {
1788         return rq->nr_pinned;
1789 }
1790 
1791 /*
1792  * Per-CPU kthreads are allowed to run on !active && online CPUs, see
1793  * __set_cpus_allowed_ptr() and select_fallback_rq().
1794  */
1795 static inline bool is_cpu_allowed(struct task_struct *p, int cpu)
1796 {
1797         /* When not in the task's cpumask, no point in looking further. */
1798         if (!cpumask_test_cpu(cpu, p->cpus_ptr))
1799                 return false;
1800 
1801         /* migrate_disabled() must be allowed to finish. */
1802         if (is_migration_disabled(p))
1803                 return cpu_online(cpu);
1804 
1805         /* Non kernel threads are not allowed during either online or offline. */
1806         if (!(p->flags & PF_KTHREAD))
1807                 return cpu_active(cpu);
1808 
1809         /* KTHREAD_IS_PER_CPU is always allowed. */
1810         if (kthread_is_per_cpu(p))
1811                 return cpu_online(cpu);
1812 
1813         /* Regular kernel threads don't get to stay during offline. */
1814         if (cpu_rq(cpu)->balance_push)
1815                 return false;
1816 
1817         /* But are allowed during online. */
1818         return cpu_online(cpu);
1819 }
1820 
1821 /*
1822  * This is how migration works:
1823  *
1824  * 1) we invoke migration_cpu_stop() on the target CPU using
1825  *    stop_one_cpu().
1826  * 2) stopper starts to run (implicitly forcing the migrated thread
1827  *    off the CPU)
1828  * 3) it checks whether the migrated task is still in the wrong runqueue.
1829  * 4) if it's in the wrong runqueue then the migration thread removes
1830  *    it and puts it into the right queue.
1831  * 5) stopper completes and stop_one_cpu() returns and the migration
1832  *    is done.
1833  */
1834 
1835 /*
1836  * move_queued_task - move a queued task to new rq.
1837  *
1838  * Returns (locked) new rq. Old rq's lock is released.
1839  */
1840 static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf,
1841                                    struct task_struct *p, int new_cpu)
1842 {
1843         lockdep_assert_held(&rq->lock);
1844 
1845         deactivate_task(rq, p, DEQUEUE_NOCLOCK);
1846         set_task_cpu(p, new_cpu);
1847         rq_unlock(rq, rf);
1848 
1849         rq = cpu_rq(new_cpu);
1850 
1851         rq_lock(rq, rf);
1852         BUG_ON(task_cpu(p) != new_cpu);
1853         activate_task(rq, p, 0);
1854         check_preempt_curr(rq, p, 0);
1855 
1856         return rq;
1857 }
1858 
1859 struct migration_arg {
1860         struct task_struct              *task;
1861         int                             dest_cpu;
1862         struct set_affinity_pending     *pending;
1863 };
1864 
1865 /*
1866  * @refs: number of wait_for_completion()
1867  * @stop_pending: is @stop_work in use
1868  */
1869 struct set_affinity_pending {
1870         refcount_t              refs;
1871         unsigned int            stop_pending;
1872         struct completion       done;
1873         struct cpu_stop_work    stop_work;
1874         struct migration_arg    arg;
1875 };
1876 
1877 /*
1878  * Move (not current) task off this CPU, onto the destination CPU. We're doing
1879  * this because either it can't run here any more (set_cpus_allowed()
1880  * away from this CPU, or CPU going down), or because we're
1881  * attempting to rebalance this task on exec (sched_exec).
1882  *
1883  * So we race with normal scheduler movements, but that's OK, as long
1884  * as the task is no longer on this CPU.
1885  */
1886 static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf,
1887                                  struct task_struct *p, int dest_cpu)
1888 {
1889         /* Affinity changed (again). */
1890         if (!is_cpu_allowed(p, dest_cpu))
1891                 return rq;
1892 
1893         update_rq_clock(rq);
1894         rq = move_queued_task(rq, rf, p, dest_cpu);
1895 
1896         return rq;
1897 }
1898 
1899 /*
1900  * migration_cpu_stop - this will be executed by a highprio stopper thread
1901  * and performs thread migration by bumping thread off CPU then
1902  * 'pushing' onto another runqueue.
1903  */
1904 static int migration_cpu_stop(void *data)
1905 {
1906         struct migration_arg *arg = data;
1907         struct set_affinity_pending *pending = arg->pending;
1908         struct task_struct *p = arg->task;
1909         struct rq *rq = this_rq();
1910         bool complete = false;
1911         struct rq_flags rf;
1912 
1913         /*
1914          * The original target CPU might have gone down and we might
1915          * be on another CPU but it doesn't matter.
1916          */
1917         local_irq_save(rf.flags);
1918         /*
1919          * We need to explicitly wake pending tasks before running
1920          * __migrate_task() such that we will not miss enforcing cpus_ptr
1921          * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test.
1922          */
1923         flush_smp_call_function_from_idle();
1924 
1925         raw_spin_lock(&p->pi_lock);
1926         rq_lock(rq, &rf);
1927 
1928         /*
1929          * If task_rq(p) != rq, it cannot be migrated here, because we're
1930          * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because
1931          * we're holding p->pi_lock.
1932          */
1933         if (task_rq(p) == rq) {
1934                 if (is_migration_disabled(p))
1935                         goto out;
1936 
1937                 if (pending) {
1938                         if (p->migration_pending == pending)
1939                                 p->migration_pending = NULL;
1940                         complete = true;
1941 
1942                         if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask))
1943                                 goto out;
1944                 }
1945 
1946                 if (task_on_rq_queued(p))
1947                         rq = __migrate_task(rq, &rf, p, arg->dest_cpu);
1948                 else
1949                         p->wake_cpu = arg->dest_cpu;
1950 
1951                 /*
1952                  * XXX __migrate_task() can fail, at which point we might end
1953                  * up running on a dodgy CPU, AFAICT this can only happen
1954                  * during CPU hotplug, at which point we'll get pushed out
1955                  * anyway, so it's probably not a big deal.
1956                  */
1957 
1958         } else if (pending) {
1959                 /*
1960                  * This happens when we get migrated between migrate_enable()'s
1961                  * preempt_enable() and scheduling the stopper task. At that
1962                  * point we're a regular task again and not current anymore.
1963                  *
1964                  * A !PREEMPT kernel has a giant hole here, which makes it far
1965                  * more likely.
1966                  */
1967 
1968                 /*
1969                  * The task moved before the stopper got to run. We're holding
1970                  * ->pi_lock, so the allowed mask is stable - if it got
1971                  * somewhere allowed, we're done.
1972                  */
1973                 if (cpumask_test_cpu(task_cpu(p), p->cpus_ptr)) {
1974                         if (p->migration_pending == pending)
1975                                 p->migration_pending = NULL;
1976                         complete = true;
1977                         goto out;
1978                 }
1979 
1980                 /*
1981                  * When migrate_enable() hits a rq mis-match we can't reliably
1982                  * determine is_migration_disabled() and so have to chase after
1983                  * it.
1984                  */
1985                 WARN_ON_ONCE(!pending->stop_pending);
1986                 task_rq_unlock(rq, p, &rf);
1987                 stop_one_cpu_nowait(task_cpu(p), migration_cpu_stop,
1988                                     &pending->arg, &pending->stop_work);
1989                 return 0;
1990         }
1991 out:
1992         if (pending)
1993                 pending->stop_pending = false;
1994         task_rq_unlock(rq, p, &rf);
1995 
1996         if (complete)
1997                 complete_all(&pending->done);
1998 
1999         return 0;
2000 }
2001 
2002 int push_cpu_stop(void *arg)
2003 {
2004         struct rq *lowest_rq = NULL, *rq = this_rq();
2005         struct task_struct *p = arg;
2006 
2007         raw_spin_lock_irq(&p->pi_lock);
2008         raw_spin_lock(&rq->lock);
2009 
2010         if (task_rq(p) != rq)
2011                 goto out_unlock;
2012 
2013         if (is_migration_disabled(p)) {
2014                 p->migration_flags |= MDF_PUSH;
2015                 goto out_unlock;
2016         }
2017 
2018         p->migration_flags &= ~MDF_PUSH;
2019 
2020         if (p->sched_class->find_lock_rq)
2021                 lowest_rq = p->sched_class->find_lock_rq(p, rq);
2022 
2023         if (!lowest_rq)
2024                 goto out_unlock;
2025 
2026         // XXX validate p is still the highest prio task
2027         if (task_rq(p) == rq) {
2028                 deactivate_task(rq, p, 0);
2029                 set_task_cpu(p, lowest_rq->cpu);
2030                 activate_task(lowest_rq, p, 0);
2031                 resched_curr(lowest_rq);
2032         }
2033 
2034         double_unlock_balance(rq, lowest_rq);
2035 
2036 out_unlock:
2037         rq->push_busy = false;
2038         raw_spin_unlock(&rq->lock);
2039         raw_spin_unlock_irq(&p->pi_lock);
2040 
2041         put_task_struct(p);
2042         return 0;
2043 }
2044 
2045 /*
2046  * sched_class::set_cpus_allowed must do the below, but is not required to
2047  * actually call this function.
2048  */
2049 void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask, u32 flags)
2050 {
2051         if (flags & (SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) {
2052                 p->cpus_ptr = new_mask;
2053                 return;
2054         }
2055 
2056         cpumask_copy(&p->cpus_mask, new_mask);
2057         p->nr_cpus_allowed = cpumask_weight(new_mask);
2058 }
2059 
2060 static void
2061 __do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32 flags)
2062 {
2063         struct rq *rq = task_rq(p);
2064         bool queued, running;
2065 
2066         /*
2067          * This here violates the locking rules for affinity, since we're only
2068          * supposed to change these variables while holding both rq->lock and
2069          * p->pi_lock.
2070          *
2071          * HOWEVER, it magically works, because ttwu() is the only code that
2072          * accesses these variables under p->pi_lock and only does so after
2073          * smp_cond_load_acquire(&p->on_cpu, !VAL), and we're in __schedule()
2074          * before finish_task().
2075          *
2076          * XXX do further audits, this smells like something putrid.
2077          */
2078         if (flags & SCA_MIGRATE_DISABLE)
2079                 SCHED_WARN_ON(!p->on_cpu);
2080         else
2081                 lockdep_assert_held(&p->pi_lock);
2082 
2083         queued = task_on_rq_queued(p);
2084         running = task_current(rq, p);
2085 
2086         if (queued) {
2087                 /*
2088                  * Because __kthread_bind() calls this on blocked tasks without
2089                  * holding rq->lock.
2090                  */
2091                 lockdep_assert_held(&rq->lock);
2092                 dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK);
2093         }
2094         if (running)
2095                 put_prev_task(rq, p);
2096 
2097         p->sched_class->set_cpus_allowed(p, new_mask, flags);
2098 
2099         if (queued)
2100                 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
2101         if (running)
2102                 set_next_task(rq, p);
2103 }
2104 
2105 void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
2106 {
2107         __do_set_cpus_allowed(p, new_mask, 0);
2108 }
2109 
2110 /*
2111  * This function is wildly self concurrent; here be dragons.
2112  *
2113  *
2114  * When given a valid mask, __set_cpus_allowed_ptr() must block until the
2115  * designated task is enqueued on an allowed CPU. If that task is currently
2116  * running, we have to kick it out using the CPU stopper.
2117  *
2118  * Migrate-Disable comes along and tramples all over our nice sandcastle.
2119  * Consider:
2120  *
2121  *     Initial conditions: P0->cpus_mask = [0, 1]
2122  *
2123  *     P0@CPU0                  P1
2124  *
2125  *     migrate_disable();
2126  *     <preempted>
2127  *                              set_cpus_allowed_ptr(P0, [1]);
2128  *
2129  * P1 *cannot* return from this set_cpus_allowed_ptr() call until P0 executes
2130  * its outermost migrate_enable() (i.e. it exits its Migrate-Disable region).
2131  * This means we need the following scheme:
2132  *
2133  *     P0@CPU0                  P1
2134  *
2135  *     migrate_disable();
2136  *     <preempted>
2137  *                              set_cpus_allowed_ptr(P0, [1]);
2138  *                                <blocks>
2139  *     <resumes>
2140  *     migrate_enable();
2141  *       __set_cpus_allowed_ptr();
2142  *       <wakes local stopper>
2143  *                         `--> <woken on migration completion>
2144  *
2145  * Now the fun stuff: there may be several P1-like tasks, i.e. multiple
2146  * concurrent set_cpus_allowed_ptr(P0, [*]) calls. CPU affinity changes of any
2147  * task p are serialized by p->pi_lock, which we can leverage: the one that
2148  * should come into effect at the end of the Migrate-Disable region is the last
2149  * one. This means we only need to track a single cpumask (i.e. p->cpus_mask),
2150  * but we still need to properly signal those waiting tasks at the appropriate
2151  * moment.
2152  *
2153  * This is implemented using struct set_affinity_pending. The first
2154  * __set_cpus_allowed_ptr() caller within a given Migrate-Disable region will
2155  * setup an instance of that struct and install it on the targeted task_struct.
2156  * Any and all further callers will reuse that instance. Those then wait for
2157  * a completion signaled at the tail of the CPU stopper callback (1), triggered
2158  * on the end of the Migrate-Disable region (i.e. outermost migrate_enable()).
2159  *
2160  *
2161  * (1) In the cases covered above. There is one more where the completion is
2162  * signaled within affine_move_task() itself: when a subsequent affinity request
2163  * cancels the need for an active migration. Consider:
2164  *
2165  *     Initial conditions: P0->cpus_mask = [0, 1]
2166  *
2167  *     P0@CPU0            P1                             P2
2168  *
2169  *     migrate_disable();
2170  *     <preempted>
2171  *                        set_cpus_allowed_ptr(P0, [1]);
2172  *                          <blocks>
2173  *                                                       set_cpus_allowed_ptr(P0, [0, 1]);
2174  *                                                         <signal completion>
2175  *                          <awakes>
2176  *
2177  * Note that the above is safe vs a concurrent migrate_enable(), as any
2178  * pending affinity completion is preceded by an uninstallation of
2179  * p->migration_pending done with p->pi_lock held.
2180  */
2181 static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flags *rf,
2182                             int dest_cpu, unsigned int flags)
2183 {
2184         struct set_affinity_pending my_pending = { }, *pending = NULL;
2185         bool stop_pending, complete = false;
2186 
2187         /* Can the task run on the task's current CPU? If so, we're done */
2188         if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) {
2189                 struct task_struct *push_task = NULL;
2190 
2191                 if ((flags & SCA_MIGRATE_ENABLE) &&
2192                     (p->migration_flags & MDF_PUSH) && !rq->push_busy) {
2193                         rq->push_busy = true;
2194                         push_task = get_task_struct(p);
2195                 }
2196 
2197                 /*
2198                  * If there are pending waiters, but no pending stop_work,
2199                  * then complete now.
2200                  */
2201                 pending = p->migration_pending;
2202                 if (pending && !pending->stop_pending) {
2203                         p->migration_pending = NULL;
2204                         complete = true;
2205                 }
2206 
2207                 task_rq_unlock(rq, p, rf);
2208 
2209                 if (push_task) {
2210                         stop_one_cpu_nowait(rq->cpu, push_cpu_stop,
2211                                             p, &rq->push_work);
2212                 }
2213 
2214                 if (complete)
2215                         complete_all(&pending->done);
2216 
2217                 return 0;
2218         }
2219 
2220         if (!(flags & SCA_MIGRATE_ENABLE)) {
2221                 /* serialized by p->pi_lock */
2222                 if (!p->migration_pending) {
2223                         /* Install the request */
2224                         refcount_set(&my_pending.refs, 1);
2225                         init_completion(&my_pending.done);
2226                         my_pending.arg = (struct migration_arg) {
2227                                 .task = p,
2228                                 .dest_cpu = dest_cpu,
2229                                 .pending = &my_pending,
2230                         };
2231 
2232                         p->migration_pending = &my_pending;
2233                 } else {
2234                         pending = p->migration_pending;
2235                         refcount_inc(&pending->refs);
2236                         /*
2237                          * Affinity has changed, but we've already installed a
2238                          * pending. migration_cpu_stop() *must* see this, else
2239                          * we risk a completion of the pending despite having a
2240                          * task on a disallowed CPU.
2241                          *
2242                          * Serialized by p->pi_lock, so this is safe.
2243                          */
2244                         pending->arg.dest_cpu = dest_cpu;
2245                 }
2246         }
2247         pending = p->migration_pending;
2248         /*
2249          * - !MIGRATE_ENABLE:
2250          *   we'll have installed a pending if there wasn't one already.
2251          *
2252          * - MIGRATE_ENABLE:
2253          *   we're here because the current CPU isn't matching anymore,
2254          *   the only way that can happen is because of a concurrent
2255          *   set_cpus_allowed_ptr() call, which should then still be
2256          *   pending completion.
2257          *
2258          * Either way, we really should have a @pending here.
2259          */
2260         if (WARN_ON_ONCE(!pending)) {
2261                 task_rq_unlock(rq, p, rf);
2262                 return -EINVAL;
2263         }
2264 
2265         if (task_running(rq, p) || p->state == TASK_WAKING) {
2266                 /*
2267                  * MIGRATE_ENABLE gets here because 'p == current', but for
2268                  * anything else we cannot do is_migration_disabled(), punt
2269                  * and have the stopper function handle it all race-free.
2270                  */
2271                 stop_pending = pending->stop_pending;
2272                 if (!stop_pending)
2273                         pending->stop_pending = true;
2274 
2275                 if (flags & SCA_MIGRATE_ENABLE)
2276                         p->migration_flags &= ~MDF_PUSH;
2277 
2278                 task_rq_unlock(rq, p, rf);
2279 
2280                 if (!stop_pending) {
2281                         stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop,
2282                                             &pending->arg, &pending->stop_work);
2283                 }
2284 
2285                 if (flags & SCA_MIGRATE_ENABLE)
2286                         return 0;
2287         } else {
2288 
2289                 if (!is_migration_disabled(p)) {
2290                         if (task_on_rq_queued(p))
2291                                 rq = move_queued_task(rq, rf, p, dest_cpu);
2292 
2293                         if (!pending->stop_pending) {
2294                                 p->migration_pending = NULL;
2295                                 complete = true;
2296                         }
2297                 }
2298                 task_rq_unlock(rq, p, rf);
2299 
2300                 if (complete)
2301                         complete_all(&pending->done);
2302         }
2303 
2304         wait_for_completion(&pending->done);
2305 
2306         if (refcount_dec_and_test(&pending->refs))
2307                 wake_up_var(&pending->refs); /* No UaF, just an address */
2308 
2309         /*
2310          * Block the original owner of &pending until all subsequent callers
2311          * have seen the completion and decremented the refcount
2312          */
2313         wait_var_event(&my_pending.refs, !refcount_read(&my_pending.refs));
2314 
2315         /* ARGH */
2316         WARN_ON_ONCE(my_pending.stop_pending);
2317 
2318         return 0;
2319 }
2320 
2321 /*
2322  * Change a given task's CPU affinity. Migrate the thread to a
2323  * proper CPU and schedule it away if the CPU it's executing on
2324  * is removed from the allowed bitmask.
2325  *
2326  * NOTE: the caller must have a valid reference to the task, the
2327  * task must not exit() & deallocate itself prematurely. The
2328  * call is not atomic; no spinlocks may be held.
2329  */
2330 static int __set_cpus_allowed_ptr(struct task_struct *p,
2331                                   const struct cpumask *new_mask,
2332                                   u32 flags)
2333 {
2334         const struct cpumask *cpu_valid_mask = cpu_active_mask;
2335         unsigned int dest_cpu;
2336         struct rq_flags rf;
2337         struct rq *rq;
2338         int ret = 0;
2339 
2340         rq = task_rq_lock(p, &rf);
2341         update_rq_clock(rq);
2342 
2343         if (p->flags & PF_KTHREAD || is_migration_disabled(p)) {
2344                 /*
2345                  * Kernel threads are allowed on online && !active CPUs,
2346                  * however, during cpu-hot-unplug, even these might get pushed
2347                  * away if not KTHREAD_IS_PER_CPU.
2348                  *
2349                  * Specifically, migration_disabled() tasks must not fail the
2350                  * cpumask_any_and_distribute() pick below, esp. so on
2351                  * SCA_MIGRATE_ENABLE, otherwise we'll not call
2352                  * set_cpus_allowed_common() and actually reset p->cpus_ptr.
2353                  */
2354                 cpu_valid_mask = cpu_online_mask;
2355         }
2356 
2357         /*
2358          * Must re-check here, to close a race against __kthread_bind(),
2359          * sched_setaffinity() is not guaranteed to observe the flag.
2360          */
2361         if ((flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) {
2362                 ret = -EINVAL;
2363                 goto out;
2364         }
2365 
2366         if (!(flags & SCA_MIGRATE_ENABLE)) {
2367                 if (cpumask_equal(&p->cpus_mask, new_mask))
2368                         goto out;
2369 
2370                 if (WARN_ON_ONCE(p == current &&
2371                                  is_migration_disabled(p) &&
2372                                  !cpumask_test_cpu(task_cpu(p), new_mask))) {
2373                         ret = -EBUSY;
2374                         goto out;
2375                 }
2376         }
2377 
2378         /*
2379          * Picking a ~random cpu helps in cases where we are changing affinity
2380          * for groups of tasks (ie. cpuset), so that load balancing is not
2381          * immediately required to distribute the tasks within their new mask.
2382          */
2383         dest_cpu = cpumask_any_and_distribute(cpu_valid_mask, new_mask);
2384         if (dest_cpu >= nr_cpu_ids) {
2385                 ret = -EINVAL;
2386                 goto out;
2387         }
2388 
2389         __do_set_cpus_allowed(p, new_mask, flags);
2390 
2391         return affine_move_task(rq, p, &rf, dest_cpu, flags);
2392 
2393 out:
2394         task_rq_unlock(rq, p, &rf);
2395 
2396         return ret;
2397 }
2398 
2399 int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
2400 {
2401         return __set_cpus_allowed_ptr(p, new_mask, 0);
2402 }
2403 EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
2404 
2405 void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
2406 {
2407 #ifdef CONFIG_SCHED_DEBUG
2408         /*
2409          * We should never call set_task_cpu() on a blocked task,
2410          * ttwu() will sort out the placement.
2411          */
2412         WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
2413                         !p->on_rq);
2414 
2415         /*
2416          * Migrating fair class task must have p->on_rq = TASK_ON_RQ_MIGRATING,
2417          * because schedstat_wait_{start,end} rebase migrating task's wait_start
2418          * time relying on p->on_rq.
2419          */
2420         WARN_ON_ONCE(p->state == TASK_RUNNING &&
2421                      p->sched_class == &fair_sched_class &&
2422                      (p->on_rq && !task_on_rq_migrating(p)));
2423 
2424 #ifdef CONFIG_LOCKDEP
2425         /*
2426          * The caller should hold either p->pi_lock or rq->lock, when changing
2427          * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks.
2428          *
2429          * sched_move_task() holds both and thus holding either pins the cgroup,
2430          * see task_group().
2431          *
2432          * Furthermore, all task_rq users should acquire both locks, see
2433          * task_rq_lock().
2434          */
2435         WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) ||
2436                                       lockdep_is_held(&task_rq(p)->lock)));
2437 #endif
2438         /*
2439          * Clearly, migrating tasks to offline CPUs is a fairly daft thing.
2440          */
2441         WARN_ON_ONCE(!cpu_online(new_cpu));
2442 
2443         WARN_ON_ONCE(is_migration_disabled(p));
2444 #endif
2445 
2446         trace_sched_migrate_task(p, new_cpu);
2447 
2448         if (task_cpu(p) != new_cpu) {
2449                 if (p->sched_class->migrate_task_rq)
2450                         p->sched_class->migrate_task_rq(p, new_cpu);
2451                 p->se.nr_migrations++;
2452                 rseq_migrate(p);
2453                 perf_event_task_migrate(p);
2454         }
2455 
2456         __set_task_cpu(p, new_cpu);
2457 }
2458 
2459 #ifdef CONFIG_NUMA_BALANCING
2460 static void __migrate_swap_task(struct task_struct *p, int cpu)
2461 {
2462         if (task_on_rq_queued(p)) {
2463                 struct rq *src_rq, *dst_rq;
2464                 struct rq_flags srf, drf;
2465 
2466                 src_rq = task_rq(p);
2467                 dst_rq = cpu_rq(cpu);
2468 
2469                 rq_pin_lock(src_rq, &srf);
2470                 rq_pin_lock(dst_rq, &drf);
2471 
2472                 deactivate_task(src_rq, p, 0);
2473                 set_task_cpu(p, cpu);
2474                 activate_task(dst_rq, p, 0);
2475                 check_preempt_curr(dst_rq, p, 0);
2476 
2477                 rq_unpin_lock(dst_rq, &drf);
2478                 rq_unpin_lock(src_rq, &srf);
2479 
2480         } else {
2481                 /*
2482                  * Task isn't running anymore; make it appear like we migrated
2483                  * it before it went to sleep. This means on wakeup we make the
2484                  * previous CPU our target instead of where it really is.
2485                  */
2486                 p->wake_cpu = cpu;
2487         }
2488 }
2489 
2490 struct migration_swap_arg {
2491         struct task_struct *src_task, *dst_task;
2492         int src_cpu, dst_cpu;
2493 };
2494 
2495 static int migrate_swap_stop(void *data)
2496 {
2497         struct migration_swap_arg *arg = data;
2498         struct rq *src_rq, *dst_rq;
2499         int ret = -EAGAIN;
2500 
2501         if (!cpu_active(arg->src_cpu) || !cpu_active(arg->dst_cpu))
2502                 return -EAGAIN;
2503 
2504         src_rq = cpu_rq(arg->src_cpu);
2505         dst_rq = cpu_rq(arg->dst_cpu);
2506 
2507         double_raw_lock(&arg->src_task->pi_lock,
2508                         &arg->dst_task->pi_lock);
2509         double_rq_lock(src_rq, dst_rq);
2510 
2511         if (task_cpu(arg->dst_task) != arg->dst_cpu)
2512                 goto unlock;
2513 
2514         if (task_cpu(arg->src_task) != arg->src_cpu)
2515                 goto unlock;
2516 
2517         if (!cpumask_test_cpu(arg->dst_cpu, arg->src_task->cpus_ptr))
2518                 goto unlock;
2519 
2520         if (!cpumask_test_cpu(arg->src_cpu, arg->dst_task->cpus_ptr))
2521                 goto unlock;
2522 
2523         __migrate_swap_task(arg->src_task, arg->dst_cpu);
2524         __migrate_swap_task(arg->dst_task, arg->src_cpu);
2525 
2526         ret = 0;
2527 
2528 unlock:
2529         double_rq_unlock(src_rq, dst_rq);
2530         raw_spin_unlock(&arg->dst_task->pi_lock);
2531         raw_spin_unlock(&arg->src_task->pi_lock);
2532 
2533         return ret;
2534 }
2535 
2536 /*
2537  * Cross migrate two tasks
2538  */
2539 int migrate_swap(struct task_struct *cur, struct task_struct *p,
2540                 int target_cpu, int curr_cpu)
2541 {
2542         struct migration_swap_arg arg;
2543         int ret = -EINVAL;
2544 
2545         arg = (struct migration_swap_arg){
2546                 .src_task = cur,
2547                 .src_cpu = curr_cpu,
2548                 .dst_task = p,
2549                 .dst_cpu = target_cpu,
2550         };
2551 
2552         if (arg.src_cpu == arg.dst_cpu)
2553                 goto out;
2554 
2555         /*
2556          * These three tests are all lockless; this is OK since all of them
2557          * will be re-checked with proper locks held further down the line.
2558          */
2559         if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu))
2560                 goto out;
2561 
2562         if (!cpumask_test_cpu(arg.dst_cpu, arg.src_task->cpus_ptr))
2563                 goto out;
2564 
2565         if (!cpumask_test_cpu(arg.src_cpu, arg.dst_task->cpus_ptr))
2566                 goto out;
2567 
2568         trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu);
2569         ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg);
2570 
2571 out:
2572         return ret;
2573 }
2574 #endif /* CONFIG_NUMA_BALANCING */
2575 
2576 /*
2577  * wait_task_inactive - wait for a thread to unschedule.
2578  *
2579  * If @match_state is nonzero, it's the @p->state value just checked and
2580  * not expected to change.  If it changes, i.e. @p might have woken up,
2581  * then return zero.  When we succeed in waiting for @p to be off its CPU,
2582  * we return a positive number (its total switch count).  If a second call
2583  * a short while later returns the same number, the caller can be sure that
2584  * @p has remained unscheduled the whole time.
2585  *
2586  * The caller must ensure that the task *will* unschedule sometime soon,
2587  * else this function might spin for a *long* time. This function can't
2588  * be called with interrupts off, or it may introduce deadlock with
2589  * smp_call_function() if an IPI is sent by the same process we are
2590  * waiting to become inactive.
2591  */
2592 unsigned long wait_task_inactive(struct task_struct *p, long match_state)
2593 {
2594         int running, queued;
2595         struct rq_flags rf;
2596         unsigned long ncsw;
2597         struct rq *rq;
2598 
2599         for (;;) {
2600                 /*
2601                  * We do the initial early heuristics without holding
2602                  * any task-queue locks at all. We'll only try to get
2603                  * the runqueue lock when things look like they will
2604                  * work out!
2605                  */
2606                 rq = task_rq(p);
2607 
2608                 /*
2609                  * If the task is actively running on another CPU
2610                  * still, just relax and busy-wait without holding
2611                  * any locks.
2612                  *
2613                  * NOTE! Since we don't hold any locks, it's not
2614                  * even sure that "rq" stays as the right runqueue!
2615                  * But we don't care, since "task_running()" will
2616                  * return false if the runqueue has changed and p
2617                  * is actually now running somewhere else!
2618                  */
2619                 while (task_running(rq, p)) {
2620                         if (match_state && unlikely(p->state != match_state))
2621                                 return 0;
2622                         cpu_relax();
2623                 }
2624 
2625                 /*
2626                  * Ok, time to look more closely! We need the rq
2627                  * lock now, to be *sure*. If we're wrong, we'll
2628                  * just go back and repeat.
2629                  */
2630                 rq = task_rq_lock(p, &rf);
2631                 trace_sched_wait_task(p);
2632                 running = task_running(rq, p);
2633                 queued = task_on_rq_queued(p);
2634                 ncsw = 0;
2635                 if (!match_state || p->state == match_state)
2636                         ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
2637                 task_rq_unlock(rq, p, &rf);
2638 
2639                 /*
2640                  * If it changed from the expected state, bail out now.
2641                  */
2642                 if (unlikely(!ncsw))
2643                         break;
2644 
2645                 /*
2646                  * Was it really running after all now that we
2647                  * checked with the proper locks actually held?
2648                  *
2649                  * Oops. Go back and try again..
2650                  */
2651                 if (unlikely(running)) {
2652                         cpu_relax();
2653                         continue;
2654                 }
2655 
2656                 /*
2657                  * It's not enough that it's not actively running,
2658                  * it must be off the runqueue _entirely_, and not
2659                  * preempted!
2660                  *
2661                  * So if it was still runnable (but just not actively
2662                  * running right now), it's preempted, and we should
2663                  * yield - it could be a while.
2664                  */
2665                 if (unlikely(queued)) {
2666                         ktime_t to = NSEC_PER_SEC / HZ;
2667 
2668                         set_current_state(TASK_UNINTERRUPTIBLE);
2669                         schedule_hrtimeout(&to, HRTIMER_MODE_REL);
2670                         continue;
2671                 }
2672 
2673                 /*
2674                  * Ahh, all good. It wasn't running, and it wasn't
2675                  * runnable, which means that it will never become
2676                  * running in the future either. We're all done!
2677                  */
2678                 break;
2679         }
2680 
2681         return ncsw;
2682 }
2683 
2684 /***
2685  * kick_process - kick a running thread to enter/exit the kernel
2686  * @p: the to-be-kicked thread
2687  *
2688  * Cause a process which is running on another CPU to enter
2689  * kernel-mode, without any delay. (to get signals handled.)
2690  *
2691  * NOTE: this function doesn't have to take the runqueue lock,
2692  * because all it wants to ensure is that the remote task enters
2693  * the kernel. If the IPI races and the task has been migrated
2694  * to another CPU then no harm is done and the purpose has been
2695  * achieved as well.
2696  */
2697 void kick_process(struct task_struct *p)
2698 {
2699         int cpu;
2700 
2701         preempt_disable();
2702         cpu = task_cpu(p);
2703         if ((cpu != smp_processor_id()) && task_curr(p))
2704                 smp_send_reschedule(cpu);
2705         preempt_enable();
2706 }
2707 EXPORT_SYMBOL_GPL(kick_process);
2708 
2709 /*
2710  * ->cpus_ptr is protected by both rq->lock and p->pi_lock
2711  *
2712  * A few notes on cpu_active vs cpu_online:
2713  *
2714  *  - cpu_active must be a subset of cpu_online
2715  *
2716  *  - on CPU-up we allow per-CPU kthreads on the online && !active CPU,
2717  *    see __set_cpus_allowed_ptr(). At this point the newly online
2718  *    CPU isn't yet part of the sched domains, and balancing will not
2719  *    see it.
2720  *
2721  *  - on CPU-down we clear cpu_active() to mask the sched domains and
2722  *    avoid the load balancer to place new tasks on the to be removed
2723  *    CPU. Existing tasks will remain running there and will be taken
2724  *    off.
2725  *
2726  * This means that fallback selection must not select !active CPUs.
2727  * And can assume that any active CPU must be online. Conversely
2728  * select_task_rq() below may allow selection of !active CPUs in order
2729  * to satisfy the above rules.
2730  */
2731 static int select_fallback_rq(int cpu, struct task_struct *p)
2732 {
2733         int nid = cpu_to_node(cpu);
2734         const struct cpumask *nodemask = NULL;
2735         enum { cpuset, possible, fail } state = cpuset;
2736         int dest_cpu;
2737 
2738         /*
2739          * If the node that the CPU is on has been offlined, cpu_to_node()
2740          * will return -1. There is no CPU on the node, and we should
2741          * select the CPU on the other node.
2742          */
2743         if (nid != -1) {
2744                 nodemask = cpumask_of_node(nid);
2745 
2746                 /* Look for allowed, online CPU in same node. */
2747                 for_each_cpu(dest_cpu, nodemask) {
2748                         if (!cpu_active(dest_cpu))
2749                                 continue;
2750                         if (cpumask_test_cpu(dest_cpu, p->cpus_ptr))
2751                                 return dest_cpu;
2752                 }
2753         }
2754 
2755         for (;;) {
2756                 /* Any allowed, online CPU? */
2757                 for_each_cpu(dest_cpu, p->cpus_ptr) {
2758                         if (!is_cpu_allowed(p, dest_cpu))
2759                                 continue;
2760 
2761                         goto out;
2762                 }
2763 
2764                 /* No more Mr. Nice Guy. */
2765                 switch (state) {
2766                 case cpuset:
2767                         if (IS_ENABLED(CONFIG_CPUSETS)) {
2768                                 cpuset_cpus_allowed_fallback(p);
2769                                 state = possible;
2770                                 break;
2771                         }
2772                         fallthrough;
2773                 case possible:
2774                         /*
2775                          * XXX When called from select_task_rq() we only
2776                          * hold p->pi_lock and again violate locking order.
2777                          *
2778                          * More yuck to audit.
2779                          */
2780                         do_set_cpus_allowed(p, cpu_possible_mask);
2781                         state = fail;
2782                         break;
2783 
2784                 case fail:
2785                         BUG();
2786                         break;
2787                 }
2788         }
2789 
2790 out:
2791         if (state != cpuset) {
2792                 /*
2793                  * Don't tell them about moving exiting tasks or
2794                  * kernel threads (both mm NULL), since they never
2795                  * leave kernel.
2796                  */
2797                 if (p->mm && printk_ratelimit()) {
2798                         printk_deferred("process %d (%s) no longer affine to cpu%d\n",
2799                                         task_pid_nr(p), p->comm, cpu);
2800                 }
2801         }
2802 
2803         return dest_cpu;
2804 }
2805 
2806 /*
2807  * The caller (fork, wakeup) owns p->pi_lock, ->cpus_ptr is stable.
2808  */
2809 static inline
2810 int select_task_rq(struct task_struct *p, int cpu, int wake_flags)
2811 {
2812         lockdep_assert_held(&p->pi_lock);
2813 
2814         if (p->nr_cpus_allowed > 1 && !is_migration_disabled(p))
2815                 cpu = p->sched_class->select_task_rq(p, cpu, wake_flags);
2816         else
2817                 cpu = cpumask_any(p->cpus_ptr);
2818 
2819         /*
2820          * In order not to call set_task_cpu() on a blocking task we need
2821          * to rely on ttwu() to place the task on a valid ->cpus_ptr
2822          * CPU.
2823          *
2824          * Since this is common to all placement strategies, this lives here.
2825          *
2826          * [ this allows ->select_task() to simply return task_cpu(p) and
2827          *   not worry about this generic constraint ]
2828          */
2829         if (unlikely(!is_cpu_allowed(p, cpu)))
2830                 cpu = select_fallback_rq(task_cpu(p), p);
2831 
2832         return cpu;
2833 }
2834 
2835 void sched_set_stop_task(int cpu, struct task_struct *stop)
2836 {
2837         static struct lock_class_key stop_pi_lock;
2838         struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
2839         struct task_struct *old_stop = cpu_rq(cpu)->stop;
2840 
2841         if (stop) {
2842                 /*
2843                  * Make it appear like a SCHED_FIFO task, its something
2844                  * userspace knows about and won't get confused about.
2845                  *
2846                  * Also, it will make PI more or less work without too
2847                  * much confusion -- but then, stop work should not
2848                  * rely on PI working anyway.
2849                  */
2850                 sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);
2851 
2852                 stop->sched_class = &stop_sched_class;
2853 
2854                 /*
2855                  * The PI code calls rt_mutex_setprio() with ->pi_lock held to
2856                  * adjust the effective priority of a task. As a result,
2857                  * rt_mutex_setprio() can trigger (RT) balancing operations,
2858                  * which can then trigger wakeups of the stop thread to push
2859                  * around the current task.
2860                  *
2861                  * The stop task itself will never be part of the PI-chain, it
2862                  * never blocks, therefore that ->pi_lock recursion is safe.
2863                  * Tell lockdep about this by placing the stop->pi_lock in its
2864                  * own class.
2865                  */
2866                 lockdep_set_class(&stop->pi_lock, &stop_pi_lock);
2867         }
2868 
2869         cpu_rq(cpu)->stop = stop;
2870 
2871         if (old_stop) {
2872                 /*
2873                  * Reset it back to a normal scheduling class so that
2874                  * it can die in pieces.
2875                  */
2876                 old_stop->sched_class = &rt_sched_class;
2877         }
2878 }
2879 
2880 #else /* CONFIG_SMP */
2881 
2882 static inline int __set_cpus_allowed_ptr(struct task_struct *p,
2883                                          const struct cpumask *new_mask,
2884                                          u32 flags)
2885 {
2886         return set_cpus_allowed_ptr(p, new_mask);
2887 }
2888 
2889 static inline void migrate_disable_switch(struct rq *rq, struct task_struct *p) { }
2890 
2891 static inline bool rq_has_pinned_tasks(struct rq *rq)
2892 {
2893         return false;
2894 }
2895 
2896 #endif /* !CONFIG_SMP */
2897 
2898 static void
2899 ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
2900 {
2901         struct rq *rq;
2902 
2903         if (!schedstat_enabled())
2904                 return;
2905 
2906         rq = this_rq();
2907 
2908 #ifdef CONFIG_SMP
2909         if (cpu == rq->cpu) {
2910                 __schedstat_inc(rq->ttwu_local);
2911                 __schedstat_inc(p->se.statistics.nr_wakeups_local);
2912         } else {
2913                 struct sched_domain *sd;
2914 
2915                 __schedstat_inc(p->se.statistics.nr_wakeups_remote);
2916                 rcu_read_lock();
2917                 for_each_domain(rq->cpu, sd) {
2918                         if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
2919                                 __schedstat_inc(sd->ttwu_wake_remote);
2920                                 break;
2921                         }
2922                 }
2923                 rcu_read_unlock();
2924         }
2925 
2926         if (wake_flags & WF_MIGRATED)
2927                 __schedstat_inc(p->se.statistics.nr_wakeups_migrate);
2928 #endif /* CONFIG_SMP */
2929 
2930         __schedstat_inc(rq->ttwu_count);
2931         __schedstat_inc(p->se.statistics.nr_wakeups);
2932 
2933         if (wake_flags & WF_SYNC)
2934                 __schedstat_inc(p->se.statistics.nr_wakeups_sync);
2935 }
2936 
2937 /*
2938  * Mark the task runnable and perform wakeup-preemption.
2939  */
2940 static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags,
2941                            struct rq_flags *rf)
2942 {
2943         check_preempt_curr(rq, p, wake_flags);
2944         p->state = TASK_RUNNING;
2945         trace_sched_wakeup(p);
2946 
2947 #ifdef CONFIG_SMP
2948         if (p->sched_class->task_woken) {
2949                 /*
2950                  * Our task @p is fully woken up and running; so it's safe to
2951                  * drop the rq->lock, hereafter rq is only used for statistics.
2952                  */
2953                 rq_unpin_lock(rq, rf);
2954                 p->sched_class->task_woken(rq, p);
2955                 rq_repin_lock(rq, rf);
2956         }
2957 
2958         if (rq->idle_stamp) {
2959                 u64 delta = rq_clock(rq) - rq->idle_stamp;
2960                 u64 max = 2*rq->max_idle_balance_cost;
2961 
2962                 update_avg(&rq->avg_idle, delta);
2963 
2964                 if (rq->avg_idle > max)
2965                         rq->avg_idle = max;
2966 
2967                 rq->idle_stamp = 0;
2968         }
2969 #endif
2970 }
2971 
2972 static void
2973 ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
2974                  struct rq_flags *rf)
2975 {
2976         int en_flags = ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK;
2977 
2978         lockdep_assert_held(&rq->lock);
2979 
2980         if (p->sched_contributes_to_load)
2981                 rq->nr_uninterruptible--;
2982 
2983 #ifdef CONFIG_SMP
2984         if (wake_flags & WF_MIGRATED)
2985                 en_flags |= ENQUEUE_MIGRATED;
2986         else
2987 #endif
2988         if (p->in_iowait) {
2989                 delayacct_blkio_end(p);
2990                 atomic_dec(&task_rq(p)->nr_iowait);
2991         }
2992 
2993         activate_task(rq, p, en_flags);
2994         ttwu_do_wakeup(rq, p, wake_flags, rf);
2995 }
2996 
2997 /*
2998  * Consider @p being inside a wait loop:
2999  *
3000  *   for (;;) {
3001  *      set_current_state(TASK_UNINTERRUPTIBLE);
3002  *
3003  *      if (CONDITION)
3004  *         break;
3005  *
3006  *      schedule();
3007  *   }
3008  *   __set_current_state(TASK_RUNNING);
3009  *
3010  * between set_current_state() and schedule(). In this case @p is still
3011  * runnable, so all that needs doing is change p->state back to TASK_RUNNING in
3012  * an atomic manner.
3013  *
3014  * By taking task_rq(p)->lock we serialize against schedule(), if @p->on_rq
3015  * then schedule() must still happen and p->state can be changed to
3016  * TASK_RUNNING. Otherwise we lost the race, schedule() has happened, and we
3017  * need to do a full wakeup with enqueue.
3018  *
3019  * Returns: %true when the wakeup is done,
3020  *          %false otherwise.
3021  */
3022 static int ttwu_runnable(struct task_struct *p, int wake_flags)
3023 {
3024         struct rq_flags rf;
3025         struct rq *rq;
3026         int ret = 0;
3027 
3028         rq = __task_rq_lock(p, &rf);
3029         if (task_on_rq_queued(p)) {
3030                 /* check_preempt_curr() may use rq clock */
3031                 update_rq_clock(rq);
3032                 ttwu_do_wakeup(rq, p, wake_flags, &rf);
3033                 ret = 1;
3034         }
3035         __task_rq_unlock(rq, &rf);
3036 
3037         return ret;
3038 }
3039 
3040 #ifdef CONFIG_SMP
3041 void sched_ttwu_pending(void *arg)
3042 {
3043         struct llist_node *llist = arg;
3044         struct rq *rq = this_rq();
3045         struct task_struct *p, *t;
3046         struct rq_flags rf;
3047 
3048         if (!llist)
3049                 return;
3050 
3051         /*
3052          * rq::ttwu_pending racy indication of out-standing wakeups.
3053          * Races such that false-negatives are possible, since they
3054          * are shorter lived that false-positives would be.
3055          */
3056         WRITE_ONCE(rq->ttwu_pending, 0);
3057 
3058         rq_lock_irqsave(rq, &rf);
3059         update_rq_clock(rq);
3060 
3061         llist_for_each_entry_safe(p, t, llist, wake_entry.llist) {
3062                 if (WARN_ON_ONCE(p->on_cpu))
3063                         smp_cond_load_acquire(&p->on_cpu, !VAL);
3064 
3065                 if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq)))
3066                         set_task_cpu(p, cpu_of(rq));
3067 
3068                 ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf);
3069         }
3070 
3071         rq_unlock_irqrestore(rq, &rf);
3072 }
3073 
3074 void send_call_function_single_ipi(int cpu)
3075 {
3076         struct rq *rq = cpu_rq(cpu);
3077 
3078         if (!set_nr_if_polling(rq->idle))
3079                 arch_send_call_function_single_ipi(cpu);
3080         else
3081                 trace_sched_wake_idle_without_ipi(cpu);
3082 }
3083 
3084 /*
3085  * Queue a task on the target CPUs wake_list and wake the CPU via IPI if
3086  * necessary. The wakee CPU on receipt of the IPI will queue the task
3087  * via sched_ttwu_wakeup() for activation so the wakee incurs the cost
3088  * of the wakeup instead of the waker.
3089  */
3090 static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags)
3091 {
3092         struct rq *rq = cpu_rq(cpu);
3093 
3094         p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
3095 
3096         WRITE_ONCE(rq->ttwu_pending, 1);
3097         __smp_call_single_queue(cpu, &p->wake_entry.llist);
3098 }
3099 
3100 void wake_up_if_idle(int cpu)
3101 {
3102         struct rq *rq = cpu_rq(cpu);
3103         struct rq_flags rf;
3104 
3105         rcu_read_lock();
3106 
3107         if (!is_idle_task(rcu_dereference(rq->curr)))
3108                 goto out;
3109 
3110         if (set_nr_if_polling(rq->idle)) {
3111                 trace_sched_wake_idle_without_ipi(cpu);
3112         } else {
3113                 rq_lock_irqsave(rq, &rf);
3114                 if (is_idle_task(rq->curr))
3115                         smp_send_reschedule(cpu);
3116                 /* Else CPU is not idle, do nothing here: */
3117                 rq_unlock_irqrestore(rq, &rf);
3118         }
3119 
3120 out:
3121         rcu_read_unlock();
3122 }
3123 
3124 bool cpus_share_cache(int this_cpu, int that_cpu)
3125 {
3126         return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
3127 }
3128 
3129 static inline bool ttwu_queue_cond(int cpu, int wake_flags)
3130 {
3131         /*
3132          * Do not complicate things with the async wake_list while the CPU is
3133          * in hotplug state.
3134          */
3135         if (!cpu_active(cpu))
3136                 return false;
3137 
3138         /*
3139          * If the CPU does not share cache, then queue the task on the
3140          * remote rqs wakelist to avoid accessing remote data.
3141          */
3142         if (!cpus_share_cache(smp_processor_id(), cpu))
3143                 return true;
3144 
3145         /*
3146          * If the task is descheduling and the only running task on the
3147          * CPU then use the wakelist to offload the task activation to
3148          * the soon-to-be-idle CPU as the current CPU is likely busy.
3149          * nr_running is checked to avoid unnecessary task stacking.
3150          */
3151         if ((wake_flags & WF_ON_CPU) && cpu_rq(cpu)->nr_running <= 1)
3152                 return true;
3153 
3154         return false;
3155 }
3156 
3157 static bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags)
3158 {
3159         if (sched_feat(TTWU_QUEUE) && ttwu_queue_cond(cpu, wake_flags)) {
3160                 if (WARN_ON_ONCE(cpu == smp_processor_id()))
3161                         return false;
3162 
3163                 sched_clock_cpu(cpu); /* Sync clocks across CPUs */
3164                 __ttwu_queue_wakelist(p, cpu, wake_flags);
3165                 return true;
3166         }
3167 
3168         return false;
3169 }
3170 
3171 #else /* !CONFIG_SMP */
3172 
3173 static inline bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags)
3174 {
3175         return false;
3176 }
3177 
3178 #endif /* CONFIG_SMP */
3179 
3180 static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags)
3181 {
3182         struct rq *rq = cpu_rq(cpu);
3183         struct rq_flags rf;
3184 
3185         if (ttwu_queue_wakelist(p, cpu, wake_flags))
3186                 return;
3187 
3188         rq_lock(rq, &rf);
3189         update_rq_clock(rq);
3190         ttwu_do_activate(rq, p, wake_flags, &rf);
3191         rq_unlock(rq, &rf);
3192 }
3193 
3194 /*
3195  * Notes on Program-Order guarantees on SMP systems.
3196  *
3197  *  MIGRATION
3198  *
3199  * The basic program-order guarantee on SMP systems is that when a task [t]
3200  * migrates, all its activity on its old CPU [c0] happens-before any subsequent
3201  * execution on its new CPU [c1].
3202  *
3203  * For migration (of runnable tasks) this is provided by the following means:
3204  *
3205  *  A) UNLOCK of the rq(c0)->lock scheduling out task t
3206  *  B) migration for t is required to synchronize *both* rq(c0)->lock and
3207  *     rq(c1)->lock (if not at the same time, then in that order).
3208  *  C) LOCK of the rq(c1)->lock scheduling in task
3209  *
3210  * Release/acquire chaining guarantees that B happens after A and C after B.
3211  * Note: the CPU doing B need not be c0 or c1
3212  *
3213  * Example:
3214  *
3215  *   CPU0            CPU1            CPU2
3216  *
3217  *   LOCK rq(0)->lock
3218  *   sched-out X
3219  *   sched-in Y
3220  *   UNLOCK rq(0)->lock
3221  *
3222  *                                   LOCK rq(0)->lock // orders against CPU0
3223  *                                   dequeue X
3224  *                                   UNLOCK rq(0)->lock
3225  *
3226  *                                   LOCK rq(1)->lock
3227  *                                   enqueue X
3228  *                                   UNLOCK rq(1)->lock
3229  *
3230  *                   LOCK rq(1)->lock // orders against CPU2
3231  *                   sched-out Z
3232  *                   sched-in X
3233  *                   UNLOCK rq(1)->lock
3234  *
3235  *
3236  *  BLOCKING -- aka. SLEEP + WAKEUP
3237  *
3238  * For blocking we (obviously) need to provide the same guarantee as for
3239  * migration. However the means are completely different as there is no lock
3240  * chain to provide order. Instead we do:
3241  *
3242  *   1) smp_store_release(X->on_cpu, 0)   -- finish_task()
3243  *   2) smp_cond_load_acquire(!X->on_cpu) -- try_to_wake_up()
3244  *
3245  * Example:
3246  *
3247  *   CPU0 (schedule)  CPU1 (try_to_wake_up) CPU2 (schedule)
3248  *
3249  *   LOCK rq(0)->lock LOCK X->pi_lock
3250  *   dequeue X
3251  *   sched-out X
3252  *   smp_store_release(X->on_cpu, 0);
3253  *
3254  *                    smp_cond_load_acquire(&X->on_cpu, !VAL);
3255  *                    X->state = WAKING
3256  *                    set_task_cpu(X,2)
3257  *
3258  *                    LOCK rq(2)->lock
3259  *                    enqueue X
3260  *                    X->state = RUNNING
3261  *                    UNLOCK rq(2)->lock
3262  *
3263  *                                          LOCK rq(2)->lock // orders against CPU1
3264  *                                          sched-out Z
3265  *                                          sched-in X
3266  *                                          UNLOCK rq(2)->lock
3267  *
3268  *                    UNLOCK X->pi_lock
3269  *   UNLOCK rq(0)->lock
3270  *
3271  *
3272  * However, for wakeups there is a second guarantee we must provide, namely we
3273  * must ensure that CONDITION=1 done by the caller can not be reordered with
3274  * accesses to the task state; see try_to_wake_up() and set_current_state().
3275  */
3276 
3277 /**
3278  * try_to_wake_up - wake up a thread
3279  * @p: the thread to be awakened
3280  * @state: the mask of task states that can be woken
3281  * @wake_flags: wake modifier flags (WF_*)
3282  *
3283  * Conceptually does:
3284  *
3285  *   If (@state & @p->state) @p->state = TASK_RUNNING.
3286  *
3287  * If the task was not queued/runnable, also place it back on a runqueue.
3288  *
3289  * This function is atomic against schedule() which would dequeue the task.
3290  *
3291  * It issues a full memory barrier before accessing @p->state, see the comment
3292  * with set_current_state().
3293  *
3294  * Uses p->pi_lock to serialize against concurrent wake-ups.
3295  *
3296  * Relies on p->pi_lock stabilizing:
3297  *  - p->sched_class
3298  *  - p->cpus_ptr
3299  *  - p->sched_task_group
3300  * in order to do migration, see its use of select_task_rq()/set_task_cpu().
3301  *
3302  * Tries really hard to only take one task_rq(p)->lock for performance.
3303  * Takes rq->lock in:
3304  *  - ttwu_runnable()    -- old rq, unavoidable, see comment there;
3305  *  - ttwu_queue()       -- new rq, for enqueue of the task;
3306  *  - psi_ttwu_dequeue() -- much sadness :-( accounting will kill us.
3307  *
3308  * As a consequence we race really badly with just about everything. See the
3309  * many memory barriers and their comments for details.
3310  *
3311  * Return: %true if @p->state changes (an actual wakeup was done),
3312  *         %false otherwise.
3313  */
3314 static int
3315 try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
3316 {
3317         unsigned long flags;
3318         int cpu, success = 0;
3319 
3320         preempt_disable();
3321         if (p == current) {
3322                 /*
3323                  * We're waking current, this means 'p->on_rq' and 'task_cpu(p)
3324                  * == smp_processor_id()'. Together this means we can special
3325                  * case the whole 'p->on_rq && ttwu_runnable()' case below
3326                  * without taking any locks.
3327                  *
3328                  * In particular:
3329                  *  - we rely on Program-Order guarantees for all the ordering,
3330                  *  - we're serialized against set_special_state() by virtue of
3331                  *    it disabling IRQs (this allows not taking ->pi_lock).
3332                  */
3333                 if (!(p->state & state))
3334                         goto out;
3335 
3336                 success = 1;
3337                 trace_sched_waking(p);
3338                 p->state = TASK_RUNNING;
3339                 trace_sched_wakeup(p);
3340                 goto out;
3341         }
3342 
3343         /*
3344          * If we are going to wake up a thread waiting for CONDITION we
3345          * need to ensure that CONDITION=1 done by the caller can not be
3346          * reordered with p->state check below. This pairs with smp_store_mb()
3347          * in set_current_state() that the waiting thread does.
3348          */
3349         raw_spin_lock_irqsave(&p->pi_lock, flags);
3350         smp_mb__after_spinlock();
3351         if (!(p->state & state))
3352                 goto unlock;
3353 
3354         trace_sched_waking(p);
3355 
3356         /* We're going to change ->state: */
3357         success = 1;
3358 
3359         /*
3360          * Ensure we load p->on_rq _after_ p->state, otherwise it would
3361          * be possible to, falsely, observe p->on_rq == 0 and get stuck
3362          * in smp_cond_load_acquire() below.
3363          *
3364          * sched_ttwu_pending()                 try_to_wake_up()
3365          *   STORE p->on_rq = 1                   LOAD p->state
3366          *   UNLOCK rq->lock
3367          *
3368          * __schedule() (switch to task 'p')
3369          *   LOCK rq->lock                        smp_rmb();
3370          *   smp_mb__after_spinlock();
3371          *   UNLOCK rq->lock
3372          *
3373          * [task p]
3374          *   STORE p->state = UNINTERRUPTIBLE     LOAD p->on_rq
3375          *
3376          * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in
3377          * __schedule().  See the comment for smp_mb__after_spinlock().
3378          *
3379          * A similar smb_rmb() lives in try_invoke_on_locked_down_task().
3380          */
3381         smp_rmb();
3382         if (READ_ONCE(p->on_rq) && ttwu_runnable(p, wake_flags))
3383                 goto unlock;
3384 
3385 #ifdef CONFIG_SMP
3386         /*
3387          * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be
3388          * possible to, falsely, observe p->on_cpu == 0.
3389          *
3390          * One must be running (->on_cpu == 1) in order to remove oneself
3391          * from the runqueue.
3392          *
3393          * __schedule() (switch to task 'p')    try_to_wake_up()
3394          *   STORE p->on_cpu = 1                  LOAD p->on_rq
3395          *   UNLOCK rq->lock
3396          *
3397          * __schedule() (put 'p' to sleep)
3398          *   LOCK rq->lock                        smp_rmb();
3399          *   smp_mb__after_spinlock();
3400          *   STORE p->on_rq = 0                   LOAD p->on_cpu
3401          *
3402          * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in
3403          * __schedule().  See the comment for smp_mb__after_spinlock().
3404          *
3405          * Form a control-dep-acquire with p->on_rq == 0 above, to ensure
3406          * schedule()'s deactivate_task() has 'happened' and p will no longer
3407          * care about it's own p->state. See the comment in __schedule().
3408          */
3409         smp_acquire__after_ctrl_dep();
3410 
3411         /*
3412          * We're doing the wakeup (@success == 1), they did a dequeue (p->on_rq
3413          * == 0), which means we need to do an enqueue, change p->state to
3414          * TASK_WAKING such that we can unlock p->pi_lock before doing the
3415          * enqueue, such as ttwu_queue_wakelist().
3416          */
3417         p->state = TASK_WAKING;
3418 
3419         /*
3420          * If the owning (remote) CPU is still in the middle of schedule() with
3421          * this task as prev, considering queueing p on the remote CPUs wake_list
3422          * which potentially sends an IPI instead of spinning on p->on_cpu to
3423          * let the waker make forward progress. This is safe because IRQs are
3424          * disabled and the IPI will deliver after on_cpu is cleared.
3425          *
3426          * Ensure we load task_cpu(p) after p->on_cpu:
3427          *
3428          * set_task_cpu(p, cpu);
3429          *   STORE p->cpu = @cpu
3430          * __schedule() (switch to task 'p')
3431          *   LOCK rq->lock
3432          *   smp_mb__after_spin_lock()          smp_cond_load_acquire(&p->on_cpu)
3433          *   STORE p->on_cpu = 1                LOAD p->cpu
3434          *
3435          * to ensure we observe the correct CPU on which the task is currently
3436          * scheduling.
3437          */
3438         if (smp_load_acquire(&p->on_cpu) &&
3439             ttwu_queue_wakelist(p, task_cpu(p), wake_flags | WF_ON_CPU))
3440                 goto unlock;
3441 
3442         /*
3443          * If the owning (remote) CPU is still in the middle of schedule() with
3444          * this task as prev, wait until it's done referencing the task.
3445          *
3446          * Pairs with the smp_store_release() in finish_task().
3447          *
3448          * This ensures that tasks getting woken will be fully ordered against
3449          * their previous state and preserve Program Order.
3450          */
3451         smp_cond_load_acquire(&p->on_cpu, !VAL);
3452 
3453         cpu = select_task_rq(p, p->wake_cpu, wake_flags | WF_TTWU);
3454         if (task_cpu(p) != cpu) {
3455                 if (p->in_iowait) {
3456                         delayacct_blkio_end(p);
3457                         atomic_dec(&task_rq(p)->nr_iowait);
3458                 }
3459 
3460                 wake_flags |= WF_MIGRATED;
3461                 psi_ttwu_dequeue(p);
3462                 set_task_cpu(p, cpu);
3463         }
3464 #else
3465         cpu = task_cpu(p);
3466 #endif /* CONFIG_SMP */
3467 
3468         ttwu_queue(p, cpu, wake_flags);
3469 unlock:
3470         raw_spin_unlock_irqrestore(&p->pi_lock, flags);
3471 out:
3472         if (success)
3473                 ttwu_stat(p, task_cpu(p), wake_flags);
3474         preempt_enable();
3475 
3476         return success;
3477 }
3478 
3479 /**
3480  * try_invoke_on_locked_down_task - Invoke a function on task in fixed state
3481  * @p: Process for which the function is to be invoked, can be @current.
3482  * @func: Function to invoke.
3483  * @arg: Argument to function.
3484  *
3485  * If the specified task can be quickly locked into a definite state
3486  * (either sleeping or on a given runqueue), arrange to keep it in that
3487  * state while invoking @func(@arg).  This function can use ->on_rq and
3488  * task_curr() to work out what the state is, if required.  Given that
3489  * @func can be invoked with a runqueue lock held, it had better be quite
3490  * lightweight.
3491  *
3492  * Returns:
3493  *      @false if the task slipped out from under the locks.
3494  *      @true if the task was locked onto a runqueue or is sleeping.
3495  *              However, @func can override this by returning @false.
3496  */
3497 bool try_invoke_on_locked_down_task(struct task_struct *p, bool (*func)(struct task_struct *t, void *arg), void *arg)
3498 {
3499         struct rq_flags rf;
3500         bool ret = false;
3501         struct rq *rq;
3502 
3503         raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
3504         if (p->on_rq) {
3505                 rq = __task_rq_lock(p, &rf);
3506                 if (task_rq(p) == rq)
3507                         ret = func(p, arg);
3508                 rq_unlock(rq, &rf);
3509         } else {
3510                 switch (p->state) {
3511                 case TASK_RUNNING:
3512                 case TASK_WAKING:
3513                         break;
3514                 default:
3515                         smp_rmb(); // See smp_rmb() comment in try_to_wake_up().
3516                         if (!p->on_rq)
3517                                 ret = func(p, arg);
3518                 }
3519         }
3520         raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags);
3521         return ret;
3522 }
3523 
3524 /**
3525  * wake_up_process - Wake up a specific process
3526  * @p: The process to be woken up.
3527  *
3528  * Attempt to wake up the nominated process and move it to the set of runnable
3529  * processes.
3530  *
3531  * Return: 1 if the process was woken up, 0 if it was already running.
3532  *
3533  * This function executes a full memory barrier before accessing the task state.
3534  */
3535 int wake_up_process(struct task_struct *p)
3536 {
3537         return try_to_wake_up(p, TASK_NORMAL, 0);
3538 }
3539 EXPORT_SYMBOL(wake_up_process);
3540 
3541 int wake_up_state(struct task_struct *p, unsigned int state)
3542 {
3543         return try_to_wake_up(p, state, 0);
3544 }
3545 
3546 /*
3547  * Perform scheduler related setup for a newly forked process p.
3548  * p is forked by current.
3549  *
3550  * __sched_fork() is basic setup used by init_idle() too:
3551  */
3552 static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
3553 {
3554         p->on_rq                        = 0;
3555 
3556         p->se.on_rq                     = 0;
3557         p->se.exec_start                = 0;
3558         p->se.sum_exec_runtime          = 0;
3559         p->se.prev_sum_exec_runtime     = 0;
3560         p->se.nr_migrations             = 0;
3561         p->se.vruntime                  = 0;
3562         INIT_LIST_HEAD(&p->se.group_node);
3563 
3564 #ifdef CONFIG_FAIR_GROUP_SCHED
3565         p->se.cfs_rq                    = NULL;
3566 #endif
3567 
3568 #ifdef CONFIG_SCHEDSTATS
3569         /* Even if schedstat is disabled, there should not be garbage */
3570         memset(&p->se.statistics, 0, sizeof(p->se.statistics));
3571 #endif
3572 
3573         RB_CLEAR_NODE(&p->dl.rb_node);
3574         init_dl_task_timer(&p->dl);
3575         init_dl_inactive_task_timer(&p->dl);
3576         __dl_clear_params(p);
3577 
3578         INIT_LIST_HEAD(&p->rt.run_list);
3579         p->rt.timeout           = 0;
3580         p->rt.time_slice        = sched_rr_timeslice;
3581         p->rt.on_rq             = 0;
3582         p->rt.on_list           = 0;
3583 
3584 #ifdef CONFIG_PREEMPT_NOTIFIERS
3585         INIT_HLIST_HEAD(&p->preempt_notifiers);
3586 #endif
3587 
3588 #ifdef CONFIG_COMPACTION
3589         p->capture_control = NULL;
3590 #endif
3591         init_numa_balancing(clone_flags, p);
3592 #ifdef CONFIG_SMP
3593         p->wake_entry.u_flags = CSD_TYPE_TTWU;
3594         p->migration_pending = NULL;
3595 #endif
3596 }
3597 
3598 DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
3599 
3600 #ifdef CONFIG_NUMA_BALANCING
3601 
3602 void set_numabalancing_state(bool enabled)
3603 {
3604         if (enabled)
3605                 static_branch_enable(&sched_numa_balancing);
3606         else
3607                 static_branch_disable(&sched_numa_balancing);
3608 }
3609 
3610 #ifdef CONFIG_PROC_SYSCTL
3611 int sysctl_numa_balancing(struct ctl_table *table, int write,
3612                           void *buffer, size_t *lenp, loff_t *ppos)
3613 {
3614         struct ctl_table t;
3615         int err;
3616         int state = static_branch_likely(&sched_numa_balancing);
3617 
3618         if (write && !capable(CAP_SYS_ADMIN))
3619                 return -EPERM;
3620 
3621         t = *table;
3622         t.data = &state;
3623         err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
3624         if (err < 0)
3625                 return err;
3626         if (write)
3627                 set_numabalancing_state(state);
3628         return err;
3629 }
3630 #endif
3631 #endif
3632 
3633 #ifdef CONFIG_SCHEDSTATS
3634 
3635 DEFINE_STATIC_KEY_FALSE(sched_schedstats);
3636 static bool __initdata __sched_schedstats = false;
3637 
3638 static void set_schedstats(bool enabled)
3639 {
3640         if (enabled)
3641                 static_branch_enable(&sched_schedstats);
3642         else
3643                 static_branch_disable(&sched_schedstats);
3644 }
3645 
3646 void force_schedstat_enabled(void)
3647 {
3648         if (!schedstat_enabled()) {
3649                 pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n");
3650                 static_branch_enable(&sched_schedstats);
3651         }
3652 }
3653 
3654 static int __init setup_schedstats(char *str)
3655 {
3656         int ret = 0;
3657         if (!str)
3658                 goto out;
3659 
3660         /*
3661          * This code is called before jump labels have been set up, so we can't
3662          * change the static branch directly just yet.  Instead set a temporary
3663          * variable so init_schedstats() can do it later.
3664          */
3665         if (!strcmp(str, "enable")) {
3666                 __sched_schedstats = true;
3667                 ret = 1;
3668         } else if (!strcmp(str, "disable")) {
3669                 __sched_schedstats = false;
3670                 ret = 1;
3671         }
3672 out:
3673         if (!ret)
3674                 pr_warn("Unable to parse schedstats=\n");
3675 
3676         return ret;
3677 }
3678 __setup("schedstats=", setup_schedstats);
3679 
3680 static void __init init_schedstats(void)
3681 {
3682         set_schedstats(__sched_schedstats);
3683 }
3684 
3685 #ifdef CONFIG_PROC_SYSCTL
3686 int sysctl_schedstats(struct ctl_table *table, int write, void *buffer,
3687                 size_t *lenp, loff_t *ppos)
3688 {
3689         struct ctl_table t;
3690         int err;
3691         int state = static_branch_likely(&sched_schedstats);
3692 
3693         if (write && !capable(CAP_SYS_ADMIN))
3694                 return -EPERM;
3695 
3696         t = *table;
3697         t.data = &state;
3698         err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
3699         if (err < 0)
3700                 return err;
3701         if (write)
3702                 set_schedstats(state);
3703         return err;
3704 }
3705 #endif /* CONFIG_PROC_SYSCTL */
3706 #else  /* !CONFIG_SCHEDSTATS */
3707 static inline void init_schedstats(void) {}
3708 #endif /* CONFIG_SCHEDSTATS */
3709 
3710 /*
3711  * fork()/clone()-time setup:
3712  */
3713 int sched_fork(unsigned long clone_flags, struct task_struct *p)
3714 {
3715         unsigned long flags;
3716 
3717         __sched_fork(clone_flags, p);
3718         /*
3719          * We mark the process as NEW here. This guarantees that
3720          * nobody will actually run it, and a signal or other external
3721          * event cannot wake it up and insert it on the runqueue either.
3722          */
3723         p->state = TASK_NEW;
3724 
3725         /*
3726          * Make sure we do not leak PI boosting priority to the child.
3727          */
3728         p->prio = current->normal_prio;
3729 
3730         uclamp_fork(p);
3731 
3732         /*
3733          * Revert to default priority/policy on fork if requested.
3734          */
3735         if (unlikely(p->sched_reset_on_fork)) {
3736                 if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
3737                         p->policy = SCHED_NORMAL;
3738                         p->static_prio = NICE_TO_PRIO(0);
3739                         p->rt_priority = 0;
3740                 } else if (PRIO_TO_NICE(p->static_prio) < 0)
3741                         p->static_prio = NICE_TO_PRIO(0);
3742 
3743                 p->prio = p->normal_prio = __normal_prio(p);
3744                 set_load_weight(p, false);
3745 
3746                 /*
3747                  * We don't need the reset flag anymore after the fork. It has
3748                  * fulfilled its duty:
3749                  */
3750                 p->sched_reset_on_fork = 0;
3751         }
3752 
3753         if (dl_prio(p->prio))
3754                 return -EAGAIN;
3755         else if (rt_prio(p->prio))
3756                 p->sched_class = &rt_sched_class;
3757         else
3758                 p->sched_class = &fair_sched_class;
3759 
3760         init_entity_runnable_average(&p->se);
3761 
3762         /*
3763          * The child is not yet in the pid-hash so no cgroup attach races,
3764          * and the cgroup is pinned to this child due to cgroup_fork()
3765          * is ran before sched_fork().
3766          *
3767          * Silence PROVE_RCU.
3768          */
3769         raw_spin_lock_irqsave(&p->pi_lock, flags);
3770         rseq_migrate(p);
3771         /*
3772          * We're setting the CPU for the first time, we don't migrate,
3773          * so use __set_task_cpu().
3774          */
3775         __set_task_cpu(p, smp_processor_id());
3776         if (p->sched_class->task_fork)
3777                 p->sched_class->task_fork(p);
3778         raw_spin_unlock_irqrestore(&p->pi_lock, flags);
3779 
3780 #ifdef CONFIG_SCHED_INFO
3781         if (likely(sched_info_on()))
3782                 memset(&p->sched_info, 0, sizeof(p->sched_info));
3783 #endif
3784 #if defined(CONFIG_SMP)
3785         p->on_cpu = 0;
3786 #endif
3787         init_task_preempt_count(p);
3788 #ifdef CONFIG_SMP
3789         plist_node_init(&p->pushable_tasks, MAX_PRIO);
3790         RB_CLEAR_NODE(&p->pushable_dl_tasks);
3791 #endif
3792         return 0;
3793 }
3794 
3795 void sched_post_fork(struct task_struct *p)
3796 {
3797         uclamp_post_fork(p);
3798 }
3799 
3800 unsigned long to_ratio(u64 period, u64 runtime)
3801 {
3802         if (runtime == RUNTIME_INF)
3803                 return BW_UNIT;
3804 
3805         /*
3806          * Doing this here saves a lot of checks in all
3807          * the calling paths, and returning zero seems
3808          * safe for them anyway.
3809          */
3810         if (period == 0)
3811                 return 0;
3812 
3813         return div64_u64(runtime << BW_SHIFT, period);
3814 }
3815 
3816 /*
3817  * wake_up_new_task - wake up a newly created task for the first time.
3818  *
3819  * This function will do some initial scheduler statistics housekeeping
3820  * that must be done for every newly created context, then puts the task
3821  * on the runqueue and wakes it.
3822  */
3823 void wake_up_new_task(struct task_struct *p)
3824 {
3825         struct rq_flags rf;
3826         struct rq *rq;
3827 
3828         raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
3829         p->state = TASK_RUNNING;
3830 #ifdef CONFIG_SMP
3831         /*
3832          * Fork balancing, do it here and not earlier because:
3833          *  - cpus_ptr can change in the fork path
3834          *  - any previously selected CPU might disappear through hotplug
3835          *
3836          * Use __set_task_cpu() to avoid calling sched_class::migrate_task_rq,
3837          * as we're not fully set-up yet.
3838          */
3839         p->recent_used_cpu = task_cpu(p);
3840         rseq_migrate(p);
3841         __set_task_cpu(p, select_task_rq(p, task_cpu(p), WF_FORK));
3842 #endif
3843         rq = __task_rq_lock(p, &rf);
3844         update_rq_clock(rq);
3845         post_init_entity_util_avg(p);
3846 
3847         activate_task(rq, p, ENQUEUE_NOCLOCK);
3848         trace_sched_wakeup_new(p);
3849         check_preempt_curr(rq, p, WF_FORK);
3850 #ifdef CONFIG_SMP
3851         if (p->sched_class->task_woken) {
3852                 /*
3853                  * Nothing relies on rq->lock after this, so it's fine to
3854                  * drop it.
3855                  */
3856                 rq_unpin_lock(rq, &rf);
3857                 p->sched_class->task_woken(rq, p);
3858                 rq_repin_lock(rq, &rf);
3859         }
3860 #endif
3861         task_rq_unlock(rq, p, &rf);
3862 }
3863 
3864 #ifdef CONFIG_PREEMPT_NOTIFIERS
3865 
3866 static DEFINE_STATIC_KEY_FALSE(preempt_notifier_key);
3867 
3868 void preempt_notifier_inc(void)
3869 {
3870         static_branch_inc(&preempt_notifier_key);
3871 }
3872 EXPORT_SYMBOL_GPL(preempt_notifier_inc);
3873 
3874 void preempt_notifier_dec(void)
3875 {
3876         static_branch_dec(&preempt_notifier_key);
3877 }
3878 EXPORT_SYMBOL_GPL(preempt_notifier_dec);
3879 
3880 /**
3881  * preempt_notifier_register - tell me when current is being preempted & rescheduled
3882  * @notifier: notifier struct to register
3883  */
3884 void preempt_notifier_register(struct preempt_notifier *notifier)
3885 {
3886         if (!static_branch_unlikely(&preempt_notifier_key))
3887                 WARN(1, "registering preempt_notifier while notifiers disabled\n");
3888 
3889         hlist_add_head(&notifier->link, &current->preempt_notifiers);
3890 }
3891 EXPORT_SYMBOL_GPL(preempt_notifier_register);
3892 
3893 /**
3894  * preempt_notifier_unregister - no longer interested in preemption notifications
3895  * @notifier: notifier struct to unregister
3896  *
3897  * This is *not* safe to call from within a preemption notifier.
3898  */
3899 void preempt_notifier_unregister(struct preempt_notifier *notifier)
3900 {
3901         hlist_del(&notifier->link);
3902 }
3903 EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
3904 
3905 static void __fire_sched_in_preempt_notifiers(struct task_struct *curr)
3906 {
3907         struct preempt_notifier *notifier;
3908 
3909         hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
3910                 notifier->ops->sched_in(notifier, raw_smp_processor_id());
3911 }
3912 
3913 static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
3914 {
3915         if (static_branch_unlikely(&preempt_notifier_key))
3916                 __fire_sched_in_preempt_notifiers(curr);
3917 }
3918 
3919 static void
3920 __fire_sched_out_preempt_notifiers(struct task_struct *curr,
3921                                    struct task_struct *next)
3922 {
3923         struct preempt_notifier *notifier;
3924 
3925         hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
3926                 notifier->ops->sched_out(notifier, next);
3927 }
3928 
3929 static __always_inline void
3930 fire_sched_out_preempt_notifiers(struct task_struct *curr,
3931                                  struct task_struct *next)
3932 {
3933         if (static_branch_unlikely(&preempt_notifier_key))
3934                 __fire_sched_out_preempt_notifiers(curr, next);
3935 }
3936 
3937 #else /* !CONFIG_PREEMPT_NOTIFIERS */
3938 
3939 static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
3940 {
3941 }
3942 
3943 static inline void
3944 fire_sched_out_preempt_notifiers(struct task_struct *curr,
3945                                  struct task_struct *next)
3946 {
3947 }
3948 
3949 #endif /* CONFIG_PREEMPT_NOTIFIERS */
3950 
3951 static inline void prepare_task(struct task_struct *next)
3952 {
3953 #ifdef CONFIG_SMP
3954         /*
3955          * Claim the task as running, we do this before switching to it
3956          * such that any running task will have this set.
3957          *
3958          * See the ttwu() WF_ON_CPU case and its ordering comment.
3959          */
3960         WRITE_ONCE(next->on_cpu, 1);
3961 #endif
3962 }
3963 
3964 static inline void finish_task(struct task_struct *prev)
3965 {
3966 #ifdef CONFIG_SMP
3967         /*
3968          * This must be the very last reference to @prev from this CPU. After
3969          * p->on_cpu is cleared, the task can be moved to a different CPU. We
3970          * must ensure this doesn't happen until the switch is completely
3971          * finished.
3972          *
3973          * In particular, the load of prev->state in finish_task_switch() must
3974          * happen before this.
3975          *
3976          * Pairs with the smp_cond_load_acquire() in try_to_wake_up().
3977          */
3978         smp_store_release(&prev->on_cpu, 0);
3979 #endif
3980 }
3981 
3982 #ifdef CONFIG_SMP
3983 
3984 static void do_balance_callbacks(struct rq *rq, struct callback_head *head)
3985 {
3986         void (*func)(struct rq *rq);
3987         struct callback_head *next;
3988 
3989         lockdep_assert_held(&rq->lock);
3990 
3991         while (head) {
3992                 func = (void (*)(struct rq *))head->func;
3993                 next = head->next;
3994                 head->next = NULL;
3995                 head = next;
3996 
3997                 func(rq);
3998         }
3999 }
4000 
4001 static void balance_push(struct rq *rq);
4002 
4003 struct callback_head balance_push_callback = {
4004         .next = NULL,
4005         .func = (void (*)(struct callback_head *))balance_push,
4006 };
4007 
4008 static inline struct callback_head *splice_balance_callbacks(struct rq *rq)
4009 {
4010         struct callback_head *head = rq->balance_callback;
4011 
4012         lockdep_assert_held(&rq->lock);
4013         if (head)
4014                 rq->balance_callback = NULL;
4015 
4016         return head;
4017 }
4018 
4019 static void __balance_callbacks(struct rq *rq)
4020 {
4021         do_balance_callbacks(rq, splice_balance_callbacks(rq));
4022 }
4023 
4024 static inline void balance_callbacks(struct rq *rq, struct callback_head *head)
4025 {
4026         unsigned long flags;
4027 
4028         if (unlikely(head)) {
4029                 raw_spin_lock_irqsave(&rq->lock, flags);
4030                 do_balance_callbacks(rq, head);
4031                 raw_spin_unlock_irqrestore(&rq->lock, flags);
4032         }
4033 }
4034 
4035 #else
4036 
4037 static inline void __balance_callbacks(struct rq *rq)
4038 {
4039 }
4040 
4041 static inline struct callback_head *splice_balance_callbacks(struct rq *rq)
4042 {
4043         return NULL;
4044 }
4045 
4046 static inline void balance_callbacks(struct rq *rq, struct callback_head *head)
4047 {
4048 }
4049 
4050 #endif
4051 
4052 static inline void
4053 prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf)
4054 {
4055         /*
4056          * Since the runqueue lock will be released by the next
4057          * task (which is an invalid locking op but in the case
4058          * of the scheduler it's an obvious special-case), so we
4059          * do an early lockdep release here:
4060          */
4061         rq_unpin_lock(rq, rf);
4062         spin_release(&rq->lock.dep_map, _THIS_IP_);
4063 #ifdef CONFIG_DEBUG_SPINLOCK
4064         /* this is a valid case when another task releases the spinlock */
4065         rq->lock.owner = next;
4066 #endif
4067 }
4068 
4069 static inline void finish_lock_switch(struct rq *rq)
4070 {
4071         /*
4072          * If we are tracking spinlock dependencies then we have to
4073          * fix up the runqueue lock - which gets 'carried over' from
4074          * prev into current:
4075          */
4076         spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);
4077         __balance_callbacks(rq);
4078         raw_spin_unlock_irq(&rq->lock);
4079 }
4080 
4081 /*
4082  * NOP if the arch has not defined these:
4083  */
4084 
4085 #ifndef prepare_arch_switch
4086 # define prepare_arch_switch(next)      do { } while (0)
4087 #endif
4088 
4089 #ifndef finish_arch_post_lock_switch
4090 # define finish_arch_post_lock_switch() do { } while (0)
4091 #endif
4092 
4093 static inline void kmap_local_sched_out(void)
4094 {
4095 #ifdef CONFIG_KMAP_LOCAL
4096         if (unlikely(current->kmap_ctrl.idx))
4097                 __kmap_local_sched_out();
4098 #endif
4099 }
4100 
4101 static inline void kmap_local_sched_in(void)
4102 {
4103 #ifdef CONFIG_KMAP_LOCAL
4104         if (unlikely(current->kmap_ctrl.idx))
4105                 __kmap_local_sched_in();
4106 #endif
4107 }
4108 
4109 /**
4110  * prepare_task_switch - prepare to switch tasks
4111  * @rq: the runqueue preparing to switch
4112  * @prev: the current task that is being switched out
4113  * @next: the task we are going to switch to.
4114  *
4115  * This is called with the rq lock held and interrupts off. It must
4116  * be paired with a subsequent finish_task_switch after the context
4117  * switch.
4118  *
4119  * prepare_task_switch sets up locking and calls architecture specific
4120  * hooks.
4121  */
4122 static inline void
4123 prepare_task_switch(struct rq *rq, struct task_struct *prev,
4124                     struct task_struct *next)
4125 {
4126         kcov_prepare_switch(prev);
4127         sched_info_switch(rq, prev, next);
4128         perf_event_task_sched_out(prev, next);
4129         rseq_preempt(prev);
4130         fire_sched_out_preempt_notifiers(prev, next);
4131         kmap_local_sched_out();
4132         prepare_task(next);
4133         prepare_arch_switch(next);
4134 }
4135 
4136 /**
4137  * finish_task_switch - clean up after a task-switch
4138  * @prev: the thread we just switched away from.
4139  *
4140  * finish_task_switch must be called after the context switch, paired
4141  * with a prepare_task_switch call before the context switch.
4142  * finish_task_switch will reconcile locking set up by prepare_task_switch,
4143  * and do any other architecture-specific cleanup actions.
4144  *
4145  * Note that we may have delayed dropping an mm in context_switch(). If
4146  * so, we finish that here outside of the runqueue lock. (Doing it
4147  * with the lock held can cause deadlocks; see schedule() for
4148  * details.)
4149  *
4150  * The context switch have flipped the stack from under us and restored the
4151  * local variables which were saved when this task called schedule() in the
4152  * past. prev == current is still correct but we need to recalculate this_rq
4153  * because prev may have moved to another CPU.
4154  */
4155 static struct rq *finish_task_switch(struct task_struct *prev)
4156         __releases(rq->lock)
4157 {
4158         struct rq *rq = this_rq();
4159         struct mm_struct *mm = rq->prev_mm;
4160         long prev_state;
4161 
4162         /*
4163          * The previous task will have left us with a preempt_count of 2
4164          * because it left us after:
4165          *
4166          *      schedule()
4167          *        preempt_disable();                    // 1
4168          *        __schedule()
4169          *          raw_spin_lock_irq(&rq->lock)        // 2
4170          *
4171          * Also, see FORK_PREEMPT_COUNT.
4172          */
4173         if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET,
4174                       "corrupted preempt_count: %s/%d/0x%x\n",
4175                       current->comm, current->pid, preempt_count()))
4176                 preempt_count_set(FORK_PREEMPT_COUNT);
4177 
4178         rq->prev_mm = NULL;
4179 
4180         /*
4181          * A task struct has one reference for the use as "current".
4182          * If a task dies, then it sets TASK_DEAD in tsk->state and calls
4183          * schedule one last time. The schedule call will never return, and
4184          * the scheduled task must drop that reference.
4185          *
4186          * We must observe prev->state before clearing prev->on_cpu (in
4187          * finish_task), otherwise a concurrent wakeup can get prev
4188          * running on another CPU and we could rave with its RUNNING -> DEAD
4189          * transition, resulting in a double drop.
4190          */
4191         prev_state = prev->state;
4192         vtime_task_switch(prev);
4193         perf_event_task_sched_in(prev, current);
4194         finish_task(prev);
4195         finish_lock_switch(rq);
4196         finish_arch_post_lock_switch();
4197         kcov_finish_switch(current);
4198         /*
4199          * kmap_local_sched_out() is invoked with rq::lock held and
4200          * interrupts disabled. There is no requirement for that, but the
4201          * sched out code does not have an interrupt enabled section.
4202          * Restoring the maps on sched in does not require interrupts being
4203          * disabled either.
4204          */
4205         kmap_local_sched_in();
4206 
4207         fire_sched_in_preempt_notifiers(current);
4208         /*
4209          * When switching through a kernel thread, the loop in
4210          * membarrier_{private,global}_expedited() may have observed that
4211          * kernel thread and not issued an IPI. It is therefore possible to
4212          * schedule between user->kernel->user threads without passing though
4213          * switch_mm(). Membarrier requires a barrier after storing to
4214          * rq->curr, before returning to userspace, so provide them here:
4215          *
4216          * - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly
4217          *   provided by mmdrop(),
4218          * - a sync_core for SYNC_CORE.
4219          */
4220         if (mm) {
4221                 membarrier_mm_sync_core_before_usermode(mm);
4222                 mmdrop(mm);
4223         }
4224         if (unlikely(prev_state == TASK_DEAD)) {
4225                 if (prev->sched_class->task_dead)
4226                         prev->sched_class->task_dead(prev);
4227 
4228                 /*
4229                  * Remove function-return probe instances associated with this
4230                  * task and put them back on the free list.
4231                  */
4232                 kprobe_flush_task(prev);
4233 
4234                 /* Task is done with its stack. */
4235                 put_task_stack(prev);
4236 
4237                 put_task_struct_rcu_user(prev);
4238         }
4239 
4240         tick_nohz_task_switch();
4241         return rq;
4242 }
4243 
4244 /**
4245  * schedule_tail - first thing a freshly forked thread must call.
4246  * @prev: the thread we just switched away from.
4247  */
4248 asmlinkage __visible void schedule_tail(struct task_struct *prev)
4249         __releases(rq->lock)
4250 {
4251         struct rq *rq;
4252 
4253         /*
4254          * New tasks start with FORK_PREEMPT_COUNT, see there and
4255          * finish_task_switch() for details.
4256          *
4257          * finish_task_switch() will drop rq->lock() and lower preempt_count
4258          * and the preempt_enable() will end up enabling preemption (on
4259          * PREEMPT_COUNT kernels).
4260          */
4261 
4262         rq = finish_task_switch(prev);
4263         preempt_enable();
4264 
4265         if (current->set_child_tid)
4266                 put_user(task_pid_vnr(current), current->set_child_tid);
4267 
4268         calculate_sigpending();
4269 }
4270 
4271 /*
4272  * context_switch - switch to the new MM and the new thread's register state.
4273  */
4274 static __always_inline struct rq *
4275 context_switch(struct rq *rq, struct task_struct *prev,
4276                struct task_struct *next, struct rq_flags *rf)
4277 {
4278         prepare_task_switch(rq, prev, next);
4279 
4280         /*
4281          * For paravirt, this is coupled with an exit in switch_to to
4282          * combine the page table reload and the switch backend into
4283          * one hypercall.
4284          */
4285         arch_start_context_switch(prev);
4286 
4287         /*
4288          * kernel -> kernel   lazy + transfer active
4289          *   user -> kernel   lazy + mmgrab() active
4290          *
4291          * kernel ->   user   switch + mmdrop() active
4292          *   user ->   user   switch
4293          */
4294         if (!next->mm) {                                // to kernel
4295                 enter_lazy_tlb(prev->active_mm, next);
4296 
4297                 next->active_mm = prev->active_mm;
4298                 if (prev->mm)                           // from user
4299                         mmgrab(prev->active_mm);
4300                 else
4301                         prev->active_mm = NULL;
4302         } else {                                        // to user
4303                 membarrier_switch_mm(rq, prev->active_mm, next->mm);
4304                 /*
4305                  * sys_membarrier() requires an smp_mb() between setting
4306                  * rq->curr / membarrier_switch_mm() and returning to userspace.
4307                  *
4308                  * The below provides this either through switch_mm(), or in
4309                  * case 'prev->active_mm == next->mm' through
4310                  * finish_task_switch()'s mmdrop().
4311                  */
4312                 switch_mm_irqs_off(prev->active_mm, next->mm, next);
4313 
4314                 if (!prev->mm) {                        // from kernel
4315                         /* will mmdrop() in finish_task_switch(). */
4316                         rq->prev_mm = prev->active_mm;
4317                         prev->active_mm = NULL;
4318                 }
4319         }
4320 
4321         rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP);
4322 
4323         prepare_lock_switch(rq, next, rf);
4324 
4325         /* Here we just switch the register state and the stack. */
4326         switch_to(prev, next, prev);
4327         barrier();
4328 
4329         return finish_task_switch(prev);
4330 }
4331 
4332 /*
4333  * nr_running and nr_context_switches:
4334  *
4335  * externally visible scheduler statistics: current number of runnable
4336  * threads, total number of context switches performed since bootup.
4337  */
4338 unsigned long nr_running(void)
4339 {
4340         unsigned long i, sum = 0;
4341 
4342         for_each_online_cpu(i)
4343                 sum += cpu_rq(i)->nr_running;
4344 
4345         return sum;
4346 }
4347 
4348 /*
4349  * Check if only the current task is running on the CPU.
4350  *
4351  * Caution: this function does not check that the caller has disabled
4352  * preemption, thus the result might have a time-of-check-to-time-of-use
4353  * race.  The caller is responsible to use it correctly, for example:
4354  *
4355  * - from a non-preemptible section (of course)
4356  *
4357  * - from a thread that is bound to a single CPU
4358  *
4359  * - in a loop with very short iterations (e.g. a polling loop)
4360  */
4361 bool single_task_running(void)
4362 {
4363         return raw_rq()->nr_running == 1;
4364 }
4365 EXPORT_SYMBOL(single_task_running);
4366 
4367 unsigned long long nr_context_switches(void)
4368 {
4369         int i;
4370         unsigned long long sum = 0;
4371 
4372         for_each_possible_cpu(i)
4373                 sum += cpu_rq(i)->nr_switches;
4374 
4375         return sum;
4376 }
4377 
4378 /*
4379  * Consumers of these two interfaces, like for example the cpuidle menu
4380  * governor, are using nonsensical data. Preferring shallow idle state selection
4381  * for a CPU that has IO-wait which might not even end up running the task when
4382  * it does become runnable.
4383  */
4384 
4385 unsigned long nr_iowait_cpu(int cpu)
4386 {
4387         return atomic_read(&cpu_rq(cpu)->nr_iowait);
4388 }
4389 
4390 /*
4391  * IO-wait accounting, and how it's mostly bollocks (on SMP).
4392  *
4393  * The idea behind IO-wait account is to account the idle time that we could
4394  * have spend running if it were not for IO. That is, if we were to improve the
4395  * storage performance, we'd have a proportional reduction in IO-wait time.
4396  *
4397  * This all works nicely on UP, where, when a task blocks on IO, we account
4398  * idle time as IO-wait, because if the storage were faster, it could've been
4399  * running and we'd not be idle.
4400  *
4401  * This has been extended to SMP, by doing the same for each CPU. This however
4402  * is broken.
4403  *
4404  * Imagine for instance the case where two tasks block on one CPU, only the one
4405  * CPU will have IO-wait accounted, while the other has regular idle. Even
4406  * though, if the storage were faster, both could've ran at the same time,
4407  * utilising both CPUs.
4408  *
4409  * This means, that when looking globally, the current IO-wait accounting on
4410  * SMP is a lower bound, by reason of under accounting.
4411  *
4412  * Worse, since the numbers are provided per CPU, they are sometimes
4413  * interpreted per CPU, and that is nonsensical. A blocked task isn't strictly
4414  * associated with any one particular CPU, it can wake to another CPU than it
4415  * blocked on. This means the per CPU IO-wait number is meaningless.
4416  *
4417  * Task CPU affinities can make all that even more 'interesting'.
4418  */
4419 
4420 unsigned long nr_iowait(void)
4421 {
4422         unsigned long i, sum = 0;
4423 
4424         for_each_possible_cpu(i)
4425                 sum += nr_iowait_cpu(i);
4426 
4427         return sum;
4428 }
4429 
4430 #ifdef CONFIG_SMP
4431 
4432 /*
4433  * sched_exec - execve() is a valuable balancing opportunity, because at
4434  * this point the task has the smallest effective memory and cache footprint.
4435  */
4436 void sched_exec(void)
4437 {
4438         struct task_struct *p = current;
4439         unsigned long flags;
4440         int dest_cpu;
4441 
4442         raw_spin_lock_irqsave(&p->pi_lock, flags);
4443         dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), WF_EXEC);
4444         if (dest_cpu == smp_processor_id())
4445                 goto unlock;
4446 
4447         if (likely(cpu_active(dest_cpu))) {
4448                 struct migration_arg arg = { p, dest_cpu };
4449 
4450                 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
4451                 stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg);
4452                 return;
4453         }
4454 unlock:
4455         raw_spin_unlock_irqrestore(&p->pi_lock, flags);
4456 }
4457 
4458 #endif
4459 
4460 DEFINE_PER_CPU(struct kernel_stat, kstat);
4461 DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat);
4462 
4463 EXPORT_PER_CPU_SYMBOL(kstat);
4464 EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
4465 
4466 /*
4467  * The function fair_sched_class.update_curr accesses the struct curr
4468  * and its field curr->exec_start; when called from task_sched_runtime(),
4469  * we observe a high rate of cache misses in practice.
4470  * Prefetching this data results in improved performance.
4471  */
4472 static inline void prefetch_curr_exec_start(struct task_struct *p)
4473 {
4474 #ifdef CONFIG_FAIR_GROUP_SCHED
4475         struct sched_entity *curr = (&p->se)->cfs_rq->curr;
4476 #else
4477         struct sched_entity *curr = (&task_rq(p)->cfs)->curr;
4478 #endif
4479         prefetch(curr);
4480         prefetch(&curr->exec_start);
4481 }
4482 
4483 /*
4484  * Return accounted runtime for the task.
4485  * In case the task is currently running, return the runtime plus current's
4486  * pending runtime that have not been accounted yet.
4487  */
4488 unsigned long long task_sched_runtime(struct task_struct *p)
4489 {
4490         struct rq_flags rf;
4491         struct rq *rq;
4492         u64 ns;
4493 
4494 #if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
4495         /*
4496          * 64-bit doesn't need locks to atomically read a 64-bit value.
4497          * So we have a optimization chance when the task's delta_exec is 0.
4498          * Reading ->on_cpu is racy, but this is ok.
4499          *
4500          * If we race with it leaving CPU, we'll take a lock. So we're correct.
4501          * If we race with it entering CPU, unaccounted time is 0. This is
4502          * indistinguishable from the read occurring a few cycles earlier.
4503          * If we see ->on_cpu without ->on_rq, the task is leaving, and has
4504          * been accounted, so we're correct here as well.
4505          */
4506         if (!p->on_cpu || !task_on_rq_queued(p))
4507                 return p->se.sum_exec_runtime;
4508 #endif
4509 
4510         rq = task_rq_lock(p, &rf);
4511         /*
4512          * Must be ->curr _and_ ->on_rq.  If dequeued, we would
4513          * project cycles that may never be accounted to this
4514          * thread, breaking clock_gettime().
4515          */
4516         if (task_current(rq, p) && task_on_rq_queued(p)) {
4517                 prefetch_curr_exec_start(p);
4518                 update_rq_clock(rq);
4519                 p->sched_class->update_curr(rq);
4520         }
4521         ns = p->se.sum_exec_runtime;
4522         task_rq_unlock(rq, p, &rf);
4523 
4524         return ns;
4525 }
4526 
4527 /*
4528  * This function gets called by the timer code, with HZ frequency.
4529  * We call it with interrupts disabled.
4530  */
4531 void scheduler_tick(void)
4532 {
4533         int cpu = smp_processor_id();
4534         struct rq *rq = cpu_rq(cpu);
4535         struct task_struct *curr = rq->curr;
4536         struct rq_flags rf;
4537         unsigned long thermal_pressure;
4538 
4539         arch_scale_freq_tick();
4540         sched_clock_tick();
4541 
4542         rq_lock(rq, &rf);
4543 
4544         update_rq_clock(rq);
4545         thermal_pressure = arch_scale_thermal_pressure(cpu_of(rq));
4546         update_thermal_load_avg(rq_clock_thermal(rq), rq, thermal_pressure);
4547         curr->sched_class->task_tick(rq, curr, 0);
4548         calc_global_load_tick(rq);
4549         psi_task_tick(rq);
4550 
4551         rq_unlock(rq, &rf);
4552 
4553         perf_event_task_tick();
4554 
4555 #ifdef CONFIG_SMP
4556         rq->idle_balance = idle_cpu(cpu);
4557         trigger_load_balance(rq);
4558 #endif
4559 }
4560 
4561 #ifdef CONFIG_NO_HZ_FULL
4562 
4563 struct tick_work {
4564         int                     cpu;
4565         atomic_t                state;
4566         struct delayed_work     work;
4567 };
4568 /* Values for ->state, see diagram below. */
4569 #define TICK_SCHED_REMOTE_OFFLINE       0
4570 #define TICK_SCHED_REMOTE_OFFLINING     1
4571 #define TICK_SCHED_REMOTE_RUNNING       2
4572 
4573 /*
4574  * State diagram for ->state:
4575  *
4576  *
4577  *          TICK_SCHED_REMOTE_OFFLINE
4578  *                    |   ^
4579  *                    |   |
4580  *                    |   | sched_tick_remote()
4581  *                    |   |
4582  *                    |   |
4583  *                    +--TICK_SCHED_REMOTE_OFFLINING
4584  *                    |   ^
4585  *                    |   |
4586  * sched_tick_start() |   | sched_tick_stop()
4587  *                    |   |
4588  *                    V   |
4589  *          TICK_SCHED_REMOTE_RUNNING
4590  *
4591  *
4592  * Other transitions get WARN_ON_ONCE(), except that sched_tick_remote()
4593  * and sched_tick_start() are happy to leave the state in RUNNING.
4594  */
4595 
4596 static struct tick_work __percpu *tick_work_cpu;
4597 
4598 static void sched_tick_remote(struct work_struct *work)
4599 {
4600         struct delayed_work *dwork = to_delayed_work(work);
4601         struct tick_work *twork = container_of(dwork, struct tick_work, work);
4602         int cpu = twork->cpu;
4603         struct rq *rq = cpu_rq(cpu);
4604         struct task_struct *curr;
4605         struct rq_flags rf;
4606         u64 delta;
4607         int os;
4608 
4609         /*
4610          * Handle the tick only if it appears the remote CPU is running in full
4611          * dynticks mode. The check is racy by nature, but missing a tick or
4612          * having one too much is no big deal because the scheduler tick updates
4613          * statistics and checks timeslices in a time-independent way, regardless
4614          * of when exactly it is running.
4615          */
4616         if (!tick_nohz_tick_stopped_cpu(cpu))
4617                 goto out_requeue;
4618 
4619         rq_lock_irq(rq, &rf);
4620         curr = rq->curr;
4621         if (cpu_is_offline(cpu))
4622                 goto out_unlock;
4623 
4624         update_rq_clock(rq);
4625 
4626         if (!is_idle_task(curr)) {
4627                 /*
4628                  * Make sure the next tick runs within a reasonable
4629                  * amount of time.
4630                  */
4631                 delta = rq_clock_task(rq) - curr->se.exec_start;
4632                 WARN_ON_ONCE(delta > (u64)NSEC_PER_SEC * 3);
4633         }
4634         curr->sched_class->task_tick(rq, curr, 0);
4635 
4636         calc_load_nohz_remote(rq);
4637 out_unlock:
4638         rq_unlock_irq(rq, &rf);
4639 out_requeue:
4640 
4641         /*
4642          * Run the remote tick once per second (1Hz). This arbitrary
4643          * frequency is large enough to avoid overload but short enough
4644          * to keep scheduler internal stats reasonably up to date.  But
4645          * first update state to reflect hotplug activity if required.
4646          */
4647         os = atomic_fetch_add_unless(&twork->state, -1, TICK_SCHED_REMOTE_RUNNING);
4648         WARN_ON_ONCE(os == TICK_SCHED_REMOTE_OFFLINE);
4649         if (os == TICK_SCHED_REMOTE_RUNNING)
4650                 queue_delayed_work(system_unbound_wq, dwork, HZ);
4651 }
4652 
4653 static void sched_tick_start(int cpu)
4654 {
4655         int os;
4656         struct tick_work *twork;
4657 
4658         if (housekeeping_cpu(cpu, HK_FLAG_TICK))
4659                 return;
4660 
4661         WARN_ON_ONCE(!tick_work_cpu);
4662 
4663         twork = per_cpu_ptr(tick_work_cpu, cpu);
4664         os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_RUNNING);
4665         WARN_ON_ONCE(os == TICK_SCHED_REMOTE_RUNNING);
4666         if (os == TICK_SCHED_REMOTE_OFFLINE) {
4667                 twork->cpu = cpu;
4668                 INIT_DELAYED_WORK(&twork->work, sched_tick_remote);
4669                 queue_delayed_work(system_unbound_wq, &twork->work, HZ);
4670         }
4671 }
4672 
4673 #ifdef CONFIG_HOTPLUG_CPU
4674 static void sched_tick_stop(int cpu)
4675 {
4676         struct tick_work *twork;
4677         int os;
4678 
4679         if (housekeeping_cpu(cpu, HK_FLAG_TICK))
4680                 return;
4681 
4682         WARN_ON_ONCE(!tick_work_cpu);
4683 
4684         twork = per_cpu_ptr(tick_work_cpu, cpu);
4685         /* There cannot be competing actions, but don't rely on stop-machine. */
4686         os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_OFFLINING);
4687         WARN_ON_ONCE(os != TICK_SCHED_REMOTE_RUNNING);
4688         /* Don't cancel, as this would mess up the state machine. */
4689 }
4690 #endif /* CONFIG_HOTPLUG_CPU */
4691 
4692 int __init sched_tick_offload_init(void)
4693 {
4694         tick_work_cpu = alloc_percpu(struct tick_work);
4695         BUG_ON(!tick_work_cpu);
4696         return 0;
4697 }
4698 
4699 #else /* !CONFIG_NO_HZ_FULL */
4700 static inline void sched_tick_start(int cpu) { }
4701 static inline void sched_tick_stop(int cpu) { }
4702 #endif
4703 
4704 #if defined(CONFIG_PREEMPTION) && (defined(CONFIG_DEBUG_PREEMPT) || \
4705                                 defined(CONFIG_TRACE_PREEMPT_TOGGLE))
4706 /*
4707  * If the value passed in is equal to the current preempt count
4708  * then we just disabled preemption. Start timing the latency.
4709  */
4710 static inline void preempt_latency_start(int val)
4711 {
4712         if (preempt_count() == val) {
4713                 unsigned long ip = get_lock_parent_ip();
4714 #ifdef CONFIG_DEBUG_PREEMPT
4715                 current->preempt_disable_ip = ip;
4716 #endif
4717                 trace_preempt_off(CALLER_ADDR0, ip);
4718         }
4719 }
4720 
4721 void preempt_count_add(int val)
4722 {
4723 #ifdef CONFIG_DEBUG_PREEMPT
4724         /*
4725          * Underflow?
4726          */
4727         if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
4728                 return;
4729 #endif
4730         __preempt_count_add(val);
4731 #ifdef CONFIG_DEBUG_PREEMPT
4732         /*
4733          * Spinlock count overflowing soon?
4734          */
4735         DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
4736                                 PREEMPT_MASK - 10);
4737 #endif
4738         preempt_latency_start(val);
4739 }
4740 EXPORT_SYMBOL(preempt_count_add);
4741 NOKPROBE_SYMBOL(preempt_count_add);
4742 
4743 /*
4744  * If the value passed in equals to the current preempt count
4745  * then we just enabled preemption. Stop timing the latency.
4746  */
4747 static inline void preempt_latency_stop(int val)
4748 {
4749         if (preempt_count() == val)
4750                 trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip());
4751 }
4752 
4753 void preempt_count_sub(int val)
4754 {
4755 #ifdef CONFIG_DEBUG_PREEMPT
4756         /*
4757          * Underflow?
4758          */
4759         if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
4760                 return;
4761         /*
4762          * Is the spinlock portion underflowing?
4763          */
4764         if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
4765                         !(preempt_count() & PREEMPT_MASK)))
4766                 return;
4767 #endif
4768 
4769         preempt_latency_stop(val);
4770         __preempt_count_sub(val);
4771 }
4772 EXPORT_SYMBOL(preempt_count_sub);
4773 NOKPROBE_SYMBOL(preempt_count_sub);
4774 
4775 #else
4776 static inline void preempt_latency_start(int val) { }
4777 static inline void preempt_latency_stop(int val) { }
4778 #endif
4779 
4780 static inline unsigned long get_preempt_disable_ip(struct task_struct *p)
4781 {
4782 #ifdef CONFIG_DEBUG_PREEMPT
4783         return p->preempt_disable_ip;
4784 #else
4785         return 0;
4786 #endif
4787 }
4788 
4789 /*
4790  * Print scheduling while atomic bug:
4791  */
4792 static noinline void __schedule_bug(struct task_struct *prev)
4793 {
4794         /* Save this before calling printk(), since that will clobber it */
4795         unsigned long preempt_disable_ip = get_preempt_disable_ip(current);
4796 
4797         if (oops_in_progress)
4798                 return;
4799 
4800         printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
4801                 prev->comm, prev->pid, preempt_count());
4802 
4803         debug_show_held_locks(prev);
4804         print_modules();
4805         if (irqs_disabled())
4806                 print_irqtrace_events(prev);
4807         if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)
4808             && in_atomic_preempt_off()) {
4809                 pr_err("Preemption disabled at:");
4810                 print_ip_sym(KERN_ERR, preempt_disable_ip);
4811         }
4812         if (panic_on_warn)
4813                 panic("scheduling while atomic\n");
4814 
4815         dump_stack();
4816         add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
4817 }
4818 
4819 /*
4820  * Various schedule()-time debugging checks and statistics:
4821  */
4822 static inline void schedule_debug(struct task_struct *prev, bool preempt)
4823 {
4824 #ifdef CONFIG_SCHED_STACK_END_CHECK
4825         if (task_stack_end_corrupted(prev))
4826                 panic("corrupted stack end detected inside scheduler\n");
4827 
4828         if (task_scs_end_corrupted(prev))
4829                 panic("corrupted shadow stack detected inside scheduler\n");
4830 #endif
4831 
4832 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
4833         if (!preempt && prev->state && prev->non_block_count) {
4834                 printk(KERN_ERR "BUG: scheduling in a non-blocking section: %s/%d/%i\n",
4835                         prev->comm, prev->pid, prev->non_block_count);
4836                 dump_stack();
4837                 add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
4838         }
4839 #endif
4840 
4841         if (unlikely(in_atomic_preempt_off())) {
4842                 __schedule_bug(prev);
4843                 preempt_count_set(PREEMPT_DISABLED);
4844         }
4845         rcu_sleep_check();
4846         SCHED_WARN_ON(ct_state() == CONTEXT_USER);
4847 
4848         profile_hit(SCHED_PROFILING, __builtin_return_address(0));
4849 
4850         schedstat_inc(this_rq()->sched_count);
4851 }
4852 
4853 static void put_prev_task_balance(struct rq *rq, struct task_struct *prev,
4854                                   struct rq_flags *rf)
4855 {
4856 #ifdef CONFIG_SMP
4857         const struct sched_class *class;
4858         /*
4859          * We must do the balancing pass before put_prev_task(), such
4860          * that when we release the rq->lock the task is in the same
4861          * state as before we took rq->lock.
4862          *
4863          * We can terminate the balance pass as soon as we know there is
4864          * a runnable task of @class priority or higher.
4865          */
4866         for_class_range(class, prev->sched_class, &idle_sched_class) {
4867                 if (class->balance(rq, prev, rf))
4868                         break;
4869         }
4870 #endif
4871 
4872         put_prev_task(rq, prev);
4873 }
4874 
4875 /*
4876  * Pick up the highest-prio task:
4877  */
4878 static inline struct task_struct *
4879 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
4880 {
4881         const struct sched_class *class;
4882         struct task_struct *p;
4883 
4884         /*
4885          * Optimization: we know that if all tasks are in the fair class we can
4886          * call that function directly, but only if the @prev task wasn't of a
4887          * higher scheduling class, because otherwise those lose the
4888          * opportunity to pull in more work from other CPUs.
4889          */
4890         if (likely(prev->sched_class <= &fair_sched_class &&
4891                    rq->nr_running == rq->cfs.h_nr_running)) {
4892 
4893                 p = pick_next_task_fair(rq, prev, rf);
4894                 if (unlikely(p == RETRY_TASK))
4895                         goto restart;
4896 
4897                 /* Assumes fair_sched_class->next == idle_sched_class */
4898                 if (!p) {
4899                         put_prev_task(rq, prev);
4900                         p = pick_next_task_idle(rq);
4901                 }
4902 
4903                 return p;
4904         }
4905 
4906 restart:
4907         put_prev_task_balance(rq, prev, rf);
4908 
4909         for_each_class(class) {
4910                 p = class->pick_next_task(rq);
4911                 if (p)
4912                         return p;
4913         }
4914 
4915         /* The idle class should always have a runnable task: */
4916         BUG();
4917 }
4918 
4919 /*
4920  * __schedule() is the main scheduler function.
4921  *
4922  * The main means of driving the scheduler and thus entering this function are:
4923  *
4924  *   1. Explicit blocking: mutex, semaphore, waitqueue, etc.
4925  *
4926  *   2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return
4927  *      paths. For example, see arch/x86/entry_64.S.
4928  *
4929  *      To drive preemption between tasks, the scheduler sets the flag in timer
4930  *      interrupt handler scheduler_tick().
4931  *
4932  *   3. Wakeups don't really cause entry into schedule(). They add a
4933  *      task to the run-queue and that's it.
4934  *
4935  *      Now, if the new task added to the run-queue preempts the current
4936  *      task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets
4937  *      called on the nearest possible occasion:
4938  *
4939  *       - If the kernel is preemptible (CONFIG_PREEMPTION=y):
4940  *
4941  *         - in syscall or exception context, at the next outmost
4942  *           preempt_enable(). (this might be as soon as the wake_up()'s
4943  *           spin_unlock()!)
4944  *
4945  *         - in IRQ context, return from interrupt-handler to
4946  *           preemptible context
4947  *
4948  *       - If the kernel is not preemptible (CONFIG_PREEMPTION is not set)
4949  *         then at the next:
4950  *
4951  *          - cond_resched() call
4952  *          - explicit schedule() call
4953  *          - return from syscall or exception to user-space
4954  *          - return from interrupt-handler to user-space
4955  *
4956  * WARNING: must be called with preemption disabled!
4957  */
4958 static void __sched notrace __schedule(bool preempt)
4959 {
4960         struct task_struct *prev, *next;
4961         unsigned long *switch_count;
4962         unsigned long prev_state;
4963         struct rq_flags rf;
4964         struct rq *rq;
4965         int cpu;
4966 
4967         cpu = smp_processor_id();
4968         rq = cpu_rq(cpu);
4969         prev = rq->curr;
4970 
4971         schedule_debug(prev, preempt);
4972 
4973         if (sched_feat(HRTICK) || sched_feat(HRTICK_DL))
4974                 hrtick_clear(rq);
4975 
4976         local_irq_disable();
4977         rcu_note_context_switch(preempt);
4978 
4979         /*
4980          * Make sure that signal_pending_state()->signal_pending() below
4981          * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE)
4982          * done by the caller to avoid the race with signal_wake_up():
4983          *
4984          * __set_current_state(@state)          signal_wake_up()
4985          * schedule()                             set_tsk_thread_flag(p, TIF_SIGPENDING)
4986          *                                        wake_up_state(p, state)
4987          *   LOCK rq->lock                          LOCK p->pi_state
4988          *   smp_mb__after_spinlock()               smp_mb__after_spinlock()
4989          *     if (signal_pending_state())          if (p->state & @state)
4990          *
4991          * Also, the membarrier system call requires a full memory barrier
4992          * after coming from user-space, before storing to rq->curr.
4993          */
4994         rq_lock(rq, &rf);
4995         smp_mb__after_spinlock();
4996 
4997         /* Promote REQ to ACT */
4998         rq->clock_update_flags <<= 1;
4999         update_rq_clock(rq);
5000 
5001         switch_count = &prev->nivcsw;
5002 
5003         /*
5004          * We must load prev->state once (task_struct::state is volatile), such
5005          * that:
5006          *
5007          *  - we form a control dependency vs deactivate_task() below.
5008          *  - ptrace_{,un}freeze_traced() can change ->state underneath us.
5009          */
5010         prev_state = prev->state;
5011         if (!preempt && prev_state) {
5012                 if (signal_pending_state(prev_state, prev)) {
5013                         prev->state = TASK_RUNNING;
5014                 } else {
5015                         prev->sched_contributes_to_load =
5016                                 (prev_state & TASK_UNINTERRUPTIBLE) &&
5017                                 !(prev_state & TASK_NOLOAD) &&
5018                                 !(prev->flags & PF_FROZEN);
5019 
5020                         if (prev->sched_contributes_to_load)
5021                                 rq->nr_uninterruptible++;
5022 
5023                         /*
5024                          * __schedule()                 ttwu()
5025                          *   prev_state = prev->state;    if (p->on_rq && ...)
5026                          *   if (prev_state)                goto out;
5027                          *     p->on_rq = 0;              smp_acquire__after_ctrl_dep();
5028                          *                                p->state = TASK_WAKING
5029                          *
5030                          * Where __schedule() and ttwu() have matching control dependencies.
5031                          *
5032                          * After this, schedule() must not care about p->state any more.
5033                          */
5034                         deactivate_task(rq, prev, DEQUEUE_SLEEP | DEQUEUE_NOCLOCK);
5035 
5036                         if (prev->in_iowait) {
5037                                 atomic_inc(&rq->nr_iowait);
5038                                 delayacct_blkio_start();
5039                         }
5040                 }
5041                 switch_count = &prev->nvcsw;
5042         }
5043 
5044         next = pick_next_task(rq, prev, &rf);
5045         clear_tsk_need_resched(prev);
5046         clear_preempt_need_resched();
5047 
5048         if (likely(prev != next)) {
5049                 rq->nr_switches++;
5050                 /*
5051                  * RCU users of rcu_dereference(rq->curr) may not see
5052                  * changes to task_struct made by pick_next_task().
5053                  */
5054                 RCU_INIT_POINTER(rq->curr, next);
5055                 /*
5056                  * The membarrier system call requires each architecture
5057                  * to have a full memory barrier after updating
5058                  * rq->curr, before returning to user-space.
5059                  *
5060                  * Here are the schemes providing that barrier on the
5061                  * various architectures:
5062                  * - mm ? switch_mm() : mmdrop() for x86, s390, sparc, PowerPC.
5063                  *   switch_mm() rely on membarrier_arch_switch_mm() on PowerPC.
5064                  * - finish_lock_switch() for weakly-ordered
5065                  *   architectures where spin_unlock is a full barrier,
5066                  * - switch_to() for arm64 (weakly-ordered, spin_unlock
5067                  *   is a RELEASE barrier),
5068                  */
5069                 ++*switch_count;
5070 
5071                 migrate_disable_switch(rq, prev);
5072                 psi_sched_switch(prev, next, !task_on_rq_queued(prev));
5073 
5074                 trace_sched_switch(preempt, prev, next);
5075 
5076                 /* Also unlocks the rq: */
5077                 rq = context_switch(rq, prev, next, &rf);
5078         } else {
5079                 rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP);
5080 
5081                 rq_unpin_lock(rq, &rf);
5082                 __balance_callbacks(rq);
5083                 raw_spin_unlock_irq(&rq->lock);
5084         }
5085 }
5086 
5087 void __noreturn do_task_dead(void)
5088 {
5089         /* Causes final put_task_struct in finish_task_switch(): */
5090         set_special_state(TASK_DEAD);
5091 
5092         /* Tell freezer to ignore us: */
5093         current->flags |= PF_NOFREEZE;
5094 
5095         __schedule(false);
5096         BUG();
5097 
5098         /* Avoid "noreturn function does return" - but don't continue if BUG() is a NOP: */
5099         for (;;)
5100                 cpu_relax();
5101 }
5102 
5103 static inline void sched_submit_work(struct task_struct *tsk)
5104 {
5105         unsigned int task_flags;
5106 
5107         if (!tsk->state)
5108                 return;
5109 
5110         task_flags = tsk->flags;
5111         /*
5112          * If a worker went to sleep, notify and ask workqueue whether
5113          * it wants to wake up a task to maintain concurrency.
5114          * As this function is called inside the schedule() context,
5115          * we disable preemption to avoid it calling schedule() again
5116          * in the possible wakeup of a kworker and because wq_worker_sleeping()
5117          * requires it.
5118          */
5119         if (task_flags & (PF_WQ_WORKER | PF_IO_WORKER)) {
5120                 preempt_disable();
5121                 if (task_flags & PF_WQ_WORKER)
5122                         wq_worker_sleeping(tsk);
5123                 else
5124                         io_wq_worker_sleeping(tsk);
5125                 preempt_enable_no_resched();
5126         }
5127 
5128         if (tsk_is_pi_blocked(tsk))
5129                 return;
5130 
5131         /*
5132          * If we are going to sleep and we have plugged IO queued,
5133          * make sure to submit it to avoid deadlocks.
5134          */
5135         if (blk_needs_flush_plug(tsk))
5136                 blk_schedule_flush_plug(tsk);
5137 }
5138 
5139 static void sched_update_worker(struct task_struct *tsk)
5140 {
5141         if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER)) {
5142                 if (tsk->flags & PF_WQ_WORKER)
5143                         wq_worker_running(tsk);
5144                 else
5145                         io_wq_worker_running(tsk);
5146         }
5147 }
5148 
5149 asmlinkage __visible void __sched schedule(void)
5150 {
5151         struct task_struct *tsk = current;
5152 
5153         sched_submit_work(tsk);
5154         do {
5155                 preempt_disable();
5156                 __schedule(false);
5157                 sched_preempt_enable_no_resched();
5158         } while (need_resched());
5159         sched_update_worker(tsk);
5160 }
5161 EXPORT_SYMBOL(schedule);
5162 
5163 /*
5164  * synchronize_rcu_tasks() makes sure that no task is stuck in preempted
5165  * state (have scheduled out non-voluntarily) by making sure that all
5166  * tasks have either left the run queue or have gone into user space.
5167  * As idle tasks do not do either, they must not ever be preempted
5168  * (schedule out non-voluntarily).
5169  *
5170  * schedule_idle() is similar to schedule_preempt_disable() except that it
5171  * never enables preemption because it does not call sched_submit_work().
5172  */
5173 void __sched schedule_idle(void)
5174 {
5175         /*
5176          * As this skips calling sched_submit_work(), which the idle task does
5177          * regardless because that function is a nop when the task is in a
5178          * TASK_RUNNING state, make sure this isn't used someplace that the
5179          * current task can be in any other state. Note, idle is always in the
5180          * TASK_RUNNING state.
5181          */
5182         WARN_ON_ONCE(current->state);
5183         do {
5184                 __schedule(false);
5185         } while (need_resched());
5186 }
5187 
5188 #if defined(CONFIG_CONTEXT_TRACKING) && !defined(CONFIG_HAVE_CONTEXT_TRACKING_OFFSTACK)
5189 asmlinkage __visible void __sched schedule_user(void)
5190 {
5191         /*
5192          * If we come here after a random call to set_need_resched(),
5193          * or we have been woken up remotely but the IPI has not yet arrived,
5194          * we haven't yet exited the RCU idle mode. Do it here manually until
5195          * we find a better solution.
5196          *
5197          * NB: There are buggy callers of this function.  Ideally we
5198          * should warn if prev_state != CONTEXT_USER, but that will trigger
5199          * too frequently to make sense yet.
5200          */
5201         enum ctx_state prev_state = exception_enter();
5202         schedule();
5203         exception_exit(prev_state);
5204 }
5205 #endif
5206 
5207 /**
5208  * schedule_preempt_disabled - called with preemption disabled
5209  *
5210  * Returns with preemption disabled. Note: preempt_count must be 1
5211  */
5212 void __sched schedule_preempt_disabled(void)
5213 {
5214         sched_preempt_enable_no_resched();
5215         schedule();
5216         preempt_disable();
5217 }
5218 
5219 static void __sched notrace preempt_schedule_common(void)
5220 {
5221         do {
5222                 /*
5223                  * Because the function tracer can trace preempt_count_sub()
5224                  * and it also uses preempt_enable/disable_notrace(), if
5225                  * NEED_RESCHED is set, the preempt_enable_notrace() called
5226                  * by the function tracer will call this function again and
5227                  * cause infinite recursion.
5228                  *
5229                  * Preemption must be disabled here before the function
5230                  * tracer can trace. Break up preempt_disable() into two
5231                  * calls. One to disable preemption without fear of being
5232                  * traced. The other to still record the preemption latency,
5233                  * which can also be traced by the function tracer.
5234                  */
5235                 preempt_disable_notrace();
5236                 preempt_latency_start(1);
5237                 __schedule(true);
5238                 preempt_latency_stop(1);
5239                 preempt_enable_no_resched_notrace();
5240 
5241                 /*
5242                  * Check again in case we missed a preemption opportunity
5243                  * between schedule and now.
5244                  */
5245         } while (need_resched());
5246 }
5247 
5248 #ifdef CONFIG_PREEMPTION
5249 /*
5250  * This is the entry point to schedule() from in-kernel preemption
5251  * off of preempt_enable.
5252  */
5253 asmlinkage __visible void __sched notrace preempt_schedule(void)
5254 {
5255         /*
5256          * If there is a non-zero preempt_count or interrupts are disabled,
5257          * we do not want to preempt the current task. Just return..
5258          */
5259         if (likely(!preemptible()))
5260                 return;
5261 
5262         preempt_schedule_common();
5263 }
5264 NOKPROBE_SYMBOL(preempt_schedule);
5265 EXPORT_SYMBOL(preempt_schedule);
5266 
5267 #ifdef CONFIG_PREEMPT_DYNAMIC
5268 DEFINE_STATIC_CALL(preempt_schedule, __preempt_schedule_func);
5269 EXPORT_STATIC_CALL_TRAMP(preempt_schedule);
5270 #endif
5271 
5272 
5273 /**
5274  * preempt_schedule_notrace - preempt_schedule called by tracing
5275  *
5276  * The tracing infrastructure uses preempt_enable_notrace to prevent
5277  * recursion and tracing preempt enabling caused by the tracing
5278  * infrastructure itself. But as tracing can happen in areas coming
5279  * from userspace or just about to enter userspace, a preempt enable
5280  * can occur before user_exit() is called. This will cause the scheduler
5281  * to be called when the system is still in usermode.
5282  *
5283  * To prevent this, the preempt_enable_notrace will use this function
5284  * instead of preempt_schedule() to exit user context if needed before
5285  * calling the scheduler.
5286  */
5287 asmlinkage __visible void __sched notrace preempt_schedule_notrace(void)
5288 {
5289         enum ctx_state prev_ctx;
5290 
5291         if (likely(!preemptible()))
5292                 return;
5293 
5294         do {
5295                 /*
5296                  * Because the function tracer can trace preempt_count_sub()
5297                  * and it also uses preempt_enable/disable_notrace(), if
5298                  * NEED_RESCHED is set, the preempt_enable_notrace() called
5299                  * by the function tracer will call this function again and
5300                  * cause infinite recursion.
5301                  *
5302                  * Preemption must be disabled here before the function
5303                  * tracer can trace. Break up preempt_disable() into two
5304                  * calls. One to disable preemption without fear of being
5305                  * traced. The other to still record the preemption latency,
5306                  * which can also be traced by the function tracer.
5307                  */
5308                 preempt_disable_notrace();
5309                 preempt_latency_start(1);
5310                 /*
5311                  * Needs preempt disabled in case user_exit() is traced
5312                  * and the tracer calls preempt_enable_notrace() causing
5313                  * an infinite recursion.
5314                  */
5315                 prev_ctx = exception_enter();
5316                 __schedule(true);
5317                 exception_exit(prev_ctx);
5318 
5319                 preempt_latency_stop(1);
5320                 preempt_enable_no_resched_notrace();
5321         } while (need_resched());
5322 }
5323 EXPORT_SYMBOL_GPL(preempt_schedule_notrace);
5324 
5325 #ifdef CONFIG_PREEMPT_DYNAMIC
5326 DEFINE_STATIC_CALL(preempt_schedule_notrace, __preempt_schedule_notrace_func);
5327 EXPORT_STATIC_CALL_TRAMP(preempt_schedule_notrace);
5328 #endif
5329 
5330 #endif /* CONFIG_PREEMPTION */
5331 
5332 #ifdef CONFIG_PREEMPT_DYNAMIC
5333 
5334 #include <linux/entry-common.h>
5335 
5336 /*
5337  * SC:cond_resched
5338  * SC:might_resched
5339  * SC:preempt_schedule
5340  * SC:preempt_schedule_notrace
5341  * SC:irqentry_exit_cond_resched
5342  *
5343  *
5344  * NONE:
5345  *   cond_resched               <- __cond_resched
5346  *   might_resched              <- RET0
5347  *   preempt_schedule           <- NOP
5348  *   preempt_schedule_notrace   <- NOP
5349  *   irqentry_exit_cond_resched <- NOP
5350  *
5351  * VOLUNTARY:
5352  *   cond_resched               <- __cond_resched
5353  *   might_resched              <- __cond_resched
5354  *   preempt_schedule           <- NOP
5355  *   preempt_schedule_notrace   <- NOP
5356  *   irqentry_exit_cond_resched <- NOP
5357  *
5358  * FULL:
5359  *   cond_resched               <- RET0
5360  *   might_resched              <- RET0
5361  *   preempt_schedule           <- preempt_schedule
5362  *   preempt_schedule_notrace   <- preempt_schedule_notrace
5363  *   irqentry_exit_cond_resched <- irqentry_exit_cond_resched
5364  */
5365 
5366 enum {
5367         preempt_dynamic_none = 0,
5368         preempt_dynamic_voluntary,
5369         preempt_dynamic_full,
5370 };
5371 
5372 static int preempt_dynamic_mode = preempt_dynamic_full;
5373 
5374 static int sched_dynamic_mode(const char *str)
5375 {
5376         if (!strcmp(str, "none"))
5377                 return 0;
5378 
5379         if (!strcmp(str, "voluntary"))
5380                 return 1;
5381 
5382         if (!strcmp(str, "full"))
5383                 return 2;
5384 
5385         return -1;
5386 }
5387 
5388 static void sched_dynamic_update(int mode)
5389 {
5390         /*
5391          * Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in
5392          * the ZERO state, which is invalid.
5393          */
5394         static_call_update(cond_resched, __cond_resched);
5395         static_call_update(might_resched, __cond_resched);
5396         static_call_update(preempt_schedule, __preempt_schedule_func);
5397         static_call_update(preempt_schedule_notrace, __preempt_schedule_notrace_func);
5398         static_call_update(irqentry_exit_cond_resched, irqentry_exit_cond_resched);
5399 
5400         switch (mode) {
5401         case preempt_dynamic_none:
5402                 static_call_update(cond_resched, __cond_resched);
5403                 static_call_update(might_resched, (typeof(&__cond_resched)) __static_call_return0);
5404                 static_call_update(preempt_schedule, (typeof(&preempt_schedule)) NULL);
5405                 static_call_update(preempt_schedule_notrace, (typeof(&preempt_schedule_notrace)) NULL);
5406                 static_call_update(irqentry_exit_cond_resched, (typeof(&irqentry_exit_cond_resched)) NULL);
5407                 pr_info("Dynamic Preempt: none\n");
5408                 break;
5409 
5410         case preempt_dynamic_voluntary:
5411                 static_call_update(cond_resched, __cond_resched);
5412                 static_call_update(might_resched, __cond_resched);
5413                 static_call_update(preempt_schedule, (typeof(&preempt_schedule)) NULL);
5414                 static_call_update(preempt_schedule_notrace, (typeof(&preempt_schedule_notrace)) NULL);
5415                 static_call_update(irqentry_exit_cond_resched, (typeof(&irqentry_exit_cond_resched)) NULL);
5416                 pr_info("Dynamic Preempt: voluntary\n");
5417                 break;
5418 
5419         case preempt_dynamic_full:
5420                 static_call_update(cond_resched, (typeof(&__cond_resched)) __static_call_return0);
5421                 static_call_update(might_resched, (typeof(&__cond_resched)) __static_call_return0);
5422                 static_call_update(preempt_schedule, __preempt_schedule_func);
5423                 static_call_update(preempt_schedule_notrace, __preempt_schedule_notrace_func);
5424                 static_call_update(irqentry_exit_cond_resched, irqentry_exit_cond_resched);
5425                 pr_info("Dynamic Preempt: full\n");
5426                 break;
5427         }
5428 
5429         preempt_dynamic_mode = mode;
5430 }
5431 
5432 static int __init setup_preempt_mode(char *str)
5433 {
5434         int mode = sched_dynamic_mode(str);
5435         if (mode < 0) {
5436                 pr_warn("Dynamic Preempt: unsupported mode: %s\n", str);
5437                 return 1;
5438         }
5439 
5440         sched_dynamic_update(mode);
5441         return 0;
5442 }
5443 __setup("preempt=", setup_preempt_mode);
5444 
5445 #ifdef CONFIG_SCHED_DEBUG
5446 
5447 static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf,
5448                                    size_t cnt, loff_t *ppos)
5449 {
5450         char buf[16];
5451         int mode;
5452 
5453         if (cnt > 15)
5454                 cnt = 15;
5455 
5456         if (copy_from_user(&buf, ubuf, cnt))
5457                 return -EFAULT;
5458 
5459         buf[cnt] = 0;
5460         mode = sched_dynamic_mode(strstrip(buf));
5461         if (mode < 0)
5462                 return mode;
5463 
5464         sched_dynamic_update(mode);
5465 
5466         *ppos += cnt;
5467 
5468         return cnt;
5469 }
5470 
5471 static int sched_dynamic_show(struct seq_file *m, void *v)
5472 {
5473         static const char * preempt_modes[] = {
5474                 "none", "voluntary", "full"
5475         };
5476         int i;
5477 
5478         for (i = 0; i < ARRAY_SIZE(preempt_modes); i++) {
5479                 if (preempt_dynamic_mode == i)
5480                         seq_puts(m, "(");
5481                 seq_puts(m, preempt_modes[i]);
5482                 if (preempt_dynamic_mode == i)
5483                         seq_puts(m, ")");
5484 
5485                 seq_puts(m, " ");
5486         }
5487 
5488         seq_puts(m, "\n");
5489         return 0;
5490 }
5491 
5492 static int sched_dynamic_open(struct inode *inode, struct file *filp)
5493 {
5494         return single_open(filp, sched_dynamic_show, NULL);
5495 }
5496 
5497 static const struct file_operations sched_dynamic_fops = {
5498         .open           = sched_dynamic_open,
5499         .write          = sched_dynamic_write,
5500         .read           = seq_read,
5501         .llseek         = seq_lseek,
5502         .release        = single_release,
5503 };
5504 
5505 static __init int sched_init_debug_dynamic(void)
5506 {
5507         debugfs_create_file("sched_preempt", 0644, NULL, NULL, &sched_dynamic_fops);
5508         return 0;
5509 }
5510 late_initcall(sched_init_debug_dynamic);
5511 
5512 #endif /* CONFIG_SCHED_DEBUG */
5513 #endif /* CONFIG_PREEMPT_DYNAMIC */
5514 
5515 
5516 /*
5517  * This is the entry point to schedule() from kernel preemption
5518  * off of irq context.
5519  * Note, that this is called and return with irqs disabled. This will
5520  * protect us against recursive calling from irq.
5521  */
5522 asmlinkage __visible void __sched preempt_schedule_irq(void)
5523 {
5524         enum ctx_state prev_state;
5525 
5526         /* Catch callers which need to be fixed */
5527         BUG_ON(preempt_count() || !irqs_disabled());
5528 
5529         prev_state = exception_enter();
5530 
5531         do {
5532                 preempt_disable();
5533                 local_irq_enable();
5534                 __schedule(true);
5535                 local_irq_disable();
5536                 sched_preempt_enable_no_resched();
5537         } while (need_resched());
5538 
5539         exception_exit(prev_state);
5540 }
5541 
5542 int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags,
5543                           void *key)
5544 {
5545         WARN_ON_ONCE(IS_ENABLED(CONFIG_SCHED_DEBUG) && wake_flags & ~WF_SYNC);
5546         return try_to_wake_up(curr->private, mode, wake_flags);
5547 }
5548 EXPORT_SYMBOL(default_wake_function);
5549 
5550 #ifdef CONFIG_RT_MUTEXES
5551 
5552 static inline int __rt_effective_prio(struct task_struct *pi_task, int prio)
5553 {
5554         if (pi_task)
5555                 prio = min(prio, pi_task->prio);
5556 
5557         return prio;
5558 }
5559 
5560 static inline int rt_effective_prio(struct task_struct *p, int prio)
5561 {
5562         struct task_struct *pi_task = rt_mutex_get_top_task(p);
5563 
5564         return __rt_effective_prio(pi_task, prio);
5565 }
5566 
5567 /*
5568  * rt_mutex_setprio - set the current priority of a task
5569  * @p: task to boost
5570  * @pi_task: donor task
5571  *
5572  * This function changes the 'effective' priority of a task. It does
5573  * not touch ->normal_prio like __setscheduler().
5574  *
5575  * Used by the rt_mutex code to implement priority inheritance
5576  * logic. Call site only calls if the priority of the task changed.
5577  */
5578 void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task)
5579 {
5580         int prio, oldprio, queued, running, queue_flag =
5581                 DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
5582         const struct sched_class *prev_class;
5583         struct rq_flags rf;
5584         struct rq *rq;
5585 
5586         /* XXX used to be waiter->prio, not waiter->task->prio */
5587         prio = __rt_effective_prio(pi_task, p->normal_prio);
5588 
5589         /*
5590          * If nothing changed; bail early.
5591          */
5592         if (p->pi_top_task == pi_task && prio == p->prio && !dl_prio(prio))
5593                 return;
5594 
5595         rq = __task_rq_lock(p, &rf);
5596         update_rq_clock(rq);
5597         /*
5598          * Set under pi_lock && rq->lock, such that the value can be used under
5599          * either lock.
5600          *
5601          * Note that there is loads of tricky to make this pointer cache work
5602          * right. rt_mutex_slowunlock()+rt_mutex_postunlock() work together to
5603          * ensure a task is de-boosted (pi_task is set to NULL) before the
5604          * task is allowed to run again (and can exit). This ensures the pointer
5605          * points to a blocked task -- which guarantees the task is present.
5606          */
5607         p->pi_top_task = pi_task;
5608 
5609         /*
5610          * For FIFO/RR we only need to set prio, if that matches we're done.
5611          */
5612         if (prio == p->prio && !dl_prio(prio))
5613                 goto out_unlock;
5614 
5615         /*
5616          * Idle task boosting is a nono in general. There is one
5617          * exception, when PREEMPT_RT and NOHZ is active:
5618          *
5619          * The idle task calls get_next_timer_interrupt() and holds
5620          * the timer wheel base->lock on the CPU and another CPU wants
5621          * to access the timer (probably to cancel it). We can safely
5622          * ignore the boosting request, as the idle CPU runs this code
5623          * with interrupts disabled and will complete the lock
5624          * protected section without being interrupted. So there is no
5625          * real need to boost.
5626          */
5627         if (unlikely(p == rq->idle)) {
5628                 WARN_ON(p != rq->curr);
5629                 WARN_ON(p->pi_blocked_on);
5630                 goto out_unlock;
5631         }
5632 
5633         trace_sched_pi_setprio(p, pi_task);
5634         oldprio = p->prio;
5635 
5636         if (oldprio == prio)
5637                 queue_flag &= ~DEQUEUE_MOVE;
5638 
5639         prev_class = p->sched_class;
5640         queued = task_on_rq_queued(p);
5641         running = task_current(rq, p);
5642         if (queued)
5643                 dequeue_task(rq, p, queue_flag);
5644         if (running)
5645                 put_prev_task(rq, p);
5646 
5647         /*
5648          * Boosting condition are:
5649          * 1. -rt task is running and holds mutex A
5650          *      --> -dl task blocks on mutex A
5651          *
5652          * 2. -dl task is running and holds mutex A
5653          *      --> -dl task blocks on mutex A and could preempt the
5654          *          running task
5655          */
5656         if (dl_prio(prio)) {
5657                 if (!dl_prio(p->normal_prio) ||
5658                     (pi_task && dl_prio(pi_task->prio) &&
5659                      dl_entity_preempt(&pi_task->dl, &p->dl))) {
5660                         p->dl.pi_se = pi_task->dl.pi_se;
5661                         queue_flag |= ENQUEUE_REPLENISH;
5662                 } else {
5663                         p->dl.pi_se = &p->dl;
5664                 }
5665                 p->sched_class = &dl_sched_class;
5666         } else if (rt_prio(prio)) {
5667                 if (dl_prio(oldprio))
5668                         p->dl.pi_se = &p->dl;
5669                 if (oldprio < prio)
5670                         queue_flag |= ENQUEUE_HEAD;
5671                 p->sched_class = &rt_sched_class;
5672         } else {
5673                 if (dl_prio(oldprio))
5674                         p->dl.pi_se = &p->dl;
5675                 if (rt_prio(oldprio))
5676                         p->rt.timeout = 0;
5677                 p->sched_class = &fair_sched_class;
5678         }
5679 
5680         p->prio = prio;
5681 
5682         if (queued)
5683                 enqueue_task(rq, p, queue_flag);
5684         if (running)
5685                 set_next_task(rq, p);
5686 
5687         check_class_changed(rq, p, prev_class, oldprio);
5688 out_unlock:
5689         /* Avoid rq from going away on us: */
5690         preempt_disable();
5691 
5692         rq_unpin_lock(rq, &rf);
5693         __balance_callbacks(rq);
5694         raw_spin_unlock(&rq->lock);
5695 
5696         preempt_enable();
5697 }
5698 #else
5699 static inline int rt_effective_prio(struct task_struct *p, int prio)
5700 {
5701         return prio;
5702 }
5703 #endif
5704 
5705 void set_user_nice(struct task_struct *p, long nice)
5706 {
5707         bool queued, running;
5708         int old_prio;
5709         struct rq_flags rf;
5710         struct rq *rq;
5711 
5712         if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE)
5713                 return;
5714         /*
5715          * We have to be careful, if called from sys_setpriority(),
5716          * the task might be in the middle of scheduling on another CPU.
5717          */
5718         rq = task_rq_lock(p, &rf);
5719         update_rq_clock(rq);
5720 
5721         /*
5722          * The RT priorities are set via sched_setscheduler(), but we still
5723          * allow the 'normal' nice value to be set - but as expected
5724          * it won't have any effect on scheduling until the task is
5725          * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR:
5726          */
5727         if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
5728                 p->static_prio = NICE_TO_PRIO(nice);
5729                 goto out_unlock;
5730         }
5731         queued = task_on_rq_queued(p);
5732         running = task_current(rq, p);
5733         if (queued)
5734                 dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK);
5735         if (running)
5736                 put_prev_task(rq, p);
5737 
5738         p->static_prio = NICE_TO_PRIO(nice);
5739         set_load_weight(p, true);
5740         old_prio = p->prio;
5741         p->prio = effective_prio(p);
5742 
5743         if (queued)
5744                 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
5745         if (running)
5746                 set_next_task(rq, p);
5747 
5748         /*
5749          * If the task increased its priority or is running and
5750          * lowered its priority, then reschedule its CPU:
5751          */
5752         p->sched_class->prio_changed(rq, p, old_prio);
5753 
5754 out_unlock:
5755         task_rq_unlock(rq, p, &rf);
5756 }
5757 EXPORT_SYMBOL(set_user_nice);
5758 
5759 /*
5760  * can_nice - check if a task can reduce its nice value
5761  * @p: task
5762  * @nice: nice value
5763  */
5764 int can_nice(const struct task_struct *p, const int nice)
5765 {
5766         /* Convert nice value [19,-20] to rlimit style value [1,40]: */
5767         int nice_rlim = nice_to_rlimit(nice);
5768 
5769         return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
5770                 capable(CAP_SYS_NICE));
5771 }
5772 
5773 #ifdef __ARCH_WANT_SYS_NICE
5774 
5775 /*
5776  * sys_nice - change the priority of the current process.
5777  * @increment: priority increment
5778  *
5779  * sys_setpriority is a more generic, but much slower function that
5780  * does similar things.
5781  */
5782 SYSCALL_DEFINE1(nice, int, increment)
5783 {
5784         long nice, retval;
5785         if (!ccs_capable(CCS_SYS_NICE))
5786                 return -EPERM;
5787 
5788         /*
5789          * Setpriority might change our priority at the same moment.
5790          * We don't have to worry. Conceptually one call occurs first
5791          * and we have a single winner.
5792          */
5793         increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH);
5794         nice = task_nice(current) + increment;
5795 
5796         nice = clamp_val(nice, MIN_NICE, MAX_NICE);
5797         if (increment < 0 && !can_nice(current, nice))
5798                 return -EPERM;
5799 
5800         retval = security_task_setnice(current, nice);
5801         if (retval)
5802                 return retval;
5803 
5804         set_user_nice(current, nice);
5805         return 0;
5806 }
5807 
5808 #endif
5809 
5810 /**
5811  * task_prio - return the priority value of a given task.
5812  * @p: the task in question.
5813  *
5814  * Return: The priority value as seen by users in /proc.
5815  *
5816  * sched policy         return value   kernel prio    user prio/nice
5817  *
5818  * normal, batch, idle     [0 ... 39]  [100 ... 139]          0/[-20 ... 19]
5819  * fifo, rr             [-2 ... -100]     [98 ... 0]  [1 ... 99]
5820  * deadline                     -101             -1           0
5821  */
5822 int task_prio(const struct task_struct *p)
5823 {
5824         return p->prio - MAX_RT_PRIO;
5825 }
5826 
5827 /**
5828  * idle_cpu - is a given CPU idle currently?
5829  * @cpu: the processor in question.
5830  *
5831  * Return: 1 if the CPU is currently idle. 0 otherwise.
5832  */
5833 int idle_cpu(int cpu)
5834 {
5835         struct rq *rq = cpu_rq(cpu);
5836 
5837         if (rq->curr != rq->idle)
5838                 return 0;
5839 
5840         if (rq->nr_running)
5841                 return 0;
5842 
5843 #ifdef CONFIG_SMP
5844         if (rq->ttwu_pending)
5845                 return 0;
5846 #endif
5847 
5848         return 1;
5849 }
5850 
5851 /**
5852  * available_idle_cpu - is a given CPU idle for enqueuing work.
5853  * @cpu: the CPU in question.
5854  *
5855  * Return: 1 if the CPU is currently idle. 0 otherwise.
5856  */
5857 int available_idle_cpu(int cpu)
5858 {
5859         if (!idle_cpu(cpu))
5860                 return 0;
5861 
5862         if (vcpu_is_preempted(cpu))
5863                 return 0;
5864 
5865         return 1;
5866 }
5867 
5868 /**
5869  * idle_task - return the idle task for a given CPU.
5870  * @cpu: the processor in question.
5871  *
5872  * Return: The idle task for the CPU @cpu.
5873  */
5874 struct task_struct *idle_task(int cpu)
5875 {
5876         return cpu_rq(cpu)->idle;
5877 }
5878 
5879 #ifdef CONFIG_SMP
5880 /*
5881  * This function computes an effective utilization for the given CPU, to be
5882  * used for frequency selection given the linear relation: f = u * f_max.
5883  *
5884  * The scheduler tracks the following metrics:
5885  *
5886  *   cpu_util_{cfs,rt,dl,irq}()
5887  *   cpu_bw_dl()
5888  *
5889  * Where the cfs,rt and dl util numbers are tracked with the same metric and
5890  * synchronized windows and are thus directly comparable.
5891  *
5892  * The cfs,rt,dl utilization are the running times measured with rq->clock_task
5893  * which excludes things like IRQ and steal-time. These latter are then accrued
5894  * in the irq utilization.
5895  *
5896  * The DL bandwidth number otoh is not a measured metric but a value computed
5897  * based on the task model parameters and gives the minimal utilization
5898  * required to meet deadlines.
5899  */
5900 unsigned long effective_cpu_util(int cpu, unsigned long util_cfs,
5901                                  unsigned long max, enum cpu_util_type type,
5902                                  struct task_struct *p)
5903 {
5904         unsigned long dl_util, util, irq;
5905         struct rq *rq = cpu_rq(cpu);
5906 
5907         if (!uclamp_is_used() &&
5908             type == FREQUENCY_UTIL && rt_rq_is_runnable(&rq->rt)) {
5909                 return max;
5910         }
5911 
5912         /*
5913          * Early check to see if IRQ/steal time saturates the CPU, can be
5914          * because of inaccuracies in how we track these -- see
5915          * update_irq_load_avg().
5916          */
5917         irq = cpu_util_irq(rq);
5918         if (unlikely(irq >= max))
5919                 return max;
5920 
5921         /*
5922          * Because the time spend on RT/DL tasks is visible as 'lost' time to
5923          * CFS tasks and we use the same metric to track the effective
5924          * utilization (PELT windows are synchronized) we can directly add them
5925          * to obtain the CPU's actual utilization.
5926          *
5927          * CFS and RT utilization can be boosted or capped, depending on
5928          * utilization clamp constraints requested by currently RUNNABLE
5929          * tasks.
5930          * When there are no CFS RUNNABLE tasks, clamps are released and
5931          * frequency will be gracefully reduced with the utilization decay.
5932          */
5933         util = util_cfs + cpu_util_rt(rq);
5934         if (type == FREQUENCY_UTIL)
5935                 util = uclamp_rq_util_with(rq, util, p);
5936 
5937         dl_util = cpu_util_dl(rq);
5938 
5939         /*
5940          * For frequency selection we do not make cpu_util_dl() a permanent part
5941          * of this sum because we want to use cpu_bw_dl() later on, but we need
5942          * to check if the CFS+RT+DL sum is saturated (ie. no idle time) such
5943          * that we select f_max when there is no idle time.
5944          *
5945          * NOTE: numerical errors or stop class might cause us to not quite hit
5946          * saturation when we should -- something for later.
5947          */
5948         if (util + dl_util >= max)
5949                 return max;
5950 
5951         /*
5952          * OTOH, for energy computation we need the estimated running time, so
5953          * include util_dl and ignore dl_bw.
5954          */
5955         if (type == ENERGY_UTIL)
5956                 util += dl_util;
5957 
5958         /*
5959          * There is still idle time; further improve the number by using the
5960          * irq metric. Because IRQ/steal time is hidden from the task clock we
5961          * need to scale the task numbers:
5962          *
5963          *              max - irq
5964          *   U' = irq + --------- * U
5965          *                 max
5966          */
5967         util = scale_irq_capacity(util, irq, max);
5968         util += irq;
5969 
5970         /*
5971          * Bandwidth required by DEADLINE must always be granted while, for
5972          * FAIR and RT, we use blocked utilization of IDLE CPUs as a mechanism
5973          * to gracefully reduce the frequency when no tasks show up for longer
5974          * periods of time.
5975          *
5976          * Ideally we would like to set bw_dl as min/guaranteed freq and util +
5977          * bw_dl as requested freq. However, cpufreq is not yet ready for such
5978          * an interface. So, we only do the latter for now.
5979          */
5980         if (type == FREQUENCY_UTIL)
5981                 util += cpu_bw_dl(rq);
5982 
5983         return min(max, util);
5984 }
5985 
5986 unsigned long sched_cpu_util(int cpu, unsigned long max)
5987 {
5988         return effective_cpu_util(cpu, cpu_util_cfs(cpu_rq(cpu)), max,
5989                                   ENERGY_UTIL, NULL);
5990 }
5991 #endif /* CONFIG_SMP */
5992 
5993 /**
5994  * find_process_by_pid - find a process with a matching PID value.
5995  * @pid: the pid in question.
5996  *
5997  * The task of @pid, if found. %NULL otherwise.
5998  */
5999 static struct task_struct *find_process_by_pid(pid_t pid)
6000 {
6001         return pid ? find_task_by_vpid(pid) : current;
6002 }
6003 
6004 /*
6005  * sched_setparam() passes in -1 for its policy, to let the functions
6006  * it calls know not to change it.
6007  */
6008 #define SETPARAM_POLICY -1
6009 
6010 static void __setscheduler_params(struct task_struct *p,
6011                 const struct sched_attr *attr)
6012 {
6013         int policy = attr->sched_policy;
6014 
6015         if (policy == SETPARAM_POLICY)
6016                 policy = p->policy;
6017 
6018         p->policy = policy;
6019 
6020         if (dl_policy(policy))
6021                 __setparam_dl(p, attr);
6022         else if (fair_policy(policy))
6023                 p->static_prio = NICE_TO_PRIO(attr->sched_nice);
6024 
6025         /*
6026          * __sched_setscheduler() ensures attr->sched_priority == 0 when
6027          * !rt_policy. Always setting this ensures that things like
6028          * getparam()/getattr() don't report silly values for !rt tasks.
6029          */
6030         p->rt_priority = attr->sched_priority;
6031         p->normal_prio = normal_prio(p);
6032         set_load_weight(p, true);
6033 }
6034 
6035 /* Actually do priority change: must hold pi & rq lock. */
6036 static void __setscheduler(struct rq *rq, struct task_struct *p,
6037                            const struct sched_attr *attr, bool keep_boost)
6038 {
6039         /*
6040          * If params can't change scheduling class changes aren't allowed
6041          * either.
6042          */
6043         if (attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)
6044                 return;
6045 
6046         __setscheduler_params(p, attr);
6047 
6048         /*
6049          * Keep a potential priority boosting if called from
6050          * sched_setscheduler().
6051          */
6052         p->prio = normal_prio(p);
6053         if (keep_boost)
6054                 p->prio = rt_effective_prio(p, p->prio);
6055 
6056         if (dl_prio(p->prio))
6057                 p->sched_class = &dl_sched_class;
6058         else if (rt_prio(p->prio))
6059                 p->sched_class = &rt_sched_class;
6060         else
6061                 p->sched_class = &fair_sched_class;
6062 }
6063 
6064 /*
6065  * Check the target process has a UID that matches the current process's:
6066  */
6067 static bool check_same_owner(struct task_struct *p)
6068 {
6069         const struct cred *cred = current_cred(), *pcred;
6070         bool match;
6071 
6072         rcu_read_lock();
6073         pcred = __task_cred(p);
6074         match = (uid_eq(cred->euid, pcred->euid) ||
6075                  uid_eq(cred->euid, pcred->uid));
6076         rcu_read_unlock();
6077         return match;
6078 }
6079 
6080 static int __sched_setscheduler(struct task_struct *p,
6081                                 const struct sched_attr *attr,
6082                                 bool user, bool pi)
6083 {
6084         int newprio = dl_policy(attr->sched_policy) ? MAX_DL_PRIO - 1 :
6085                       MAX_RT_PRIO - 1 - attr->sched_priority;
6086         int retval, oldprio, oldpolicy = -1, queued, running;
6087         int new_effective_prio, policy = attr->sched_policy;
6088         const struct sched_class *prev_class;
6089         struct callback_head *head;
6090         struct rq_flags rf;
6091         int reset_on_fork;
6092         int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
6093         struct rq *rq;
6094 
6095         /* The pi code expects interrupts enabled */
6096         BUG_ON(pi && in_interrupt());
6097 recheck:
6098         /* Double check policy once rq lock held: */
6099         if (policy < 0) {
6100                 reset_on_fork = p->sched_reset_on_fork;
6101                 policy = oldpolicy = p->policy;
6102         } else {
6103                 reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK);
6104 
6105                 if (!valid_policy(policy))
6106                         return -EINVAL;
6107         }
6108 
6109         if (attr->sched_flags & ~(SCHED_FLAG_ALL | SCHED_FLAG_SUGOV))
6110                 return -EINVAL;
6111 
6112         /*
6113          * Valid priorities for SCHED_FIFO and SCHED_RR are
6114          * 1..MAX_RT_PRIO-1, valid priority for SCHED_NORMAL,
6115          * SCHED_BATCH and SCHED_IDLE is 0.
6116          */
6117         if (attr->sched_priority > MAX_RT_PRIO-1)
6118                 return -EINVAL;
6119         if ((dl_policy(policy) && !__checkparam_dl(attr)) ||
6120             (rt_policy(policy) != (attr->sched_priority != 0)))
6121                 return -EINVAL;
6122 
6123         /*
6124          * Allow unprivileged RT tasks to decrease priority:
6125          */
6126         if (user && !capable(CAP_SYS_NICE)) {
6127                 if (fair_policy(policy)) {
6128                         if (attr->sched_nice < task_nice(p) &&
6129                             !can_nice(p, attr->sched_nice))
6130                                 return -EPERM;
6131                 }
6132 
6133                 if (rt_policy(policy)) {
6134                         unsigned long rlim_rtprio =
6135                                         task_rlimit(p, RLIMIT_RTPRIO);
6136 
6137                         /* Can't set/change the rt policy: */
6138                         if (policy != p->policy && !rlim_rtprio)
6139                                 return -EPERM;
6140 
6141                         /* Can't increase priority: */
6142                         if (attr->sched_priority > p->rt_priority &&
6143                             attr->sched_priority > rlim_rtprio)
6144                                 return -EPERM;
6145                 }
6146 
6147                  /*
6148                   * Can't set/change SCHED_DEADLINE policy at all for now
6149                   * (safest behavior); in the future we would like to allow
6150                   * unprivileged DL tasks to increase their relative deadline
6151                   * or reduce their runtime (both ways reducing utilization)
6152                   */
6153                 if (dl_policy(policy))
6154                         return -EPERM;
6155 
6156                 /*
6157                  * Treat SCHED_IDLE as nice 20. Only allow a switch to
6158                  * SCHED_NORMAL if the RLIMIT_NICE would normally permit it.
6159                  */
6160                 if (task_has_idle_policy(p) && !idle_policy(policy)) {
6161                         if (!can_nice(p, task_nice(p)))
6162                                 return -EPERM;
6163                 }
6164 
6165                 /* Can't change other user's priorities: */
6166                 if (!check_same_owner(p))
6167                         return -EPERM;
6168 
6169                 /* Normal users shall not reset the sched_reset_on_fork flag: */
6170                 if (p->sched_reset_on_fork && !reset_on_fork)
6171                         return -EPERM;
6172         }
6173 
6174         if (user) {
6175                 if (attr->sched_flags & SCHED_FLAG_SUGOV)
6176                         return -EINVAL;
6177 
6178                 retval = security_task_setscheduler(p);
6179                 if (retval)
6180                         return retval;
6181         }
6182 
6183         /* Update task specific "requested" clamps */
6184         if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) {
6185                 retval = uclamp_validate(p, attr);
6186                 if (retval)
6187                         return retval;
6188         }
6189 
6190         if (pi)
6191                 cpuset_read_lock();
6192 
6193         /*
6194          * Make sure no PI-waiters arrive (or leave) while we are
6195          * changing the priority of the task:
6196          *
6197          * To be able to change p->policy safely, the appropriate
6198          * runqueue lock must be held.
6199          */
6200         rq = task_rq_lock(p, &rf);
6201         update_rq_clock(rq);
6202 
6203         /*
6204          * Changing the policy of the stop threads its a very bad idea:
6205          */
6206         if (p == rq->stop) {
6207                 retval = -EINVAL;
6208                 goto unlock;
6209         }
6210 
6211         /*
6212          * If not changing anything there's no need to proceed further,
6213          * but store a possible modification of reset_on_fork.
6214          */
6215         if (unlikely(policy == p->policy)) {
6216                 if (fair_policy(policy) && attr->sched_nice != task_nice(p))
6217                         goto change;
6218                 if (rt_policy(policy) && attr->sched_priority != p->rt_priority)
6219                         goto change;
6220                 if (dl_policy(policy) && dl_param_changed(p, attr))
6221                         goto change;
6222                 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP)
6223                         goto change;
6224 
6225                 p->sched_reset_on_fork = reset_on_fork;
6226                 retval = 0;
6227                 goto unlock;
6228         }
6229 change:
6230 
6231         if (user) {
6232 #ifdef CONFIG_RT_GROUP_SCHED
6233                 /*
6234                  * Do not allow realtime tasks into groups that have no runtime
6235                  * assigned.
6236                  */
6237                 if (rt_bandwidth_enabled() && rt_policy(policy) &&
6238                                 task_group(p)->rt_bandwidth.rt_runtime == 0 &&
6239                                 !task_group_is_autogroup(task_group(p))) {
6240                         retval = -EPERM;