~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/sched/core.c

Version: ~ [ linux-5.12-rc5 ] ~ [ linux-5.11.11 ] ~ [ linux-5.10.27 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.109 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.184 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.228 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.264 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.264 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-only
  2 /*
  3  *  kernel/sched/core.c
  4  *
  5  *  Core kernel scheduler code and related syscalls
  6  *
  7  *  Copyright (C) 1991-2002  Linus Torvalds
  8  */
  9 #include "sched.h"
 10 
 11 #include <linux/nospec.h>
 12 
 13 #include <linux/kcov.h>
 14 
 15 #include <asm/switch_to.h>
 16 #include <asm/tlb.h>
 17 
 18 #include "../workqueue_internal.h"
 19 #include "../smpboot.h"
 20 
 21 #include "pelt.h"
 22 
 23 #define CREATE_TRACE_POINTS
 24 #include <trace/events/sched.h>
 25 
 26 /*
 27  * Export tracepoints that act as a bare tracehook (ie: have no trace event
 28  * associated with them) to allow external modules to probe them.
 29  */
 30 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_cfs_tp);
 31 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_rt_tp);
 32 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_dl_tp);
 33 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_irq_tp);
 34 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_se_tp);
 35 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_overutilized_tp);
 36 
 37 DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
 38 
 39 #if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_JUMP_LABEL)
 40 /*
 41  * Debugging: various feature bits
 42  *
 43  * If SCHED_DEBUG is disabled, each compilation unit has its own copy of
 44  * sysctl_sched_features, defined in sched.h, to allow constants propagation
 45  * at compile time and compiler optimization based on features default.
 46  */
 47 #define SCHED_FEAT(name, enabled)       \
 48         (1UL << __SCHED_FEAT_##name) * enabled |
 49 const_debug unsigned int sysctl_sched_features =
 50 #include "features.h"
 51         0;
 52 #undef SCHED_FEAT
 53 #endif
 54 
 55 /*
 56  * Number of tasks to iterate in a single balance run.
 57  * Limited because this is done with IRQs disabled.
 58  */
 59 const_debug unsigned int sysctl_sched_nr_migrate = 32;
 60 
 61 /*
 62  * period over which we measure -rt task CPU usage in us.
 63  * default: 1s
 64  */
 65 unsigned int sysctl_sched_rt_period = 1000000;
 66 
 67 __read_mostly int scheduler_running;
 68 
 69 /*
 70  * part of the period that we allow rt tasks to run in us.
 71  * default: 0.95s
 72  */
 73 int sysctl_sched_rt_runtime = 950000;
 74 
 75 /*
 76  * __task_rq_lock - lock the rq @p resides on.
 77  */
 78 struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf)
 79         __acquires(rq->lock)
 80 {
 81         struct rq *rq;
 82 
 83         lockdep_assert_held(&p->pi_lock);
 84 
 85         for (;;) {
 86                 rq = task_rq(p);
 87                 raw_spin_lock(&rq->lock);
 88                 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
 89                         rq_pin_lock(rq, rf);
 90                         return rq;
 91                 }
 92                 raw_spin_unlock(&rq->lock);
 93 
 94                 while (unlikely(task_on_rq_migrating(p)))
 95                         cpu_relax();
 96         }
 97 }
 98 
 99 /*
100  * task_rq_lock - lock p->pi_lock and lock the rq @p resides on.
101  */
102 struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf)
103         __acquires(p->pi_lock)
104         __acquires(rq->lock)
105 {
106         struct rq *rq;
107 
108         for (;;) {
109                 raw_spin_lock_irqsave(&p->pi_lock, rf->flags);
110                 rq = task_rq(p);
111                 raw_spin_lock(&rq->lock);
112                 /*
113                  *      move_queued_task()              task_rq_lock()
114                  *
115                  *      ACQUIRE (rq->lock)
116                  *      [S] ->on_rq = MIGRATING         [L] rq = task_rq()
117                  *      WMB (__set_task_cpu())          ACQUIRE (rq->lock);
118                  *      [S] ->cpu = new_cpu             [L] task_rq()
119                  *                                      [L] ->on_rq
120                  *      RELEASE (rq->lock)
121                  *
122                  * If we observe the old CPU in task_rq_lock(), the acquire of
123                  * the old rq->lock will fully serialize against the stores.
124                  *
125                  * If we observe the new CPU in task_rq_lock(), the address
126                  * dependency headed by '[L] rq = task_rq()' and the acquire
127                  * will pair with the WMB to ensure we then also see migrating.
128                  */
129                 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
130                         rq_pin_lock(rq, rf);
131                         return rq;
132                 }
133                 raw_spin_unlock(&rq->lock);
134                 raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);
135 
136                 while (unlikely(task_on_rq_migrating(p)))
137                         cpu_relax();
138         }
139 }
140 
141 /*
142  * RQ-clock updating methods:
143  */
144 
145 static void update_rq_clock_task(struct rq *rq, s64 delta)
146 {
147 /*
148  * In theory, the compile should just see 0 here, and optimize out the call
149  * to sched_rt_avg_update. But I don't trust it...
150  */
151         s64 __maybe_unused steal = 0, irq_delta = 0;
152 
153 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
154         irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;
155 
156         /*
157          * Since irq_time is only updated on {soft,}irq_exit, we might run into
158          * this case when a previous update_rq_clock() happened inside a
159          * {soft,}irq region.
160          *
161          * When this happens, we stop ->clock_task and only update the
162          * prev_irq_time stamp to account for the part that fit, so that a next
163          * update will consume the rest. This ensures ->clock_task is
164          * monotonic.
165          *
166          * It does however cause some slight miss-attribution of {soft,}irq
167          * time, a more accurate solution would be to update the irq_time using
168          * the current rq->clock timestamp, except that would require using
169          * atomic ops.
170          */
171         if (irq_delta > delta)
172                 irq_delta = delta;
173 
174         rq->prev_irq_time += irq_delta;
175         delta -= irq_delta;
176 #endif
177 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
178         if (static_key_false((&paravirt_steal_rq_enabled))) {
179                 steal = paravirt_steal_clock(cpu_of(rq));
180                 steal -= rq->prev_steal_time_rq;
181 
182                 if (unlikely(steal > delta))
183                         steal = delta;
184 
185                 rq->prev_steal_time_rq += steal;
186                 delta -= steal;
187         }
188 #endif
189 
190         rq->clock_task += delta;
191 
192 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ
193         if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY))
194                 update_irq_load_avg(rq, irq_delta + steal);
195 #endif
196         update_rq_clock_pelt(rq, delta);
197 }
198 
199 void update_rq_clock(struct rq *rq)
200 {
201         s64 delta;
202 
203         lockdep_assert_held(&rq->lock);
204 
205         if (rq->clock_update_flags & RQCF_ACT_SKIP)
206                 return;
207 
208 #ifdef CONFIG_SCHED_DEBUG
209         if (sched_feat(WARN_DOUBLE_CLOCK))
210                 SCHED_WARN_ON(rq->clock_update_flags & RQCF_UPDATED);
211         rq->clock_update_flags |= RQCF_UPDATED;
212 #endif
213 
214         delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
215         if (delta < 0)
216                 return;
217         rq->clock += delta;
218         update_rq_clock_task(rq, delta);
219 }
220 
221 
222 #ifdef CONFIG_SCHED_HRTICK
223 /*
224  * Use HR-timers to deliver accurate preemption points.
225  */
226 
227 static void hrtick_clear(struct rq *rq)
228 {
229         if (hrtimer_active(&rq->hrtick_timer))
230                 hrtimer_cancel(&rq->hrtick_timer);
231 }
232 
233 /*
234  * High-resolution timer tick.
235  * Runs from hardirq context with interrupts disabled.
236  */
237 static enum hrtimer_restart hrtick(struct hrtimer *timer)
238 {
239         struct rq *rq = container_of(timer, struct rq, hrtick_timer);
240         struct rq_flags rf;
241 
242         WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
243 
244         rq_lock(rq, &rf);
245         update_rq_clock(rq);
246         rq->curr->sched_class->task_tick(rq, rq->curr, 1);
247         rq_unlock(rq, &rf);
248 
249         return HRTIMER_NORESTART;
250 }
251 
252 #ifdef CONFIG_SMP
253 
254 static void __hrtick_restart(struct rq *rq)
255 {
256         struct hrtimer *timer = &rq->hrtick_timer;
257 
258         hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED);
259 }
260 
261 /*
262  * called from hardirq (IPI) context
263  */
264 static void __hrtick_start(void *arg)
265 {
266         struct rq *rq = arg;
267         struct rq_flags rf;
268 
269         rq_lock(rq, &rf);
270         __hrtick_restart(rq);
271         rq->hrtick_csd_pending = 0;
272         rq_unlock(rq, &rf);
273 }
274 
275 /*
276  * Called to set the hrtick timer state.
277  *
278  * called with rq->lock held and irqs disabled
279  */
280 void hrtick_start(struct rq *rq, u64 delay)
281 {
282         struct hrtimer *timer = &rq->hrtick_timer;
283         ktime_t time;
284         s64 delta;
285 
286         /*
287          * Don't schedule slices shorter than 10000ns, that just
288          * doesn't make sense and can cause timer DoS.
289          */
290         delta = max_t(s64, delay, 10000LL);
291         time = ktime_add_ns(timer->base->get_time(), delta);
292 
293         hrtimer_set_expires(timer, time);
294 
295         if (rq == this_rq()) {
296                 __hrtick_restart(rq);
297         } else if (!rq->hrtick_csd_pending) {
298                 smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd);
299                 rq->hrtick_csd_pending = 1;
300         }
301 }
302 
303 #else
304 /*
305  * Called to set the hrtick timer state.
306  *
307  * called with rq->lock held and irqs disabled
308  */
309 void hrtick_start(struct rq *rq, u64 delay)
310 {
311         /*
312          * Don't schedule slices shorter than 10000ns, that just
313          * doesn't make sense. Rely on vruntime for fairness.
314          */
315         delay = max_t(u64, delay, 10000LL);
316         hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay),
317                       HRTIMER_MODE_REL_PINNED);
318 }
319 #endif /* CONFIG_SMP */
320 
321 static void hrtick_rq_init(struct rq *rq)
322 {
323 #ifdef CONFIG_SMP
324         rq->hrtick_csd_pending = 0;
325 
326         rq->hrtick_csd.flags = 0;
327         rq->hrtick_csd.func = __hrtick_start;
328         rq->hrtick_csd.info = rq;
329 #endif
330 
331         hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
332         rq->hrtick_timer.function = hrtick;
333 }
334 #else   /* CONFIG_SCHED_HRTICK */
335 static inline void hrtick_clear(struct rq *rq)
336 {
337 }
338 
339 static inline void hrtick_rq_init(struct rq *rq)
340 {
341 }
342 #endif  /* CONFIG_SCHED_HRTICK */
343 
344 /*
345  * cmpxchg based fetch_or, macro so it works for different integer types
346  */
347 #define fetch_or(ptr, mask)                                             \
348         ({                                                              \
349                 typeof(ptr) _ptr = (ptr);                               \
350                 typeof(mask) _mask = (mask);                            \
351                 typeof(*_ptr) _old, _val = *_ptr;                       \
352                                                                         \
353                 for (;;) {                                              \
354                         _old = cmpxchg(_ptr, _val, _val | _mask);       \
355                         if (_old == _val)                               \
356                                 break;                                  \
357                         _val = _old;                                    \
358                 }                                                       \
359         _old;                                                           \
360 })
361 
362 #if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG)
363 /*
364  * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG,
365  * this avoids any races wrt polling state changes and thereby avoids
366  * spurious IPIs.
367  */
368 static bool set_nr_and_not_polling(struct task_struct *p)
369 {
370         struct thread_info *ti = task_thread_info(p);
371         return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG);
372 }
373 
374 /*
375  * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set.
376  *
377  * If this returns true, then the idle task promises to call
378  * sched_ttwu_pending() and reschedule soon.
379  */
380 static bool set_nr_if_polling(struct task_struct *p)
381 {
382         struct thread_info *ti = task_thread_info(p);
383         typeof(ti->flags) old, val = READ_ONCE(ti->flags);
384 
385         for (;;) {
386                 if (!(val & _TIF_POLLING_NRFLAG))
387                         return false;
388                 if (val & _TIF_NEED_RESCHED)
389                         return true;
390                 old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED);
391                 if (old == val)
392                         break;
393                 val = old;
394         }
395         return true;
396 }
397 
398 #else
399 static bool set_nr_and_not_polling(struct task_struct *p)
400 {
401         set_tsk_need_resched(p);
402         return true;
403 }
404 
405 #ifdef CONFIG_SMP
406 static bool set_nr_if_polling(struct task_struct *p)
407 {
408         return false;
409 }
410 #endif
411 #endif
412 
413 static bool __wake_q_add(struct wake_q_head *head, struct task_struct *task)
414 {
415         struct wake_q_node *node = &task->wake_q;
416 
417         /*
418          * Atomically grab the task, if ->wake_q is !nil already it means
419          * its already queued (either by us or someone else) and will get the
420          * wakeup due to that.
421          *
422          * In order to ensure that a pending wakeup will observe our pending
423          * state, even in the failed case, an explicit smp_mb() must be used.
424          */
425         smp_mb__before_atomic();
426         if (unlikely(cmpxchg_relaxed(&node->next, NULL, WAKE_Q_TAIL)))
427                 return false;
428 
429         /*
430          * The head is context local, there can be no concurrency.
431          */
432         *head->lastp = node;
433         head->lastp = &node->next;
434         return true;
435 }
436 
437 /**
438  * wake_q_add() - queue a wakeup for 'later' waking.
439  * @head: the wake_q_head to add @task to
440  * @task: the task to queue for 'later' wakeup
441  *
442  * Queue a task for later wakeup, most likely by the wake_up_q() call in the
443  * same context, _HOWEVER_ this is not guaranteed, the wakeup can come
444  * instantly.
445  *
446  * This function must be used as-if it were wake_up_process(); IOW the task
447  * must be ready to be woken at this location.
448  */
449 void wake_q_add(struct wake_q_head *head, struct task_struct *task)
450 {
451         if (__wake_q_add(head, task))
452                 get_task_struct(task);
453 }
454 
455 /**
456  * wake_q_add_safe() - safely queue a wakeup for 'later' waking.
457  * @head: the wake_q_head to add @task to
458  * @task: the task to queue for 'later' wakeup
459  *
460  * Queue a task for later wakeup, most likely by the wake_up_q() call in the
461  * same context, _HOWEVER_ this is not guaranteed, the wakeup can come
462  * instantly.
463  *
464  * This function must be used as-if it were wake_up_process(); IOW the task
465  * must be ready to be woken at this location.
466  *
467  * This function is essentially a task-safe equivalent to wake_q_add(). Callers
468  * that already hold reference to @task can call the 'safe' version and trust
469  * wake_q to do the right thing depending whether or not the @task is already
470  * queued for wakeup.
471  */
472 void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task)
473 {
474         if (!__wake_q_add(head, task))
475                 put_task_struct(task);
476 }
477 
478 void wake_up_q(struct wake_q_head *head)
479 {
480         struct wake_q_node *node = head->first;
481 
482         while (node != WAKE_Q_TAIL) {
483                 struct task_struct *task;
484 
485                 task = container_of(node, struct task_struct, wake_q);
486                 BUG_ON(!task);
487                 /* Task can safely be re-inserted now: */
488                 node = node->next;
489                 task->wake_q.next = NULL;
490 
491                 /*
492                  * wake_up_process() executes a full barrier, which pairs with
493                  * the queueing in wake_q_add() so as not to miss wakeups.
494                  */
495                 wake_up_process(task);
496                 put_task_struct(task);
497         }
498 }
499 
500 /*
501  * resched_curr - mark rq's current task 'to be rescheduled now'.
502  *
503  * On UP this means the setting of the need_resched flag, on SMP it
504  * might also involve a cross-CPU call to trigger the scheduler on
505  * the target CPU.
506  */
507 void resched_curr(struct rq *rq)
508 {
509         struct task_struct *curr = rq->curr;
510         int cpu;
511 
512         lockdep_assert_held(&rq->lock);
513 
514         if (test_tsk_need_resched(curr))
515                 return;
516 
517         cpu = cpu_of(rq);
518 
519         if (cpu == smp_processor_id()) {
520                 set_tsk_need_resched(curr);
521                 set_preempt_need_resched();
522                 return;
523         }
524 
525         if (set_nr_and_not_polling(curr))
526                 smp_send_reschedule(cpu);
527         else
528                 trace_sched_wake_idle_without_ipi(cpu);
529 }
530 
531 void resched_cpu(int cpu)
532 {
533         struct rq *rq = cpu_rq(cpu);
534         unsigned long flags;
535 
536         raw_spin_lock_irqsave(&rq->lock, flags);
537         if (cpu_online(cpu) || cpu == smp_processor_id())
538                 resched_curr(rq);
539         raw_spin_unlock_irqrestore(&rq->lock, flags);
540 }
541 
542 #ifdef CONFIG_SMP
543 #ifdef CONFIG_NO_HZ_COMMON
544 /*
545  * In the semi idle case, use the nearest busy CPU for migrating timers
546  * from an idle CPU.  This is good for power-savings.
547  *
548  * We don't do similar optimization for completely idle system, as
549  * selecting an idle CPU will add more delays to the timers than intended
550  * (as that CPU's timer base may not be uptodate wrt jiffies etc).
551  */
552 int get_nohz_timer_target(void)
553 {
554         int i, cpu = smp_processor_id();
555         struct sched_domain *sd;
556 
557         if (!idle_cpu(cpu) && housekeeping_cpu(cpu, HK_FLAG_TIMER))
558                 return cpu;
559 
560         rcu_read_lock();
561         for_each_domain(cpu, sd) {
562                 for_each_cpu(i, sched_domain_span(sd)) {
563                         if (cpu == i)
564                                 continue;
565 
566                         if (!idle_cpu(i) && housekeeping_cpu(i, HK_FLAG_TIMER)) {
567                                 cpu = i;
568                                 goto unlock;
569                         }
570                 }
571         }
572 
573         if (!housekeeping_cpu(cpu, HK_FLAG_TIMER))
574                 cpu = housekeeping_any_cpu(HK_FLAG_TIMER);
575 unlock:
576         rcu_read_unlock();
577         return cpu;
578 }
579 
580 /*
581  * When add_timer_on() enqueues a timer into the timer wheel of an
582  * idle CPU then this timer might expire before the next timer event
583  * which is scheduled to wake up that CPU. In case of a completely
584  * idle system the next event might even be infinite time into the
585  * future. wake_up_idle_cpu() ensures that the CPU is woken up and
586  * leaves the inner idle loop so the newly added timer is taken into
587  * account when the CPU goes back to idle and evaluates the timer
588  * wheel for the next timer event.
589  */
590 static void wake_up_idle_cpu(int cpu)
591 {
592         struct rq *rq = cpu_rq(cpu);
593 
594         if (cpu == smp_processor_id())
595                 return;
596 
597         if (set_nr_and_not_polling(rq->idle))
598                 smp_send_reschedule(cpu);
599         else
600                 trace_sched_wake_idle_without_ipi(cpu);
601 }
602 
603 static bool wake_up_full_nohz_cpu(int cpu)
604 {
605         /*
606          * We just need the target to call irq_exit() and re-evaluate
607          * the next tick. The nohz full kick at least implies that.
608          * If needed we can still optimize that later with an
609          * empty IRQ.
610          */
611         if (cpu_is_offline(cpu))
612                 return true;  /* Don't try to wake offline CPUs. */
613         if (tick_nohz_full_cpu(cpu)) {
614                 if (cpu != smp_processor_id() ||
615                     tick_nohz_tick_stopped())
616                         tick_nohz_full_kick_cpu(cpu);
617                 return true;
618         }
619 
620         return false;
621 }
622 
623 /*
624  * Wake up the specified CPU.  If the CPU is going offline, it is the
625  * caller's responsibility to deal with the lost wakeup, for example,
626  * by hooking into the CPU_DEAD notifier like timers and hrtimers do.
627  */
628 void wake_up_nohz_cpu(int cpu)
629 {
630         if (!wake_up_full_nohz_cpu(cpu))
631                 wake_up_idle_cpu(cpu);
632 }
633 
634 static inline bool got_nohz_idle_kick(void)
635 {
636         int cpu = smp_processor_id();
637 
638         if (!(atomic_read(nohz_flags(cpu)) & NOHZ_KICK_MASK))
639                 return false;
640 
641         if (idle_cpu(cpu) && !need_resched())
642                 return true;
643 
644         /*
645          * We can't run Idle Load Balance on this CPU for this time so we
646          * cancel it and clear NOHZ_BALANCE_KICK
647          */
648         atomic_andnot(NOHZ_KICK_MASK, nohz_flags(cpu));
649         return false;
650 }
651 
652 #else /* CONFIG_NO_HZ_COMMON */
653 
654 static inline bool got_nohz_idle_kick(void)
655 {
656         return false;
657 }
658 
659 #endif /* CONFIG_NO_HZ_COMMON */
660 
661 #ifdef CONFIG_NO_HZ_FULL
662 bool sched_can_stop_tick(struct rq *rq)
663 {
664         int fifo_nr_running;
665 
666         /* Deadline tasks, even if single, need the tick */
667         if (rq->dl.dl_nr_running)
668                 return false;
669 
670         /*
671          * If there are more than one RR tasks, we need the tick to effect the
672          * actual RR behaviour.
673          */
674         if (rq->rt.rr_nr_running) {
675                 if (rq->rt.rr_nr_running == 1)
676                         return true;
677                 else
678                         return false;
679         }
680 
681         /*
682          * If there's no RR tasks, but FIFO tasks, we can skip the tick, no
683          * forced preemption between FIFO tasks.
684          */
685         fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running;
686         if (fifo_nr_running)
687                 return true;
688 
689         /*
690          * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left;
691          * if there's more than one we need the tick for involuntary
692          * preemption.
693          */
694         if (rq->nr_running > 1)
695                 return false;
696 
697         return true;
698 }
699 #endif /* CONFIG_NO_HZ_FULL */
700 #endif /* CONFIG_SMP */
701 
702 #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
703                         (defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH)))
704 /*
705  * Iterate task_group tree rooted at *from, calling @down when first entering a
706  * node and @up when leaving it for the final time.
707  *
708  * Caller must hold rcu_lock or sufficient equivalent.
709  */
710 int walk_tg_tree_from(struct task_group *from,
711                              tg_visitor down, tg_visitor up, void *data)
712 {
713         struct task_group *parent, *child;
714         int ret;
715 
716         parent = from;
717 
718 down:
719         ret = (*down)(parent, data);
720         if (ret)
721                 goto out;
722         list_for_each_entry_rcu(child, &parent->children, siblings) {
723                 parent = child;
724                 goto down;
725 
726 up:
727                 continue;
728         }
729         ret = (*up)(parent, data);
730         if (ret || parent == from)
731                 goto out;
732 
733         child = parent;
734         parent = parent->parent;
735         if (parent)
736                 goto up;
737 out:
738         return ret;
739 }
740 
741 int tg_nop(struct task_group *tg, void *data)
742 {
743         return 0;
744 }
745 #endif
746 
747 static void set_load_weight(struct task_struct *p, bool update_load)
748 {
749         int prio = p->static_prio - MAX_RT_PRIO;
750         struct load_weight *load = &p->se.load;
751 
752         /*
753          * SCHED_IDLE tasks get minimal weight:
754          */
755         if (task_has_idle_policy(p)) {
756                 load->weight = scale_load(WEIGHT_IDLEPRIO);
757                 load->inv_weight = WMULT_IDLEPRIO;
758                 p->se.runnable_weight = load->weight;
759                 return;
760         }
761 
762         /*
763          * SCHED_OTHER tasks have to update their load when changing their
764          * weight
765          */
766         if (update_load && p->sched_class == &fair_sched_class) {
767                 reweight_task(p, prio);
768         } else {
769                 load->weight = scale_load(sched_prio_to_weight[prio]);
770                 load->inv_weight = sched_prio_to_wmult[prio];
771                 p->se.runnable_weight = load->weight;
772         }
773 }
774 
775 #ifdef CONFIG_UCLAMP_TASK
776 /* Max allowed minimum utilization */
777 unsigned int sysctl_sched_uclamp_util_min = SCHED_CAPACITY_SCALE;
778 
779 /* Max allowed maximum utilization */
780 unsigned int sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE;
781 
782 /* All clamps are required to be less or equal than these values */
783 static struct uclamp_se uclamp_default[UCLAMP_CNT];
784 
785 /* Integer rounded range for each bucket */
786 #define UCLAMP_BUCKET_DELTA DIV_ROUND_CLOSEST(SCHED_CAPACITY_SCALE, UCLAMP_BUCKETS)
787 
788 #define for_each_clamp_id(clamp_id) \
789         for ((clamp_id) = 0; (clamp_id) < UCLAMP_CNT; (clamp_id)++)
790 
791 static inline unsigned int uclamp_bucket_id(unsigned int clamp_value)
792 {
793         return clamp_value / UCLAMP_BUCKET_DELTA;
794 }
795 
796 static inline unsigned int uclamp_bucket_base_value(unsigned int clamp_value)
797 {
798         return UCLAMP_BUCKET_DELTA * uclamp_bucket_id(clamp_value);
799 }
800 
801 static inline unsigned int uclamp_none(int clamp_id)
802 {
803         if (clamp_id == UCLAMP_MIN)
804                 return 0;
805         return SCHED_CAPACITY_SCALE;
806 }
807 
808 static inline void uclamp_se_set(struct uclamp_se *uc_se,
809                                  unsigned int value, bool user_defined)
810 {
811         uc_se->value = value;
812         uc_se->bucket_id = uclamp_bucket_id(value);
813         uc_se->user_defined = user_defined;
814 }
815 
816 static inline unsigned int
817 uclamp_idle_value(struct rq *rq, unsigned int clamp_id,
818                   unsigned int clamp_value)
819 {
820         /*
821          * Avoid blocked utilization pushing up the frequency when we go
822          * idle (which drops the max-clamp) by retaining the last known
823          * max-clamp.
824          */
825         if (clamp_id == UCLAMP_MAX) {
826                 rq->uclamp_flags |= UCLAMP_FLAG_IDLE;
827                 return clamp_value;
828         }
829 
830         return uclamp_none(UCLAMP_MIN);
831 }
832 
833 static inline void uclamp_idle_reset(struct rq *rq, unsigned int clamp_id,
834                                      unsigned int clamp_value)
835 {
836         /* Reset max-clamp retention only on idle exit */
837         if (!(rq->uclamp_flags & UCLAMP_FLAG_IDLE))
838                 return;
839 
840         WRITE_ONCE(rq->uclamp[clamp_id].value, clamp_value);
841 }
842 
843 static inline
844 unsigned int uclamp_rq_max_value(struct rq *rq, unsigned int clamp_id,
845                                  unsigned int clamp_value)
846 {
847         struct uclamp_bucket *bucket = rq->uclamp[clamp_id].bucket;
848         int bucket_id = UCLAMP_BUCKETS - 1;
849 
850         /*
851          * Since both min and max clamps are max aggregated, find the
852          * top most bucket with tasks in.
853          */
854         for ( ; bucket_id >= 0; bucket_id--) {
855                 if (!bucket[bucket_id].tasks)
856                         continue;
857                 return bucket[bucket_id].value;
858         }
859 
860         /* No tasks -- default clamp values */
861         return uclamp_idle_value(rq, clamp_id, clamp_value);
862 }
863 
864 /*
865  * The effective clamp bucket index of a task depends on, by increasing
866  * priority:
867  * - the task specific clamp value, when explicitly requested from userspace
868  * - the system default clamp value, defined by the sysadmin
869  */
870 static inline struct uclamp_se
871 uclamp_eff_get(struct task_struct *p, unsigned int clamp_id)
872 {
873         struct uclamp_se uc_req = p->uclamp_req[clamp_id];
874         struct uclamp_se uc_max = uclamp_default[clamp_id];
875 
876         /* System default restrictions always apply */
877         if (unlikely(uc_req.value > uc_max.value))
878                 return uc_max;
879 
880         return uc_req;
881 }
882 
883 unsigned int uclamp_eff_value(struct task_struct *p, unsigned int clamp_id)
884 {
885         struct uclamp_se uc_eff;
886 
887         /* Task currently refcounted: use back-annotated (effective) value */
888         if (p->uclamp[clamp_id].active)
889                 return p->uclamp[clamp_id].value;
890 
891         uc_eff = uclamp_eff_get(p, clamp_id);
892 
893         return uc_eff.value;
894 }
895 
896 /*
897  * When a task is enqueued on a rq, the clamp bucket currently defined by the
898  * task's uclamp::bucket_id is refcounted on that rq. This also immediately
899  * updates the rq's clamp value if required.
900  *
901  * Tasks can have a task-specific value requested from user-space, track
902  * within each bucket the maximum value for tasks refcounted in it.
903  * This "local max aggregation" allows to track the exact "requested" value
904  * for each bucket when all its RUNNABLE tasks require the same clamp.
905  */
906 static inline void uclamp_rq_inc_id(struct rq *rq, struct task_struct *p,
907                                     unsigned int clamp_id)
908 {
909         struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id];
910         struct uclamp_se *uc_se = &p->uclamp[clamp_id];
911         struct uclamp_bucket *bucket;
912 
913         lockdep_assert_held(&rq->lock);
914 
915         /* Update task effective clamp */
916         p->uclamp[clamp_id] = uclamp_eff_get(p, clamp_id);
917 
918         bucket = &uc_rq->bucket[uc_se->bucket_id];
919         bucket->tasks++;
920         uc_se->active = true;
921 
922         uclamp_idle_reset(rq, clamp_id, uc_se->value);
923 
924         /*
925          * Local max aggregation: rq buckets always track the max
926          * "requested" clamp value of its RUNNABLE tasks.
927          */
928         if (bucket->tasks == 1 || uc_se->value > bucket->value)
929                 bucket->value = uc_se->value;
930 
931         if (uc_se->value > READ_ONCE(uc_rq->value))
932                 WRITE_ONCE(uc_rq->value, uc_se->value);
933 }
934 
935 /*
936  * When a task is dequeued from a rq, the clamp bucket refcounted by the task
937  * is released. If this is the last task reference counting the rq's max
938  * active clamp value, then the rq's clamp value is updated.
939  *
940  * Both refcounted tasks and rq's cached clamp values are expected to be
941  * always valid. If it's detected they are not, as defensive programming,
942  * enforce the expected state and warn.
943  */
944 static inline void uclamp_rq_dec_id(struct rq *rq, struct task_struct *p,
945                                     unsigned int clamp_id)
946 {
947         struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id];
948         struct uclamp_se *uc_se = &p->uclamp[clamp_id];
949         struct uclamp_bucket *bucket;
950         unsigned int bkt_clamp;
951         unsigned int rq_clamp;
952 
953         lockdep_assert_held(&rq->lock);
954 
955         bucket = &uc_rq->bucket[uc_se->bucket_id];
956         SCHED_WARN_ON(!bucket->tasks);
957         if (likely(bucket->tasks))
958                 bucket->tasks--;
959         uc_se->active = false;
960 
961         /*
962          * Keep "local max aggregation" simple and accept to (possibly)
963          * overboost some RUNNABLE tasks in the same bucket.
964          * The rq clamp bucket value is reset to its base value whenever
965          * there are no more RUNNABLE tasks refcounting it.
966          */
967         if (likely(bucket->tasks))
968                 return;
969 
970         rq_clamp = READ_ONCE(uc_rq->value);
971         /*
972          * Defensive programming: this should never happen. If it happens,
973          * e.g. due to future modification, warn and fixup the expected value.
974          */
975         SCHED_WARN_ON(bucket->value > rq_clamp);
976         if (bucket->value >= rq_clamp) {
977                 bkt_clamp = uclamp_rq_max_value(rq, clamp_id, uc_se->value);
978                 WRITE_ONCE(uc_rq->value, bkt_clamp);
979         }
980 }
981 
982 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p)
983 {
984         unsigned int clamp_id;
985 
986         if (unlikely(!p->sched_class->uclamp_enabled))
987                 return;
988 
989         for_each_clamp_id(clamp_id)
990                 uclamp_rq_inc_id(rq, p, clamp_id);
991 
992         /* Reset clamp idle holding when there is one RUNNABLE task */
993         if (rq->uclamp_flags & UCLAMP_FLAG_IDLE)
994                 rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE;
995 }
996 
997 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p)
998 {
999         unsigned int clamp_id;
1000 
1001         if (unlikely(!p->sched_class->uclamp_enabled))
1002                 return;
1003 
1004         for_each_clamp_id(clamp_id)
1005                 uclamp_rq_dec_id(rq, p, clamp_id);
1006 }
1007 
1008 int sysctl_sched_uclamp_handler(struct ctl_table *table, int write,
1009                                 void __user *buffer, size_t *lenp,
1010                                 loff_t *ppos)
1011 {
1012         int old_min, old_max;
1013         static DEFINE_MUTEX(mutex);
1014         int result;
1015 
1016         mutex_lock(&mutex);
1017         old_min = sysctl_sched_uclamp_util_min;
1018         old_max = sysctl_sched_uclamp_util_max;
1019 
1020         result = proc_dointvec(table, write, buffer, lenp, ppos);
1021         if (result)
1022                 goto undo;
1023         if (!write)
1024                 goto done;
1025 
1026         if (sysctl_sched_uclamp_util_min > sysctl_sched_uclamp_util_max ||
1027             sysctl_sched_uclamp_util_max > SCHED_CAPACITY_SCALE) {
1028                 result = -EINVAL;
1029                 goto undo;
1030         }
1031 
1032         if (old_min != sysctl_sched_uclamp_util_min) {
1033                 uclamp_se_set(&uclamp_default[UCLAMP_MIN],
1034                               sysctl_sched_uclamp_util_min, false);
1035         }
1036         if (old_max != sysctl_sched_uclamp_util_max) {
1037                 uclamp_se_set(&uclamp_default[UCLAMP_MAX],
1038                               sysctl_sched_uclamp_util_max, false);
1039         }
1040 
1041         /*
1042          * Updating all the RUNNABLE task is expensive, keep it simple and do
1043          * just a lazy update at each next enqueue time.
1044          */
1045         goto done;
1046 
1047 undo:
1048         sysctl_sched_uclamp_util_min = old_min;
1049         sysctl_sched_uclamp_util_max = old_max;
1050 done:
1051         mutex_unlock(&mutex);
1052 
1053         return result;
1054 }
1055 
1056 static int uclamp_validate(struct task_struct *p,
1057                            const struct sched_attr *attr)
1058 {
1059         unsigned int lower_bound = p->uclamp_req[UCLAMP_MIN].value;
1060         unsigned int upper_bound = p->uclamp_req[UCLAMP_MAX].value;
1061 
1062         if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN)
1063                 lower_bound = attr->sched_util_min;
1064         if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX)
1065                 upper_bound = attr->sched_util_max;
1066 
1067         if (lower_bound > upper_bound)
1068                 return -EINVAL;
1069         if (upper_bound > SCHED_CAPACITY_SCALE)
1070                 return -EINVAL;
1071 
1072         return 0;
1073 }
1074 
1075 static void __setscheduler_uclamp(struct task_struct *p,
1076                                   const struct sched_attr *attr)
1077 {
1078         unsigned int clamp_id;
1079 
1080         /*
1081          * On scheduling class change, reset to default clamps for tasks
1082          * without a task-specific value.
1083          */
1084         for_each_clamp_id(clamp_id) {
1085                 struct uclamp_se *uc_se = &p->uclamp_req[clamp_id];
1086                 unsigned int clamp_value = uclamp_none(clamp_id);
1087 
1088                 /* Keep using defined clamps across class changes */
1089                 if (uc_se->user_defined)
1090                         continue;
1091 
1092                 /* By default, RT tasks always get 100% boost */
1093                 if (unlikely(rt_task(p) && clamp_id == UCLAMP_MIN))
1094                         clamp_value = uclamp_none(UCLAMP_MAX);
1095 
1096                 uclamp_se_set(uc_se, clamp_value, false);
1097         }
1098 
1099         if (likely(!(attr->sched_flags & SCHED_FLAG_UTIL_CLAMP)))
1100                 return;
1101 
1102         if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN) {
1103                 uclamp_se_set(&p->uclamp_req[UCLAMP_MIN],
1104                               attr->sched_util_min, true);
1105         }
1106 
1107         if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX) {
1108                 uclamp_se_set(&p->uclamp_req[UCLAMP_MAX],
1109                               attr->sched_util_max, true);
1110         }
1111 }
1112 
1113 static void uclamp_fork(struct task_struct *p)
1114 {
1115         unsigned int clamp_id;
1116 
1117         for_each_clamp_id(clamp_id)
1118                 p->uclamp[clamp_id].active = false;
1119 
1120         if (likely(!p->sched_reset_on_fork))
1121                 return;
1122 
1123         for_each_clamp_id(clamp_id) {
1124                 unsigned int clamp_value = uclamp_none(clamp_id);
1125 
1126                 /* By default, RT tasks always get 100% boost */
1127                 if (unlikely(rt_task(p) && clamp_id == UCLAMP_MIN))
1128                         clamp_value = uclamp_none(UCLAMP_MAX);
1129 
1130                 uclamp_se_set(&p->uclamp_req[clamp_id], clamp_value, false);
1131         }
1132 }
1133 
1134 static void __init init_uclamp(void)
1135 {
1136         struct uclamp_se uc_max = {};
1137         unsigned int clamp_id;
1138         int cpu;
1139 
1140         for_each_possible_cpu(cpu) {
1141                 memset(&cpu_rq(cpu)->uclamp, 0, sizeof(struct uclamp_rq));
1142                 cpu_rq(cpu)->uclamp_flags = 0;
1143         }
1144 
1145         for_each_clamp_id(clamp_id) {
1146                 uclamp_se_set(&init_task.uclamp_req[clamp_id],
1147                               uclamp_none(clamp_id), false);
1148         }
1149 
1150         /* System defaults allow max clamp values for both indexes */
1151         uclamp_se_set(&uc_max, uclamp_none(UCLAMP_MAX), false);
1152         for_each_clamp_id(clamp_id)
1153                 uclamp_default[clamp_id] = uc_max;
1154 }
1155 
1156 #else /* CONFIG_UCLAMP_TASK */
1157 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) { }
1158 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) { }
1159 static inline int uclamp_validate(struct task_struct *p,
1160                                   const struct sched_attr *attr)
1161 {
1162         return -EOPNOTSUPP;
1163 }
1164 static void __setscheduler_uclamp(struct task_struct *p,
1165                                   const struct sched_attr *attr) { }
1166 static inline void uclamp_fork(struct task_struct *p) { }
1167 static inline void init_uclamp(void) { }
1168 #endif /* CONFIG_UCLAMP_TASK */
1169 
1170 static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
1171 {
1172         if (!(flags & ENQUEUE_NOCLOCK))
1173                 update_rq_clock(rq);
1174 
1175         if (!(flags & ENQUEUE_RESTORE)) {
1176                 sched_info_queued(rq, p);
1177                 psi_enqueue(p, flags & ENQUEUE_WAKEUP);
1178         }
1179 
1180         uclamp_rq_inc(rq, p);
1181         p->sched_class->enqueue_task(rq, p, flags);
1182 }
1183 
1184 static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
1185 {
1186         if (!(flags & DEQUEUE_NOCLOCK))
1187                 update_rq_clock(rq);
1188 
1189         if (!(flags & DEQUEUE_SAVE)) {
1190                 sched_info_dequeued(rq, p);
1191                 psi_dequeue(p, flags & DEQUEUE_SLEEP);
1192         }
1193 
1194         uclamp_rq_dec(rq, p);
1195         p->sched_class->dequeue_task(rq, p, flags);
1196 }
1197 
1198 void activate_task(struct rq *rq, struct task_struct *p, int flags)
1199 {
1200         if (task_contributes_to_load(p))
1201                 rq->nr_uninterruptible--;
1202 
1203         enqueue_task(rq, p, flags);
1204 
1205         p->on_rq = TASK_ON_RQ_QUEUED;
1206 }
1207 
1208 void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
1209 {
1210         p->on_rq = (flags & DEQUEUE_SLEEP) ? 0 : TASK_ON_RQ_MIGRATING;
1211 
1212         if (task_contributes_to_load(p))
1213                 rq->nr_uninterruptible++;
1214 
1215         dequeue_task(rq, p, flags);
1216 }
1217 
1218 /*
1219  * __normal_prio - return the priority that is based on the static prio
1220  */
1221 static inline int __normal_prio(struct task_struct *p)
1222 {
1223         return p->static_prio;
1224 }
1225 
1226 /*
1227  * Calculate the expected normal priority: i.e. priority
1228  * without taking RT-inheritance into account. Might be
1229  * boosted by interactivity modifiers. Changes upon fork,
1230  * setprio syscalls, and whenever the interactivity
1231  * estimator recalculates.
1232  */
1233 static inline int normal_prio(struct task_struct *p)
1234 {
1235         int prio;
1236 
1237         if (task_has_dl_policy(p))
1238                 prio = MAX_DL_PRIO-1;
1239         else if (task_has_rt_policy(p))
1240                 prio = MAX_RT_PRIO-1 - p->rt_priority;
1241         else
1242                 prio = __normal_prio(p);
1243         return prio;
1244 }
1245 
1246 /*
1247  * Calculate the current priority, i.e. the priority
1248  * taken into account by the scheduler. This value might
1249  * be boosted by RT tasks, or might be boosted by
1250  * interactivity modifiers. Will be RT if the task got
1251  * RT-boosted. If not then it returns p->normal_prio.
1252  */
1253 static int effective_prio(struct task_struct *p)
1254 {
1255         p->normal_prio = normal_prio(p);
1256         /*
1257          * If we are RT tasks or we were boosted to RT priority,
1258          * keep the priority unchanged. Otherwise, update priority
1259          * to the normal priority:
1260          */
1261         if (!rt_prio(p->prio))
1262                 return p->normal_prio;
1263         return p->prio;
1264 }
1265 
1266 /**
1267  * task_curr - is this task currently executing on a CPU?
1268  * @p: the task in question.
1269  *
1270  * Return: 1 if the task is currently executing. 0 otherwise.
1271  */
1272 inline int task_curr(const struct task_struct *p)
1273 {
1274         return cpu_curr(task_cpu(p)) == p;
1275 }
1276 
1277 /*
1278  * switched_from, switched_to and prio_changed must _NOT_ drop rq->lock,
1279  * use the balance_callback list if you want balancing.
1280  *
1281  * this means any call to check_class_changed() must be followed by a call to
1282  * balance_callback().
1283  */
1284 static inline void check_class_changed(struct rq *rq, struct task_struct *p,
1285                                        const struct sched_class *prev_class,
1286                                        int oldprio)
1287 {
1288         if (prev_class != p->sched_class) {
1289                 if (prev_class->switched_from)
1290                         prev_class->switched_from(rq, p);
1291 
1292                 p->sched_class->switched_to(rq, p);
1293         } else if (oldprio != p->prio || dl_task(p))
1294                 p->sched_class->prio_changed(rq, p, oldprio);
1295 }
1296 
1297 void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
1298 {
1299         const struct sched_class *class;
1300 
1301         if (p->sched_class == rq->curr->sched_class) {
1302                 rq->curr->sched_class->check_preempt_curr(rq, p, flags);
1303         } else {
1304                 for_each_class(class) {
1305                         if (class == rq->curr->sched_class)
1306                                 break;
1307                         if (class == p->sched_class) {
1308                                 resched_curr(rq);
1309                                 break;
1310                         }
1311                 }
1312         }
1313 
1314         /*
1315          * A queue event has occurred, and we're going to schedule.  In
1316          * this case, we can save a useless back to back clock update.
1317          */
1318         if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr))
1319                 rq_clock_skip_update(rq);
1320 }
1321 
1322 #ifdef CONFIG_SMP
1323 
1324 static inline bool is_per_cpu_kthread(struct task_struct *p)
1325 {
1326         if (!(p->flags & PF_KTHREAD))
1327                 return false;
1328 
1329         if (p->nr_cpus_allowed != 1)
1330                 return false;
1331 
1332         return true;
1333 }
1334 
1335 /*
1336  * Per-CPU kthreads are allowed to run on !active && online CPUs, see
1337  * __set_cpus_allowed_ptr() and select_fallback_rq().
1338  */
1339 static inline bool is_cpu_allowed(struct task_struct *p, int cpu)
1340 {
1341         if (!cpumask_test_cpu(cpu, p->cpus_ptr))
1342                 return false;
1343 
1344         if (is_per_cpu_kthread(p))
1345                 return cpu_online(cpu);
1346 
1347         return cpu_active(cpu);
1348 }
1349 
1350 /*
1351  * This is how migration works:
1352  *
1353  * 1) we invoke migration_cpu_stop() on the target CPU using
1354  *    stop_one_cpu().
1355  * 2) stopper starts to run (implicitly forcing the migrated thread
1356  *    off the CPU)
1357  * 3) it checks whether the migrated task is still in the wrong runqueue.
1358  * 4) if it's in the wrong runqueue then the migration thread removes
1359  *    it and puts it into the right queue.
1360  * 5) stopper completes and stop_one_cpu() returns and the migration
1361  *    is done.
1362  */
1363 
1364 /*
1365  * move_queued_task - move a queued task to new rq.
1366  *
1367  * Returns (locked) new rq. Old rq's lock is released.
1368  */
1369 static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf,
1370                                    struct task_struct *p, int new_cpu)
1371 {
1372         lockdep_assert_held(&rq->lock);
1373 
1374         WRITE_ONCE(p->on_rq, TASK_ON_RQ_MIGRATING);
1375         dequeue_task(rq, p, DEQUEUE_NOCLOCK);
1376         set_task_cpu(p, new_cpu);
1377         rq_unlock(rq, rf);
1378 
1379         rq = cpu_rq(new_cpu);
1380 
1381         rq_lock(rq, rf);
1382         BUG_ON(task_cpu(p) != new_cpu);
1383         enqueue_task(rq, p, 0);
1384         p->on_rq = TASK_ON_RQ_QUEUED;
1385         check_preempt_curr(rq, p, 0);
1386 
1387         return rq;
1388 }
1389 
1390 struct migration_arg {
1391         struct task_struct *task;
1392         int dest_cpu;
1393 };
1394 
1395 /*
1396  * Move (not current) task off this CPU, onto the destination CPU. We're doing
1397  * this because either it can't run here any more (set_cpus_allowed()
1398  * away from this CPU, or CPU going down), or because we're
1399  * attempting to rebalance this task on exec (sched_exec).
1400  *
1401  * So we race with normal scheduler movements, but that's OK, as long
1402  * as the task is no longer on this CPU.
1403  */
1404 static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf,
1405                                  struct task_struct *p, int dest_cpu)
1406 {
1407         /* Affinity changed (again). */
1408         if (!is_cpu_allowed(p, dest_cpu))
1409                 return rq;
1410 
1411         update_rq_clock(rq);
1412         rq = move_queued_task(rq, rf, p, dest_cpu);
1413 
1414         return rq;
1415 }
1416 
1417 /*
1418  * migration_cpu_stop - this will be executed by a highprio stopper thread
1419  * and performs thread migration by bumping thread off CPU then
1420  * 'pushing' onto another runqueue.
1421  */
1422 static int migration_cpu_stop(void *data)
1423 {
1424         struct migration_arg *arg = data;
1425         struct task_struct *p = arg->task;
1426         struct rq *rq = this_rq();
1427         struct rq_flags rf;
1428 
1429         /*
1430          * The original target CPU might have gone down and we might
1431          * be on another CPU but it doesn't matter.
1432          */
1433         local_irq_disable();
1434         /*
1435          * We need to explicitly wake pending tasks before running
1436          * __migrate_task() such that we will not miss enforcing cpus_ptr
1437          * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test.
1438          */
1439         sched_ttwu_pending();
1440 
1441         raw_spin_lock(&p->pi_lock);
1442         rq_lock(rq, &rf);
1443         /*
1444          * If task_rq(p) != rq, it cannot be migrated here, because we're
1445          * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because
1446          * we're holding p->pi_lock.
1447          */
1448         if (task_rq(p) == rq) {
1449                 if (task_on_rq_queued(p))
1450                         rq = __migrate_task(rq, &rf, p, arg->dest_cpu);
1451                 else
1452                         p->wake_cpu = arg->dest_cpu;
1453         }
1454         rq_unlock(rq, &rf);
1455         raw_spin_unlock(&p->pi_lock);
1456 
1457         local_irq_enable();
1458         return 0;
1459 }
1460 
1461 /*
1462  * sched_class::set_cpus_allowed must do the below, but is not required to
1463  * actually call this function.
1464  */
1465 void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask)
1466 {
1467         cpumask_copy(&p->cpus_mask, new_mask);
1468         p->nr_cpus_allowed = cpumask_weight(new_mask);
1469 }
1470 
1471 void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
1472 {
1473         struct rq *rq = task_rq(p);
1474         bool queued, running;
1475 
1476         lockdep_assert_held(&p->pi_lock);
1477 
1478         queued = task_on_rq_queued(p);
1479         running = task_current(rq, p);
1480 
1481         if (queued) {
1482                 /*
1483                  * Because __kthread_bind() calls this on blocked tasks without
1484                  * holding rq->lock.
1485                  */
1486                 lockdep_assert_held(&rq->lock);
1487                 dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK);
1488         }
1489         if (running)
1490                 put_prev_task(rq, p);
1491 
1492         p->sched_class->set_cpus_allowed(p, new_mask);
1493 
1494         if (queued)
1495                 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
1496         if (running)
1497                 set_curr_task(rq, p);
1498 }
1499 
1500 /*
1501  * Change a given task's CPU affinity. Migrate the thread to a
1502  * proper CPU and schedule it away if the CPU it's executing on
1503  * is removed from the allowed bitmask.
1504  *
1505  * NOTE: the caller must have a valid reference to the task, the
1506  * task must not exit() & deallocate itself prematurely. The
1507  * call is not atomic; no spinlocks may be held.
1508  */
1509 static int __set_cpus_allowed_ptr(struct task_struct *p,
1510                                   const struct cpumask *new_mask, bool check)
1511 {
1512         const struct cpumask *cpu_valid_mask = cpu_active_mask;
1513         unsigned int dest_cpu;
1514         struct rq_flags rf;
1515         struct rq *rq;
1516         int ret = 0;
1517 
1518         rq = task_rq_lock(p, &rf);
1519         update_rq_clock(rq);
1520 
1521         if (p->flags & PF_KTHREAD) {
1522                 /*
1523                  * Kernel threads are allowed on online && !active CPUs
1524                  */
1525                 cpu_valid_mask = cpu_online_mask;
1526         }
1527 
1528         /*
1529          * Must re-check here, to close a race against __kthread_bind(),
1530          * sched_setaffinity() is not guaranteed to observe the flag.
1531          */
1532         if (check && (p->flags & PF_NO_SETAFFINITY)) {
1533                 ret = -EINVAL;
1534                 goto out;
1535         }
1536 
1537         if (cpumask_equal(p->cpus_ptr, new_mask))
1538                 goto out;
1539 
1540         dest_cpu = cpumask_any_and(cpu_valid_mask, new_mask);
1541         if (dest_cpu >= nr_cpu_ids) {
1542                 ret = -EINVAL;
1543                 goto out;
1544         }
1545 
1546         do_set_cpus_allowed(p, new_mask);
1547 
1548         if (p->flags & PF_KTHREAD) {
1549                 /*
1550                  * For kernel threads that do indeed end up on online &&
1551                  * !active we want to ensure they are strict per-CPU threads.
1552                  */
1553                 WARN_ON(cpumask_intersects(new_mask, cpu_online_mask) &&
1554                         !cpumask_intersects(new_mask, cpu_active_mask) &&
1555                         p->nr_cpus_allowed != 1);
1556         }
1557 
1558         /* Can the task run on the task's current CPU? If so, we're done */
1559         if (cpumask_test_cpu(task_cpu(p), new_mask))
1560                 goto out;
1561 
1562         if (task_running(rq, p) || p->state == TASK_WAKING) {
1563                 struct migration_arg arg = { p, dest_cpu };
1564                 /* Need help from migration thread: drop lock and wait. */
1565                 task_rq_unlock(rq, p, &rf);
1566                 stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
1567                 return 0;
1568         } else if (task_on_rq_queued(p)) {
1569                 /*
1570                  * OK, since we're going to drop the lock immediately
1571                  * afterwards anyway.
1572                  */
1573                 rq = move_queued_task(rq, &rf, p, dest_cpu);
1574         }
1575 out:
1576         task_rq_unlock(rq, p, &rf);
1577 
1578         return ret;
1579 }
1580 
1581 int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
1582 {
1583         return __set_cpus_allowed_ptr(p, new_mask, false);
1584 }
1585 EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
1586 
1587 void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
1588 {
1589 #ifdef CONFIG_SCHED_DEBUG
1590         /*
1591          * We should never call set_task_cpu() on a blocked task,
1592          * ttwu() will sort out the placement.
1593          */
1594         WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
1595                         !p->on_rq);
1596 
1597         /*
1598          * Migrating fair class task must have p->on_rq = TASK_ON_RQ_MIGRATING,
1599          * because schedstat_wait_{start,end} rebase migrating task's wait_start
1600          * time relying on p->on_rq.
1601          */
1602         WARN_ON_ONCE(p->state == TASK_RUNNING &&
1603                      p->sched_class == &fair_sched_class &&
1604                      (p->on_rq && !task_on_rq_migrating(p)));
1605 
1606 #ifdef CONFIG_LOCKDEP
1607         /*
1608          * The caller should hold either p->pi_lock or rq->lock, when changing
1609          * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks.
1610          *
1611          * sched_move_task() holds both and thus holding either pins the cgroup,
1612          * see task_group().
1613          *
1614          * Furthermore, all task_rq users should acquire both locks, see
1615          * task_rq_lock().
1616          */
1617         WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) ||
1618                                       lockdep_is_held(&task_rq(p)->lock)));
1619 #endif
1620         /*
1621          * Clearly, migrating tasks to offline CPUs is a fairly daft thing.
1622          */
1623         WARN_ON_ONCE(!cpu_online(new_cpu));
1624 #endif
1625 
1626         trace_sched_migrate_task(p, new_cpu);
1627 
1628         if (task_cpu(p) != new_cpu) {
1629                 if (p->sched_class->migrate_task_rq)
1630                         p->sched_class->migrate_task_rq(p, new_cpu);
1631                 p->se.nr_migrations++;
1632                 rseq_migrate(p);
1633                 perf_event_task_migrate(p);
1634         }
1635 
1636         __set_task_cpu(p, new_cpu);
1637 }
1638 
1639 #ifdef CONFIG_NUMA_BALANCING
1640 static void __migrate_swap_task(struct task_struct *p, int cpu)
1641 {
1642         if (task_on_rq_queued(p)) {
1643                 struct rq *src_rq, *dst_rq;
1644                 struct rq_flags srf, drf;
1645 
1646                 src_rq = task_rq(p);
1647                 dst_rq = cpu_rq(cpu);
1648 
1649                 rq_pin_lock(src_rq, &srf);
1650                 rq_pin_lock(dst_rq, &drf);
1651 
1652                 deactivate_task(src_rq, p, 0);
1653                 set_task_cpu(p, cpu);
1654                 activate_task(dst_rq, p, 0);
1655                 check_preempt_curr(dst_rq, p, 0);
1656 
1657                 rq_unpin_lock(dst_rq, &drf);
1658                 rq_unpin_lock(src_rq, &srf);
1659 
1660         } else {
1661                 /*
1662                  * Task isn't running anymore; make it appear like we migrated
1663                  * it before it went to sleep. This means on wakeup we make the
1664                  * previous CPU our target instead of where it really is.
1665                  */
1666                 p->wake_cpu = cpu;
1667         }
1668 }
1669 
1670 struct migration_swap_arg {
1671         struct task_struct *src_task, *dst_task;
1672         int src_cpu, dst_cpu;
1673 };
1674 
1675 static int migrate_swap_stop(void *data)
1676 {
1677         struct migration_swap_arg *arg = data;
1678         struct rq *src_rq, *dst_rq;
1679         int ret = -EAGAIN;
1680 
1681         if (!cpu_active(arg->src_cpu) || !cpu_active(arg->dst_cpu))
1682                 return -EAGAIN;
1683 
1684         src_rq = cpu_rq(arg->src_cpu);
1685         dst_rq = cpu_rq(arg->dst_cpu);
1686 
1687         double_raw_lock(&arg->src_task->pi_lock,
1688                         &arg->dst_task->pi_lock);
1689         double_rq_lock(src_rq, dst_rq);
1690 
1691         if (task_cpu(arg->dst_task) != arg->dst_cpu)
1692                 goto unlock;
1693 
1694         if (task_cpu(arg->src_task) != arg->src_cpu)
1695                 goto unlock;
1696 
1697         if (!cpumask_test_cpu(arg->dst_cpu, arg->src_task->cpus_ptr))
1698                 goto unlock;
1699 
1700         if (!cpumask_test_cpu(arg->src_cpu, arg->dst_task->cpus_ptr))
1701                 goto unlock;
1702 
1703         __migrate_swap_task(arg->src_task, arg->dst_cpu);
1704         __migrate_swap_task(arg->dst_task, arg->src_cpu);
1705 
1706         ret = 0;
1707 
1708 unlock:
1709         double_rq_unlock(src_rq, dst_rq);
1710         raw_spin_unlock(&arg->dst_task->pi_lock);
1711         raw_spin_unlock(&arg->src_task->pi_lock);
1712 
1713         return ret;
1714 }
1715 
1716 /*
1717  * Cross migrate two tasks
1718  */
1719 int migrate_swap(struct task_struct *cur, struct task_struct *p,
1720                 int target_cpu, int curr_cpu)
1721 {
1722         struct migration_swap_arg arg;
1723         int ret = -EINVAL;
1724 
1725         arg = (struct migration_swap_arg){
1726                 .src_task = cur,
1727                 .src_cpu = curr_cpu,
1728                 .dst_task = p,
1729                 .dst_cpu = target_cpu,
1730         };
1731 
1732         if (arg.src_cpu == arg.dst_cpu)
1733                 goto out;
1734 
1735         /*
1736          * These three tests are all lockless; this is OK since all of them
1737          * will be re-checked with proper locks held further down the line.
1738          */
1739         if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu))
1740                 goto out;
1741 
1742         if (!cpumask_test_cpu(arg.dst_cpu, arg.src_task->cpus_ptr))
1743                 goto out;
1744 
1745         if (!cpumask_test_cpu(arg.src_cpu, arg.dst_task->cpus_ptr))
1746                 goto out;
1747 
1748         trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu);
1749         ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg);
1750 
1751 out:
1752         return ret;
1753 }
1754 #endif /* CONFIG_NUMA_BALANCING */
1755 
1756 /*
1757  * wait_task_inactive - wait for a thread to unschedule.
1758  *
1759  * If @match_state is nonzero, it's the @p->state value just checked and
1760  * not expected to change.  If it changes, i.e. @p might have woken up,
1761  * then return zero.  When we succeed in waiting for @p to be off its CPU,
1762  * we return a positive number (its total switch count).  If a second call
1763  * a short while later returns the same number, the caller can be sure that
1764  * @p has remained unscheduled the whole time.
1765  *
1766  * The caller must ensure that the task *will* unschedule sometime soon,
1767  * else this function might spin for a *long* time. This function can't
1768  * be called with interrupts off, or it may introduce deadlock with
1769  * smp_call_function() if an IPI is sent by the same process we are
1770  * waiting to become inactive.
1771  */
1772 unsigned long wait_task_inactive(struct task_struct *p, long match_state)
1773 {
1774         int running, queued;
1775         struct rq_flags rf;
1776         unsigned long ncsw;
1777         struct rq *rq;
1778 
1779         for (;;) {
1780                 /*
1781                  * We do the initial early heuristics without holding
1782                  * any task-queue locks at all. We'll only try to get
1783                  * the runqueue lock when things look like they will
1784                  * work out!
1785                  */
1786                 rq = task_rq(p);
1787 
1788                 /*
1789                  * If the task is actively running on another CPU
1790                  * still, just relax and busy-wait without holding
1791                  * any locks.
1792                  *
1793                  * NOTE! Since we don't hold any locks, it's not
1794                  * even sure that "rq" stays as the right runqueue!
1795                  * But we don't care, since "task_running()" will
1796                  * return false if the runqueue has changed and p
1797                  * is actually now running somewhere else!
1798                  */
1799                 while (task_running(rq, p)) {
1800                         if (match_state && unlikely(p->state != match_state))
1801                                 return 0;
1802                         cpu_relax();
1803                 }
1804 
1805                 /*
1806                  * Ok, time to look more closely! We need the rq
1807                  * lock now, to be *sure*. If we're wrong, we'll
1808                  * just go back and repeat.
1809                  */
1810                 rq = task_rq_lock(p, &rf);
1811                 trace_sched_wait_task(p);
1812                 running = task_running(rq, p);
1813                 queued = task_on_rq_queued(p);
1814                 ncsw = 0;
1815                 if (!match_state || p->state == match_state)
1816                         ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
1817                 task_rq_unlock(rq, p, &rf);
1818 
1819                 /*
1820                  * If it changed from the expected state, bail out now.
1821                  */
1822                 if (unlikely(!ncsw))
1823                         break;
1824 
1825                 /*
1826                  * Was it really running after all now that we
1827                  * checked with the proper locks actually held?
1828                  *
1829                  * Oops. Go back and try again..
1830                  */
1831                 if (unlikely(running)) {
1832                         cpu_relax();
1833                         continue;
1834                 }
1835 
1836                 /*
1837                  * It's not enough that it's not actively running,
1838                  * it must be off the runqueue _entirely_, and not
1839                  * preempted!
1840                  *
1841                  * So if it was still runnable (but just not actively
1842                  * running right now), it's preempted, and we should
1843                  * yield - it could be a while.
1844                  */
1845                 if (unlikely(queued)) {
1846                         ktime_t to = NSEC_PER_SEC / HZ;
1847 
1848                         set_current_state(TASK_UNINTERRUPTIBLE);
1849                         schedule_hrtimeout(&to, HRTIMER_MODE_REL);
1850                         continue;
1851                 }
1852 
1853                 /*
1854                  * Ahh, all good. It wasn't running, and it wasn't
1855                  * runnable, which means that it will never become
1856                  * running in the future either. We're all done!
1857                  */
1858                 break;
1859         }
1860 
1861         return ncsw;
1862 }
1863 
1864 /***
1865  * kick_process - kick a running thread to enter/exit the kernel
1866  * @p: the to-be-kicked thread
1867  *
1868  * Cause a process which is running on another CPU to enter
1869  * kernel-mode, without any delay. (to get signals handled.)
1870  *
1871  * NOTE: this function doesn't have to take the runqueue lock,
1872  * because all it wants to ensure is that the remote task enters
1873  * the kernel. If the IPI races and the task has been migrated
1874  * to another CPU then no harm is done and the purpose has been
1875  * achieved as well.
1876  */
1877 void kick_process(struct task_struct *p)
1878 {
1879         int cpu;
1880 
1881         preempt_disable();
1882         cpu = task_cpu(p);
1883         if ((cpu != smp_processor_id()) && task_curr(p))
1884                 smp_send_reschedule(cpu);
1885         preempt_enable();
1886 }
1887 EXPORT_SYMBOL_GPL(kick_process);
1888 
1889 /*
1890  * ->cpus_ptr is protected by both rq->lock and p->pi_lock
1891  *
1892  * A few notes on cpu_active vs cpu_online:
1893  *
1894  *  - cpu_active must be a subset of cpu_online
1895  *
1896  *  - on CPU-up we allow per-CPU kthreads on the online && !active CPU,
1897  *    see __set_cpus_allowed_ptr(). At this point the newly online
1898  *    CPU isn't yet part of the sched domains, and balancing will not
1899  *    see it.
1900  *
1901  *  - on CPU-down we clear cpu_active() to mask the sched domains and
1902  *    avoid the load balancer to place new tasks on the to be removed
1903  *    CPU. Existing tasks will remain running there and will be taken
1904  *    off.
1905  *
1906  * This means that fallback selection must not select !active CPUs.
1907  * And can assume that any active CPU must be online. Conversely
1908  * select_task_rq() below may allow selection of !active CPUs in order
1909  * to satisfy the above rules.
1910  */
1911 static int select_fallback_rq(int cpu, struct task_struct *p)
1912 {
1913         int nid = cpu_to_node(cpu);
1914         const struct cpumask *nodemask = NULL;
1915         enum { cpuset, possible, fail } state = cpuset;
1916         int dest_cpu;
1917 
1918         /*
1919          * If the node that the CPU is on has been offlined, cpu_to_node()
1920          * will return -1. There is no CPU on the node, and we should
1921          * select the CPU on the other node.
1922          */
1923         if (nid != -1) {
1924                 nodemask = cpumask_of_node(nid);
1925 
1926                 /* Look for allowed, online CPU in same node. */
1927                 for_each_cpu(dest_cpu, nodemask) {
1928                         if (!cpu_active(dest_cpu))
1929                                 continue;
1930                         if (cpumask_test_cpu(dest_cpu, p->cpus_ptr))
1931                                 return dest_cpu;
1932                 }
1933         }
1934 
1935         for (;;) {
1936                 /* Any allowed, online CPU? */
1937                 for_each_cpu(dest_cpu, p->cpus_ptr) {
1938                         if (!is_cpu_allowed(p, dest_cpu))
1939                                 continue;
1940 
1941                         goto out;
1942                 }
1943 
1944                 /* No more Mr. Nice Guy. */
1945                 switch (state) {
1946                 case cpuset:
1947                         if (IS_ENABLED(CONFIG_CPUSETS)) {
1948                                 cpuset_cpus_allowed_fallback(p);
1949                                 state = possible;
1950                                 break;
1951                         }
1952                         /* Fall-through */
1953                 case possible:
1954                         do_set_cpus_allowed(p, cpu_possible_mask);
1955                         state = fail;
1956                         break;
1957 
1958                 case fail:
1959                         BUG();
1960                         break;
1961                 }
1962         }
1963 
1964 out:
1965         if (state != cpuset) {
1966                 /*
1967                  * Don't tell them about moving exiting tasks or
1968                  * kernel threads (both mm NULL), since they never
1969                  * leave kernel.
1970                  */
1971                 if (p->mm && printk_ratelimit()) {
1972                         printk_deferred("process %d (%s) no longer affine to cpu%d\n",
1973                                         task_pid_nr(p), p->comm, cpu);
1974                 }
1975         }
1976 
1977         return dest_cpu;
1978 }
1979 
1980 /*
1981  * The caller (fork, wakeup) owns p->pi_lock, ->cpus_ptr is stable.
1982  */
1983 static inline
1984 int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)
1985 {
1986         lockdep_assert_held(&p->pi_lock);
1987 
1988         if (p->nr_cpus_allowed > 1)
1989                 cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags);
1990         else
1991                 cpu = cpumask_any(p->cpus_ptr);
1992 
1993         /*
1994          * In order not to call set_task_cpu() on a blocking task we need
1995          * to rely on ttwu() to place the task on a valid ->cpus_ptr
1996          * CPU.
1997          *
1998          * Since this is common to all placement strategies, this lives here.
1999          *
2000          * [ this allows ->select_task() to simply return task_cpu(p) and
2001          *   not worry about this generic constraint ]
2002          */
2003         if (unlikely(!is_cpu_allowed(p, cpu)))
2004                 cpu = select_fallback_rq(task_cpu(p), p);
2005 
2006         return cpu;
2007 }
2008 
2009 static void update_avg(u64 *avg, u64 sample)
2010 {
2011         s64 diff = sample - *avg;
2012         *avg += diff >> 3;
2013 }
2014 
2015 void sched_set_stop_task(int cpu, struct task_struct *stop)
2016 {
2017         struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
2018         struct task_struct *old_stop = cpu_rq(cpu)->stop;
2019 
2020         if (stop) {
2021                 /*
2022                  * Make it appear like a SCHED_FIFO task, its something
2023                  * userspace knows about and won't get confused about.
2024                  *
2025                  * Also, it will make PI more or less work without too
2026                  * much confusion -- but then, stop work should not
2027                  * rely on PI working anyway.
2028                  */
2029                 sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);
2030 
2031                 stop->sched_class = &stop_sched_class;
2032         }
2033 
2034         cpu_rq(cpu)->stop = stop;
2035 
2036         if (old_stop) {
2037                 /*
2038                  * Reset it back to a normal scheduling class so that
2039                  * it can die in pieces.
2040                  */
2041                 old_stop->sched_class = &rt_sched_class;
2042         }
2043 }
2044 
2045 #else
2046 
2047 static inline int __set_cpus_allowed_ptr(struct task_struct *p,
2048                                          const struct cpumask *new_mask, bool check)
2049 {
2050         return set_cpus_allowed_ptr(p, new_mask);
2051 }
2052 
2053 #endif /* CONFIG_SMP */
2054 
2055 static void
2056 ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
2057 {
2058         struct rq *rq;
2059 
2060         if (!schedstat_enabled())
2061                 return;
2062 
2063         rq = this_rq();
2064 
2065 #ifdef CONFIG_SMP
2066         if (cpu == rq->cpu) {
2067                 __schedstat_inc(rq->ttwu_local);
2068                 __schedstat_inc(p->se.statistics.nr_wakeups_local);
2069         } else {
2070                 struct sched_domain *sd;
2071 
2072                 __schedstat_inc(p->se.statistics.nr_wakeups_remote);
2073                 rcu_read_lock();
2074                 for_each_domain(rq->cpu, sd) {
2075                         if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
2076                                 __schedstat_inc(sd->ttwu_wake_remote);
2077                                 break;
2078                         }
2079                 }
2080                 rcu_read_unlock();
2081         }
2082 
2083         if (wake_flags & WF_MIGRATED)
2084                 __schedstat_inc(p->se.statistics.nr_wakeups_migrate);
2085 #endif /* CONFIG_SMP */
2086 
2087         __schedstat_inc(rq->ttwu_count);
2088         __schedstat_inc(p->se.statistics.nr_wakeups);
2089 
2090         if (wake_flags & WF_SYNC)
2091                 __schedstat_inc(p->se.statistics.nr_wakeups_sync);
2092 }
2093 
2094 /*
2095  * Mark the task runnable and perform wakeup-preemption.
2096  */
2097 static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags,
2098                            struct rq_flags *rf)
2099 {
2100         check_preempt_curr(rq, p, wake_flags);
2101         p->state = TASK_RUNNING;
2102         trace_sched_wakeup(p);
2103 
2104 #ifdef CONFIG_SMP
2105         if (p->sched_class->task_woken) {
2106                 /*
2107                  * Our task @p is fully woken up and running; so its safe to
2108                  * drop the rq->lock, hereafter rq is only used for statistics.
2109                  */
2110                 rq_unpin_lock(rq, rf);
2111                 p->sched_class->task_woken(rq, p);
2112                 rq_repin_lock(rq, rf);
2113         }
2114 
2115         if (rq->idle_stamp) {
2116                 u64 delta = rq_clock(rq) - rq->idle_stamp;
2117                 u64 max = 2*rq->max_idle_balance_cost;
2118 
2119                 update_avg(&rq->avg_idle, delta);
2120 
2121                 if (rq->avg_idle > max)
2122                         rq->avg_idle = max;
2123 
2124                 rq->idle_stamp = 0;
2125         }
2126 #endif
2127 }
2128 
2129 static void
2130 ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
2131                  struct rq_flags *rf)
2132 {
2133         int en_flags = ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK;
2134 
2135         lockdep_assert_held(&rq->lock);
2136 
2137 #ifdef CONFIG_SMP
2138         if (p->sched_contributes_to_load)
2139                 rq->nr_uninterruptible--;
2140 
2141         if (wake_flags & WF_MIGRATED)
2142                 en_flags |= ENQUEUE_MIGRATED;
2143 #endif
2144 
2145         activate_task(rq, p, en_flags);
2146         ttwu_do_wakeup(rq, p, wake_flags, rf);
2147 }
2148 
2149 /*
2150  * Called in case the task @p isn't fully descheduled from its runqueue,
2151  * in this case we must do a remote wakeup. Its a 'light' wakeup though,
2152  * since all we need to do is flip p->state to TASK_RUNNING, since
2153  * the task is still ->on_rq.
2154  */
2155 static int ttwu_remote(struct task_struct *p, int wake_flags)
2156 {
2157         struct rq_flags rf;
2158         struct rq *rq;
2159         int ret = 0;
2160 
2161         rq = __task_rq_lock(p, &rf);
2162         if (task_on_rq_queued(p)) {
2163                 /* check_preempt_curr() may use rq clock */
2164                 update_rq_clock(rq);
2165                 ttwu_do_wakeup(rq, p, wake_flags, &rf);
2166                 ret = 1;
2167         }
2168         __task_rq_unlock(rq, &rf);
2169 
2170         return ret;
2171 }
2172 
2173 #ifdef CONFIG_SMP
2174 void sched_ttwu_pending(void)
2175 {
2176         struct rq *rq = this_rq();
2177         struct llist_node *llist = llist_del_all(&rq->wake_list);
2178         struct task_struct *p, *t;
2179         struct rq_flags rf;
2180 
2181         if (!llist)
2182                 return;
2183 
2184         rq_lock_irqsave(rq, &rf);
2185         update_rq_clock(rq);
2186 
2187         llist_for_each_entry_safe(p, t, llist, wake_entry)
2188                 ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf);
2189 
2190         rq_unlock_irqrestore(rq, &rf);
2191 }
2192 
2193 void scheduler_ipi(void)
2194 {
2195         /*
2196          * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
2197          * TIF_NEED_RESCHED remotely (for the first time) will also send
2198          * this IPI.
2199          */
2200         preempt_fold_need_resched();
2201 
2202         if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick())
2203                 return;
2204 
2205         /*
2206          * Not all reschedule IPI handlers call irq_enter/irq_exit, since
2207          * traditionally all their work was done from the interrupt return
2208          * path. Now that we actually do some work, we need to make sure
2209          * we do call them.
2210          *
2211          * Some archs already do call them, luckily irq_enter/exit nest
2212          * properly.
2213          *
2214          * Arguably we should visit all archs and update all handlers,
2215          * however a fair share of IPIs are still resched only so this would
2216          * somewhat pessimize the simple resched case.
2217          */
2218         irq_enter();
2219         sched_ttwu_pending();
2220 
2221         /*
2222          * Check if someone kicked us for doing the nohz idle load balance.
2223          */
2224         if (unlikely(got_nohz_idle_kick())) {
2225                 this_rq()->idle_balance = 1;
2226                 raise_softirq_irqoff(SCHED_SOFTIRQ);
2227         }
2228         irq_exit();
2229 }
2230 
2231 static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags)
2232 {
2233         struct rq *rq = cpu_rq(cpu);
2234 
2235         p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
2236 
2237         if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) {
2238                 if (!set_nr_if_polling(rq->idle))
2239                         smp_send_reschedule(cpu);
2240                 else
2241                         trace_sched_wake_idle_without_ipi(cpu);
2242         }
2243 }
2244 
2245 void wake_up_if_idle(int cpu)
2246 {
2247         struct rq *rq = cpu_rq(cpu);
2248         struct rq_flags rf;
2249 
2250         rcu_read_lock();
2251 
2252         if (!is_idle_task(rcu_dereference(rq->curr)))
2253                 goto out;
2254 
2255         if (set_nr_if_polling(rq->idle)) {
2256                 trace_sched_wake_idle_without_ipi(cpu);
2257         } else {
2258                 rq_lock_irqsave(rq, &rf);
2259                 if (is_idle_task(rq->curr))
2260                         smp_send_reschedule(cpu);
2261                 /* Else CPU is not idle, do nothing here: */
2262                 rq_unlock_irqrestore(rq, &rf);
2263         }
2264 
2265 out:
2266         rcu_read_unlock();
2267 }
2268 
2269 bool cpus_share_cache(int this_cpu, int that_cpu)
2270 {
2271         return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
2272 }
2273 #endif /* CONFIG_SMP */
2274 
2275 static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags)
2276 {
2277         struct rq *rq = cpu_rq(cpu);
2278         struct rq_flags rf;
2279 
2280 #if defined(CONFIG_SMP)
2281         if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) {
2282                 sched_clock_cpu(cpu); /* Sync clocks across CPUs */
2283                 ttwu_queue_remote(p, cpu, wake_flags);
2284                 return;
2285         }
2286 #endif
2287 
2288         rq_lock(rq, &rf);
2289         update_rq_clock(rq);
2290         ttwu_do_activate(rq, p, wake_flags, &rf);
2291         rq_unlock(rq, &rf);
2292 }
2293 
2294 /*
2295  * Notes on Program-Order guarantees on SMP systems.
2296  *
2297  *  MIGRATION
2298  *
2299  * The basic program-order guarantee on SMP systems is that when a task [t]
2300  * migrates, all its activity on its old CPU [c0] happens-before any subsequent
2301  * execution on its new CPU [c1].
2302  *
2303  * For migration (of runnable tasks) this is provided by the following means:
2304  *
2305  *  A) UNLOCK of the rq(c0)->lock scheduling out task t
2306  *  B) migration for t is required to synchronize *both* rq(c0)->lock and
2307  *     rq(c1)->lock (if not at the same time, then in that order).
2308  *  C) LOCK of the rq(c1)->lock scheduling in task
2309  *
2310  * Release/acquire chaining guarantees that B happens after A and C after B.
2311  * Note: the CPU doing B need not be c0 or c1
2312  *
2313  * Example:
2314  *
2315  *   CPU0            CPU1            CPU2
2316  *
2317  *   LOCK rq(0)->lock
2318  *   sched-out X
2319  *   sched-in Y
2320  *   UNLOCK rq(0)->lock
2321  *
2322  *                                   LOCK rq(0)->lock // orders against CPU0
2323  *                                   dequeue X
2324  *                                   UNLOCK rq(0)->lock
2325  *
2326  *                                   LOCK rq(1)->lock
2327  *                                   enqueue X
2328  *                                   UNLOCK rq(1)->lock
2329  *
2330  *                   LOCK rq(1)->lock // orders against CPU2
2331  *                   sched-out Z
2332  *                   sched-in X
2333  *                   UNLOCK rq(1)->lock
2334  *
2335  *
2336  *  BLOCKING -- aka. SLEEP + WAKEUP
2337  *
2338  * For blocking we (obviously) need to provide the same guarantee as for
2339  * migration. However the means are completely different as there is no lock
2340  * chain to provide order. Instead we do:
2341  *
2342  *   1) smp_store_release(X->on_cpu, 0)
2343  *   2) smp_cond_load_acquire(!X->on_cpu)
2344  *
2345  * Example:
2346  *
2347  *   CPU0 (schedule)  CPU1 (try_to_wake_up) CPU2 (schedule)
2348  *
2349  *   LOCK rq(0)->lock LOCK X->pi_lock
2350  *   dequeue X
2351  *   sched-out X
2352  *   smp_store_release(X->on_cpu, 0);
2353  *
2354  *                    smp_cond_load_acquire(&X->on_cpu, !VAL);
2355  *                    X->state = WAKING
2356  *                    set_task_cpu(X,2)
2357  *
2358  *                    LOCK rq(2)->lock
2359  *                    enqueue X
2360  *                    X->state = RUNNING
2361  *                    UNLOCK rq(2)->lock
2362  *
2363  *                                          LOCK rq(2)->lock // orders against CPU1
2364  *                                          sched-out Z
2365  *                                          sched-in X
2366  *                                          UNLOCK rq(2)->lock
2367  *
2368  *                    UNLOCK X->pi_lock
2369  *   UNLOCK rq(0)->lock
2370  *
2371  *
2372  * However, for wakeups there is a second guarantee we must provide, namely we
2373  * must ensure that CONDITION=1 done by the caller can not be reordered with
2374  * accesses to the task state; see try_to_wake_up() and set_current_state().
2375  */
2376 
2377 /**
2378  * try_to_wake_up - wake up a thread
2379  * @p: the thread to be awakened
2380  * @state: the mask of task states that can be woken
2381  * @wake_flags: wake modifier flags (WF_*)
2382  *
2383  * If (@state & @p->state) @p->state = TASK_RUNNING.
2384  *
2385  * If the task was not queued/runnable, also place it back on a runqueue.
2386  *
2387  * Atomic against schedule() which would dequeue a task, also see
2388  * set_current_state().
2389  *
2390  * This function executes a full memory barrier before accessing the task
2391  * state; see set_current_state().
2392  *
2393  * Return: %true if @p->state changes (an actual wakeup was done),
2394  *         %false otherwise.
2395  */
2396 static int
2397 try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
2398 {
2399         unsigned long flags;
2400         int cpu, success = 0;
2401 
2402         preempt_disable();
2403         if (p == current) {
2404                 /*
2405                  * We're waking current, this means 'p->on_rq' and 'task_cpu(p)
2406                  * == smp_processor_id()'. Together this means we can special
2407                  * case the whole 'p->on_rq && ttwu_remote()' case below
2408                  * without taking any locks.
2409                  *
2410                  * In particular:
2411                  *  - we rely on Program-Order guarantees for all the ordering,
2412                  *  - we're serialized against set_special_state() by virtue of
2413                  *    it disabling IRQs (this allows not taking ->pi_lock).
2414                  */
2415                 if (!(p->state & state))
2416                         goto out;
2417 
2418                 success = 1;
2419                 cpu = task_cpu(p);
2420                 trace_sched_waking(p);
2421                 p->state = TASK_RUNNING;
2422                 trace_sched_wakeup(p);
2423                 goto out;
2424         }
2425 
2426         /*
2427          * If we are going to wake up a thread waiting for CONDITION we
2428          * need to ensure that CONDITION=1 done by the caller can not be
2429          * reordered with p->state check below. This pairs with mb() in
2430          * set_current_state() the waiting thread does.
2431          */
2432         raw_spin_lock_irqsave(&p->pi_lock, flags);
2433         smp_mb__after_spinlock();
2434         if (!(p->state & state))
2435                 goto unlock;
2436 
2437         trace_sched_waking(p);
2438 
2439         /* We're going to change ->state: */
2440         success = 1;
2441         cpu = task_cpu(p);
2442 
2443         /*
2444          * Ensure we load p->on_rq _after_ p->state, otherwise it would
2445          * be possible to, falsely, observe p->on_rq == 0 and get stuck
2446          * in smp_cond_load_acquire() below.
2447          *
2448          * sched_ttwu_pending()                 try_to_wake_up()
2449          *   STORE p->on_rq = 1                   LOAD p->state
2450          *   UNLOCK rq->lock
2451          *
2452          * __schedule() (switch to task 'p')
2453          *   LOCK rq->lock                        smp_rmb();
2454          *   smp_mb__after_spinlock();
2455          *   UNLOCK rq->lock
2456          *
2457          * [task p]
2458          *   STORE p->state = UNINTERRUPTIBLE     LOAD p->on_rq
2459          *
2460          * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in
2461          * __schedule().  See the comment for smp_mb__after_spinlock().
2462          */
2463         smp_rmb();
2464         if (p->on_rq && ttwu_remote(p, wake_flags))
2465                 goto unlock;
2466 
2467 #ifdef CONFIG_SMP
2468         /*
2469          * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be
2470          * possible to, falsely, observe p->on_cpu == 0.
2471          *
2472          * One must be running (->on_cpu == 1) in order to remove oneself
2473          * from the runqueue.
2474          *
2475          * __schedule() (switch to task 'p')    try_to_wake_up()
2476          *   STORE p->on_cpu = 1                  LOAD p->on_rq
2477          *   UNLOCK rq->lock
2478          *
2479          * __schedule() (put 'p' to sleep)
2480          *   LOCK rq->lock                        smp_rmb();
2481          *   smp_mb__after_spinlock();
2482          *   STORE p->on_rq = 0                   LOAD p->on_cpu
2483          *
2484          * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in
2485          * __schedule().  See the comment for smp_mb__after_spinlock().
2486          */
2487         smp_rmb();
2488 
2489         /*
2490          * If the owning (remote) CPU is still in the middle of schedule() with
2491          * this task as prev, wait until its done referencing the task.
2492          *
2493          * Pairs with the smp_store_release() in finish_task().
2494          *
2495          * This ensures that tasks getting woken will be fully ordered against
2496          * their previous state and preserve Program Order.
2497          */
2498         smp_cond_load_acquire(&p->on_cpu, !VAL);
2499 
2500         p->sched_contributes_to_load = !!task_contributes_to_load(p);
2501         p->state = TASK_WAKING;
2502 
2503         if (p->in_iowait) {
2504                 delayacct_blkio_end(p);
2505                 atomic_dec(&task_rq(p)->nr_iowait);
2506         }
2507 
2508         cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags);
2509         if (task_cpu(p) != cpu) {
2510                 wake_flags |= WF_MIGRATED;
2511                 psi_ttwu_dequeue(p);
2512                 set_task_cpu(p, cpu);
2513         }
2514 
2515 #else /* CONFIG_SMP */
2516 
2517         if (p->in_iowait) {
2518                 delayacct_blkio_end(p);
2519                 atomic_dec(&task_rq(p)->nr_iowait);
2520         }
2521 
2522 #endif /* CONFIG_SMP */
2523 
2524         ttwu_queue(p, cpu, wake_flags);
2525 unlock:
2526         raw_spin_unlock_irqrestore(&p->pi_lock, flags);
2527 out:
2528         if (success)
2529                 ttwu_stat(p, cpu, wake_flags);
2530         preempt_enable();
2531 
2532         return success;
2533 }
2534 
2535 /**
2536  * wake_up_process - Wake up a specific process
2537  * @p: The process to be woken up.
2538  *
2539  * Attempt to wake up the nominated process and move it to the set of runnable
2540  * processes.
2541  *
2542  * Return: 1 if the process was woken up, 0 if it was already running.
2543  *
2544  * This function executes a full memory barrier before accessing the task state.
2545  */
2546 int wake_up_process(struct task_struct *p)
2547 {
2548         return try_to_wake_up(p, TASK_NORMAL, 0);
2549 }
2550 EXPORT_SYMBOL(wake_up_process);
2551 
2552 int wake_up_state(struct task_struct *p, unsigned int state)
2553 {
2554         return try_to_wake_up(p, state, 0);
2555 }
2556 
2557 /*
2558  * Perform scheduler related setup for a newly forked process p.
2559  * p is forked by current.
2560  *
2561  * __sched_fork() is basic setup used by init_idle() too:
2562  */
2563 static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
2564 {
2565         p->on_rq                        = 0;
2566 
2567         p->se.on_rq                     = 0;
2568         p->se.exec_start                = 0;
2569         p->se.sum_exec_runtime          = 0;
2570         p->se.prev_sum_exec_runtime     = 0;
2571         p->se.nr_migrations             = 0;
2572         p->se.vruntime                  = 0;
2573         INIT_LIST_HEAD(&p->se.group_node);
2574 
2575 #ifdef CONFIG_FAIR_GROUP_SCHED
2576         p->se.cfs_rq                    = NULL;
2577 #endif
2578 
2579 #ifdef CONFIG_SCHEDSTATS
2580         /* Even if schedstat is disabled, there should not be garbage */
2581         memset(&p->se.statistics, 0, sizeof(p->se.statistics));
2582 #endif
2583 
2584         RB_CLEAR_NODE(&p->dl.rb_node);
2585         init_dl_task_timer(&p->dl);
2586         init_dl_inactive_task_timer(&p->dl);
2587         __dl_clear_params(p);
2588 
2589         INIT_LIST_HEAD(&p->rt.run_list);
2590         p->rt.timeout           = 0;
2591         p->rt.time_slice        = sched_rr_timeslice;
2592         p->rt.on_rq             = 0;
2593         p->rt.on_list           = 0;
2594 
2595 #ifdef CONFIG_PREEMPT_NOTIFIERS
2596         INIT_HLIST_HEAD(&p->preempt_notifiers);
2597 #endif
2598 
2599 #ifdef CONFIG_COMPACTION
2600         p->capture_control = NULL;
2601 #endif
2602         init_numa_balancing(clone_flags, p);
2603 }
2604 
2605 DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
2606 
2607 #ifdef CONFIG_NUMA_BALANCING
2608 
2609 void set_numabalancing_state(bool enabled)
2610 {
2611         if (enabled)
2612                 static_branch_enable(&sched_numa_balancing);
2613         else
2614                 static_branch_disable(&sched_numa_balancing);
2615 }
2616 
2617 #ifdef CONFIG_PROC_SYSCTL
2618 int sysctl_numa_balancing(struct ctl_table *table, int write,
2619                          void __user *buffer, size_t *lenp, loff_t *ppos)
2620 {
2621         struct ctl_table t;
2622         int err;
2623         int state = static_branch_likely(&sched_numa_balancing);
2624 
2625         if (write && !capable(CAP_SYS_ADMIN))
2626                 return -EPERM;
2627 
2628         t = *table;
2629         t.data = &state;
2630         err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
2631         if (err < 0)
2632                 return err;
2633         if (write)
2634                 set_numabalancing_state(state);
2635         return err;
2636 }
2637 #endif
2638 #endif
2639 
2640 #ifdef CONFIG_SCHEDSTATS
2641 
2642 DEFINE_STATIC_KEY_FALSE(sched_schedstats);
2643 static bool __initdata __sched_schedstats = false;
2644 
2645 static void set_schedstats(bool enabled)
2646 {
2647         if (enabled)
2648                 static_branch_enable(&sched_schedstats);
2649         else
2650                 static_branch_disable(&sched_schedstats);
2651 }
2652 
2653 void force_schedstat_enabled(void)
2654 {
2655         if (!schedstat_enabled()) {
2656                 pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n");
2657                 static_branch_enable(&sched_schedstats);
2658         }
2659 }
2660 
2661 static int __init setup_schedstats(char *str)
2662 {
2663         int ret = 0;
2664         if (!str)
2665                 goto out;
2666 
2667         /*
2668          * This code is called before jump labels have been set up, so we can't
2669          * change the static branch directly just yet.  Instead set a temporary
2670          * variable so init_schedstats() can do it later.
2671          */
2672         if (!strcmp(str, "enable")) {
2673                 __sched_schedstats = true;
2674                 ret = 1;
2675         } else if (!strcmp(str, "disable")) {
2676                 __sched_schedstats = false;
2677                 ret = 1;
2678         }
2679 out:
2680         if (!ret)
2681                 pr_warn("Unable to parse schedstats=\n");
2682 
2683         return ret;
2684 }
2685 __setup("schedstats=", setup_schedstats);
2686 
2687 static void __init init_schedstats(void)
2688 {
2689         set_schedstats(__sched_schedstats);
2690 }
2691 
2692 #ifdef CONFIG_PROC_SYSCTL
2693 int sysctl_schedstats(struct ctl_table *table, int write,
2694                          void __user *buffer, size_t *lenp, loff_t *ppos)
2695 {
2696         struct ctl_table t;
2697         int err;
2698         int state = static_branch_likely(&sched_schedstats);
2699 
2700         if (write && !capable(CAP_SYS_ADMIN))
2701                 return -EPERM;
2702 
2703         t = *table;
2704         t.data = &state;
2705         err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
2706         if (err < 0)
2707                 return err;
2708         if (write)
2709                 set_schedstats(state);
2710         return err;
2711 }
2712 #endif /* CONFIG_PROC_SYSCTL */
2713 #else  /* !CONFIG_SCHEDSTATS */
2714 static inline void init_schedstats(void) {}
2715 #endif /* CONFIG_SCHEDSTATS */
2716 
2717 /*
2718  * fork()/clone()-time setup:
2719  */
2720 int sched_fork(unsigned long clone_flags, struct task_struct *p)
2721 {
2722         unsigned long flags;
2723 
2724         __sched_fork(clone_flags, p);
2725         /*
2726          * We mark the process as NEW here. This guarantees that
2727          * nobody will actually run it, and a signal or other external
2728          * event cannot wake it up and insert it on the runqueue either.
2729          */
2730         p->state = TASK_NEW;
2731 
2732         /*
2733          * Make sure we do not leak PI boosting priority to the child.
2734          */
2735         p->prio = current->normal_prio;
2736 
2737         uclamp_fork(p);
2738 
2739         /*
2740          * Revert to default priority/policy on fork if requested.
2741          */
2742         if (unlikely(p->sched_reset_on_fork)) {
2743                 if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
2744                         p->policy = SCHED_NORMAL;
2745                         p->static_prio = NICE_TO_PRIO(0);
2746                         p->rt_priority = 0;
2747                 } else if (PRIO_TO_NICE(p->static_prio) < 0)
2748                         p->static_prio = NICE_TO_PRIO(0);
2749 
2750                 p->prio = p->normal_prio = __normal_prio(p);
2751                 set_load_weight(p, false);
2752 
2753                 /*
2754                  * We don't need the reset flag anymore after the fork. It has
2755                  * fulfilled its duty:
2756                  */
2757                 p->sched_reset_on_fork = 0;
2758         }
2759 
2760         if (dl_prio(p->prio))
2761                 return -EAGAIN;
2762         else if (rt_prio(p->prio))
2763                 p->sched_class = &rt_sched_class;
2764         else
2765                 p->sched_class = &fair_sched_class;
2766 
2767         init_entity_runnable_average(&p->se);
2768 
2769         /*
2770          * The child is not yet in the pid-hash so no cgroup attach races,
2771          * and the cgroup is pinned to this child due to cgroup_fork()
2772          * is ran before sched_fork().
2773          *
2774          * Silence PROVE_RCU.
2775          */
2776         raw_spin_lock_irqsave(&p->pi_lock, flags);
2777         /*
2778          * We're setting the CPU for the first time, we don't migrate,
2779          * so use __set_task_cpu().
2780          */
2781         __set_task_cpu(p, smp_processor_id());
2782         if (p->sched_class->task_fork)
2783                 p->sched_class->task_fork(p);
2784         raw_spin_unlock_irqrestore(&p->pi_lock, flags);
2785 
2786 #ifdef CONFIG_SCHED_INFO
2787         if (likely(sched_info_on()))
2788                 memset(&p->sched_info, 0, sizeof(p->sched_info));
2789 #endif
2790 #if defined(CONFIG_SMP)
2791         p->on_cpu = 0;
2792 #endif
2793         init_task_preempt_count(p);
2794 #ifdef CONFIG_SMP
2795         plist_node_init(&p->pushable_tasks, MAX_PRIO);
2796         RB_CLEAR_NODE(&p->pushable_dl_tasks);
2797 #endif
2798         return 0;
2799 }
2800 
2801 unsigned long to_ratio(u64 period, u64 runtime)
2802 {
2803         if (runtime == RUNTIME_INF)
2804                 return BW_UNIT;
2805 
2806         /*
2807          * Doing this here saves a lot of checks in all
2808          * the calling paths, and returning zero seems
2809          * safe for them anyway.
2810          */
2811         if (period == 0)
2812                 return 0;
2813 
2814         return div64_u64(runtime << BW_SHIFT, period);
2815 }
2816 
2817 /*
2818  * wake_up_new_task - wake up a newly created task for the first time.
2819  *
2820  * This function will do some initial scheduler statistics housekeeping
2821  * that must be done for every newly created context, then puts the task
2822  * on the runqueue and wakes it.
2823  */
2824 void wake_up_new_task(struct task_struct *p)
2825 {
2826         struct rq_flags rf;
2827         struct rq *rq;
2828 
2829         raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
2830         p->state = TASK_RUNNING;
2831 #ifdef CONFIG_SMP
2832         /*
2833          * Fork balancing, do it here and not earlier because:
2834          *  - cpus_ptr can change in the fork path
2835          *  - any previously selected CPU might disappear through hotplug
2836          *
2837          * Use __set_task_cpu() to avoid calling sched_class::migrate_task_rq,
2838          * as we're not fully set-up yet.
2839          */
2840         p->recent_used_cpu = task_cpu(p);
2841         __set_task_cpu(p, select_task_rq(p, task_cpu(p), SD_BALANCE_FORK, 0));
2842 #endif
2843         rq = __task_rq_lock(p, &rf);
2844         update_rq_clock(rq);
2845         post_init_entity_util_avg(p);
2846 
2847         activate_task(rq, p, ENQUEUE_NOCLOCK);
2848         trace_sched_wakeup_new(p);
2849         check_preempt_curr(rq, p, WF_FORK);
2850 #ifdef CONFIG_SMP
2851         if (p->sched_class->task_woken) {
2852                 /*
2853                  * Nothing relies on rq->lock after this, so its fine to
2854                  * drop it.
2855                  */
2856                 rq_unpin_lock(rq, &rf);
2857                 p->sched_class->task_woken(rq, p);
2858                 rq_repin_lock(rq, &rf);
2859         }
2860 #endif
2861         task_rq_unlock(rq, p, &rf);
2862 }
2863 
2864 #ifdef CONFIG_PREEMPT_NOTIFIERS
2865 
2866 static DEFINE_STATIC_KEY_FALSE(preempt_notifier_key);
2867 
2868 void preempt_notifier_inc(void)
2869 {
2870         static_branch_inc(&preempt_notifier_key);
2871 }
2872 EXPORT_SYMBOL_GPL(preempt_notifier_inc);
2873 
2874 void preempt_notifier_dec(void)
2875 {
2876         static_branch_dec(&preempt_notifier_key);
2877 }
2878 EXPORT_SYMBOL_GPL(preempt_notifier_dec);
2879 
2880 /**
2881  * preempt_notifier_register - tell me when current is being preempted & rescheduled
2882  * @notifier: notifier struct to register
2883  */
2884 void preempt_notifier_register(struct preempt_notifier *notifier)
2885 {
2886         if (!static_branch_unlikely(&preempt_notifier_key))
2887                 WARN(1, "registering preempt_notifier while notifiers disabled\n");
2888 
2889         hlist_add_head(&notifier->link, &current->preempt_notifiers);
2890 }
2891 EXPORT_SYMBOL_GPL(preempt_notifier_register);
2892 
2893 /**
2894  * preempt_notifier_unregister - no longer interested in preemption notifications
2895  * @notifier: notifier struct to unregister
2896  *
2897  * This is *not* safe to call from within a preemption notifier.
2898  */
2899 void preempt_notifier_unregister(struct preempt_notifier *notifier)
2900 {
2901         hlist_del(&notifier->link);
2902 }
2903 EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
2904 
2905 static void __fire_sched_in_preempt_notifiers(struct task_struct *curr)
2906 {
2907         struct preempt_notifier *notifier;
2908 
2909         hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
2910                 notifier->ops->sched_in(notifier, raw_smp_processor_id());
2911 }
2912 
2913 static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
2914 {
2915         if (static_branch_unlikely(&preempt_notifier_key))
2916                 __fire_sched_in_preempt_notifiers(curr);
2917 }
2918 
2919 static void
2920 __fire_sched_out_preempt_notifiers(struct task_struct *curr,
2921                                    struct task_struct *next)
2922 {
2923         struct preempt_notifier *notifier;
2924 
2925         hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
2926                 notifier->ops->sched_out(notifier, next);
2927 }
2928 
2929 static __always_inline void
2930 fire_sched_out_preempt_notifiers(struct task_struct *curr,
2931                                  struct task_struct *next)
2932 {
2933         if (static_branch_unlikely(&preempt_notifier_key))
2934                 __fire_sched_out_preempt_notifiers(curr, next);
2935 }
2936 
2937 #else /* !CONFIG_PREEMPT_NOTIFIERS */
2938 
2939 static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
2940 {
2941 }
2942 
2943 static inline void
2944 fire_sched_out_preempt_notifiers(struct task_struct *curr,
2945                                  struct task_struct *next)
2946 {
2947 }
2948 
2949 #endif /* CONFIG_PREEMPT_NOTIFIERS */
2950 
2951 static inline void prepare_task(struct task_struct *next)
2952 {
2953 #ifdef CONFIG_SMP
2954         /*
2955          * Claim the task as running, we do this before switching to it
2956          * such that any running task will have this set.
2957          */
2958         next->on_cpu = 1;
2959 #endif
2960 }
2961 
2962 static inline void finish_task(struct task_struct *prev)
2963 {
2964 #ifdef CONFIG_SMP
2965         /*
2966          * After ->on_cpu is cleared, the task can be moved to a different CPU.
2967          * We must ensure this doesn't happen until the switch is completely
2968          * finished.
2969          *
2970          * In particular, the load of prev->state in finish_task_switch() must
2971          * happen before this.
2972          *
2973          * Pairs with the smp_cond_load_acquire() in try_to_wake_up().
2974          */
2975         smp_store_release(&prev->on_cpu, 0);
2976 #endif
2977 }
2978 
2979 static inline void
2980 prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf)
2981 {
2982         /*
2983          * Since the runqueue lock will be released by the next
2984          * task (which is an invalid locking op but in the case
2985          * of the scheduler it's an obvious special-case), so we
2986          * do an early lockdep release here:
2987          */
2988         rq_unpin_lock(rq, rf);
2989         spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
2990 #ifdef CONFIG_DEBUG_SPINLOCK
2991         /* this is a valid case when another task releases the spinlock */
2992         rq->lock.owner = next;
2993 #endif
2994 }
2995 
2996 static inline void finish_lock_switch(struct rq *rq)
2997 {
2998         /*
2999          * If we are tracking spinlock dependencies then we have to
3000          * fix up the runqueue lock - which gets 'carried over' from
3001          * prev into current:
3002          */
3003         spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);
3004         raw_spin_unlock_irq(&rq->lock);
3005 }
3006 
3007 /*
3008  * NOP if the arch has not defined these:
3009  */
3010 
3011 #ifndef prepare_arch_switch
3012 # define prepare_arch_switch(next)      do { } while (0)
3013 #endif
3014 
3015 #ifndef finish_arch_post_lock_switch
3016 # define finish_arch_post_lock_switch() do { } while (0)
3017 #endif
3018 
3019 /**
3020  * prepare_task_switch - prepare to switch tasks
3021  * @rq: the runqueue preparing to switch
3022  * @prev: the current task that is being switched out
3023  * @next: the task we are going to switch to.
3024  *
3025  * This is called with the rq lock held and interrupts off. It must
3026  * be paired with a subsequent finish_task_switch after the context
3027  * switch.
3028  *
3029  * prepare_task_switch sets up locking and calls architecture specific
3030  * hooks.
3031  */
3032 static inline void
3033 prepare_task_switch(struct rq *rq, struct task_struct *prev,
3034                     struct task_struct *next)
3035 {
3036         kcov_prepare_switch(prev);
3037         sched_info_switch(rq, prev, next);
3038         perf_event_task_sched_out(prev, next);
3039         rseq_preempt(prev);
3040         fire_sched_out_preempt_notifiers(prev, next);
3041         prepare_task(next);
3042         prepare_arch_switch(next);
3043 }
3044 
3045 /**
3046  * finish_task_switch - clean up after a task-switch
3047  * @prev: the thread we just switched away from.
3048  *
3049  * finish_task_switch must be called after the context switch, paired
3050  * with a prepare_task_switch call before the context switch.
3051  * finish_task_switch will reconcile locking set up by prepare_task_switch,
3052  * and do any other architecture-specific cleanup actions.
3053  *
3054  * Note that we may have delayed dropping an mm in context_switch(). If
3055  * so, we finish that here outside of the runqueue lock. (Doing it
3056  * with the lock held can cause deadlocks; see schedule() for
3057  * details.)
3058  *
3059  * The context switch have flipped the stack from under us and restored the
3060  * local variables which were saved when this task called schedule() in the
3061  * past. prev == current is still correct but we need to recalculate this_rq
3062  * because prev may have moved to another CPU.
3063  */
3064 static struct rq *finish_task_switch(struct task_struct *prev)
3065         __releases(rq->lock)
3066 {
3067         struct rq *rq = this_rq();
3068         struct mm_struct *mm = rq->prev_mm;
3069         long prev_state;
3070 
3071         /*
3072          * The previous task will have left us with a preempt_count of 2
3073          * because it left us after:
3074          *
3075          *      schedule()
3076          *        preempt_disable();                    // 1
3077          *        __schedule()
3078          *          raw_spin_lock_irq(&rq->lock)        // 2
3079          *
3080          * Also, see FORK_PREEMPT_COUNT.
3081          */
3082         if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET,
3083                       "corrupted preempt_count: %s/%d/0x%x\n",
3084                       current->comm, current->pid, preempt_count()))
3085                 preempt_count_set(FORK_PREEMPT_COUNT);
3086 
3087         rq->prev_mm = NULL;
3088 
3089         /*
3090          * A task struct has one reference for the use as "current".
3091          * If a task dies, then it sets TASK_DEAD in tsk->state and calls
3092          * schedule one last time. The schedule call will never return, and
3093          * the scheduled task must drop that reference.
3094          *
3095          * We must observe prev->state before clearing prev->on_cpu (in
3096          * finish_task), otherwise a concurrent wakeup can get prev
3097          * running on another CPU and we could rave with its RUNNING -> DEAD
3098          * transition, resulting in a double drop.
3099          */
3100         prev_state = prev->state;
3101         vtime_task_switch(prev);
3102         perf_event_task_sched_in(prev, current);
3103         finish_task(prev);
3104         finish_lock_switch(rq);
3105         finish_arch_post_lock_switch();
3106         kcov_finish_switch(current);
3107 
3108         fire_sched_in_preempt_notifiers(current);
3109         /*
3110          * When switching through a kernel thread, the loop in
3111          * membarrier_{private,global}_expedited() may have observed that
3112          * kernel thread and not issued an IPI. It is therefore possible to
3113          * schedule between user->kernel->user threads without passing though
3114          * switch_mm(). Membarrier requires a barrier after storing to
3115          * rq->curr, before returning to userspace, so provide them here:
3116          *
3117          * - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly
3118          *   provided by mmdrop(),
3119          * - a sync_core for SYNC_CORE.
3120          */
3121         if (mm) {
3122                 membarrier_mm_sync_core_before_usermode(mm);
3123                 mmdrop(mm);
3124         }
3125         if (unlikely(prev_state == TASK_DEAD)) {
3126                 if (prev->sched_class->task_dead)
3127                         prev->sched_class->task_dead(prev);
3128 
3129                 /*
3130                  * Remove function-return probe instances associated with this
3131                  * task and put them back on the free list.
3132                  */
3133                 kprobe_flush_task(prev);
3134 
3135                 /* Task is done with its stack. */
3136                 put_task_stack(prev);
3137 
3138                 put_task_struct(prev);
3139         }
3140 
3141         tick_nohz_task_switch();
3142         return rq;
3143 }
3144 
3145 #ifdef CONFIG_SMP
3146 
3147 /* rq->lock is NOT held, but preemption is disabled */
3148 static void __balance_callback(struct rq *rq)
3149 {
3150         struct callback_head *head, *next;
3151         void (*func)(struct rq *rq);
3152         unsigned long flags;
3153 
3154         raw_spin_lock_irqsave(&rq->lock, flags);
3155         head = rq->balance_callback;
3156         rq->balance_callback = NULL;
3157         while (head) {
3158                 func = (void (*)(struct rq *))head->func;
3159                 next = head->next;
3160                 head->next = NULL;
3161                 head = next;
3162 
3163                 func(rq);
3164         }
3165         raw_spin_unlock_irqrestore(&rq->lock, flags);
3166 }
3167 
3168 static inline void balance_callback(struct rq *rq)
3169 {
3170         if (unlikely(rq->balance_callback))
3171                 __balance_callback(rq);
3172 }
3173 
3174 #else
3175 
3176 static inline void balance_callback(struct rq *rq)
3177 {
3178 }
3179 
3180 #endif
3181 
3182 /**
3183  * schedule_tail - first thing a freshly forked thread must call.
3184  * @prev: the thread we just switched away from.
3185  */
3186 asmlinkage __visible void schedule_tail(struct task_struct *prev)
3187         __releases(rq->lock)
3188 {
3189         struct rq *rq;
3190 
3191         /*
3192          * New tasks start with FORK_PREEMPT_COUNT, see there and
3193          * finish_task_switch() for details.
3194          *
3195          * finish_task_switch() will drop rq->lock() and lower preempt_count
3196          * and the preempt_enable() will end up enabling preemption (on
3197          * PREEMPT_COUNT kernels).
3198          */
3199 
3200         rq = finish_task_switch(prev);
3201         balance_callback(rq);
3202         preempt_enable();
3203 
3204         if (current->set_child_tid)
3205                 put_user(task_pid_vnr(current), current->set_child_tid);
3206 
3207         calculate_sigpending();
3208 }
3209 
3210 /*
3211  * context_switch - switch to the new MM and the new thread's register state.
3212  */
3213 static __always_inline struct rq *
3214 context_switch(struct rq *rq, struct task_struct *prev,
3215                struct task_struct *next, struct rq_flags *rf)
3216 {
3217         struct mm_struct *mm, *oldmm;
3218 
3219         prepare_task_switch(rq, prev, next);
3220 
3221         mm = next->mm;
3222         oldmm = prev->active_mm;
3223         /*
3224          * For paravirt, this is coupled with an exit in switch_to to
3225          * combine the page table reload and the switch backend into
3226          * one hypercall.
3227          */
3228         arch_start_context_switch(prev);
3229 
3230         /*
3231          * If mm is non-NULL, we pass through switch_mm(). If mm is
3232          * NULL, we will pass through mmdrop() in finish_task_switch().
3233          * Both of these contain the full memory barrier required by
3234          * membarrier after storing to rq->curr, before returning to
3235          * user-space.
3236          */
3237         if (!mm) {
3238                 next->active_mm = oldmm;
3239                 mmgrab(oldmm);
3240                 enter_lazy_tlb(oldmm, next);
3241         } else
3242                 switch_mm_irqs_off(oldmm, mm, next);
3243 
3244         if (!prev->mm) {
3245                 prev->active_mm = NULL;
3246                 rq->prev_mm = oldmm;
3247         }
3248 
3249         rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP);
3250 
3251         prepare_lock_switch(rq, next, rf);
3252 
3253         /* Here we just switch the register state and the stack. */
3254         switch_to(prev, next, prev);
3255         barrier();
3256 
3257         return finish_task_switch(prev);
3258 }
3259 
3260 /*
3261  * nr_running and nr_context_switches:
3262  *
3263  * externally visible scheduler statistics: current number of runnable
3264  * threads, total number of context switches performed since bootup.
3265  */
3266 unsigned long nr_running(void)
3267 {
3268         unsigned long i, sum = 0;
3269 
3270         for_each_online_cpu(i)
3271                 sum += cpu_rq(i)->nr_running;
3272 
3273         return sum;
3274 }
3275 
3276 /*
3277  * Check if only the current task is running on the CPU.
3278  *
3279  * Caution: this function does not check that the caller has disabled
3280  * preemption, thus the result might have a time-of-check-to-time-of-use
3281  * race.  The caller is responsible to use it correctly, for example:
3282  *
3283  * - from a non-preemptible section (of course)
3284  *
3285  * - from a thread that is bound to a single CPU
3286  *
3287  * - in a loop with very short iterations (e.g. a polling loop)
3288  */
3289 bool single_task_running(void)
3290 {
3291         return raw_rq()->nr_running == 1;
3292 }
3293 EXPORT_SYMBOL(single_task_running);
3294 
3295 unsigned long long nr_context_switches(void)
3296 {
3297         int i;
3298         unsigned long long sum = 0;
3299 
3300         for_each_possible_cpu(i)
3301                 sum += cpu_rq(i)->nr_switches;
3302 
3303         return sum;
3304 }
3305 
3306 /*
3307  * Consumers of these two interfaces, like for example the cpuidle menu
3308  * governor, are using nonsensical data. Preferring shallow idle state selection
3309  * for a CPU that has IO-wait which might not even end up running the task when
3310  * it does become runnable.
3311  */
3312 
3313 unsigned long nr_iowait_cpu(int cpu)
3314 {
3315         return atomic_read(&cpu_rq(cpu)->nr_iowait);
3316 }
3317 
3318 /*
3319  * IO-wait accounting, and how its mostly bollocks (on SMP).
3320  *
3321  * The idea behind IO-wait account is to account the idle time that we could
3322  * have spend running if it were not for IO. That is, if we were to improve the
3323  * storage performance, we'd have a proportional reduction in IO-wait time.
3324  *
3325  * This all works nicely on UP, where, when a task blocks on IO, we account
3326  * idle time as IO-wait, because if the storage were faster, it could've been
3327  * running and we'd not be idle.
3328  *
3329  * This has been extended to SMP, by doing the same for each CPU. This however
3330  * is broken.
3331  *
3332  * Imagine for instance the case where two tasks block on one CPU, only the one
3333  * CPU will have IO-wait accounted, while the other has regular idle. Even
3334  * though, if the storage were faster, both could've ran at the same time,
3335  * utilising both CPUs.
3336  *
3337  * This means, that when looking globally, the current IO-wait accounting on
3338  * SMP is a lower bound, by reason of under accounting.
3339  *
3340  * Worse, since the numbers are provided per CPU, they are sometimes
3341  * interpreted per CPU, and that is nonsensical. A blocked task isn't strictly
3342  * associated with any one particular CPU, it can wake to another CPU than it
3343  * blocked on. This means the per CPU IO-wait number is meaningless.
3344  *
3345  * Task CPU affinities can make all that even more 'interesting'.
3346  */
3347 
3348 unsigned long nr_iowait(void)
3349 {
3350         unsigned long i, sum = 0;
3351 
3352         for_each_possible_cpu(i)
3353                 sum += nr_iowait_cpu(i);
3354 
3355         return sum;
3356 }
3357 
3358 #ifdef CONFIG_SMP
3359 
3360 /*
3361  * sched_exec - execve() is a valuable balancing opportunity, because at
3362  * this point the task has the smallest effective memory and cache footprint.
3363  */
3364 void sched_exec(void)
3365 {
3366         struct task_struct *p = current;
3367         unsigned long flags;
3368         int dest_cpu;
3369 
3370         raw_spin_lock_irqsave(&p->pi_lock, flags);
3371         dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0);
3372         if (dest_cpu == smp_processor_id())
3373                 goto unlock;
3374 
3375         if (likely(cpu_active(dest_cpu))) {
3376                 struct migration_arg arg = { p, dest_cpu };
3377 
3378                 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
3379                 stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg);
3380                 return;
3381         }
3382 unlock:
3383         raw_spin_unlock_irqrestore(&p->pi_lock, flags);
3384 }
3385 
3386 #endif
3387 
3388 DEFINE_PER_CPU(struct kernel_stat, kstat);
3389 DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat);
3390 
3391 EXPORT_PER_CPU_SYMBOL(kstat);
3392 EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
3393 
3394 /*
3395  * The function fair_sched_class.update_curr accesses the struct curr
3396  * and its field curr->exec_start; when called from task_sched_runtime(),
3397  * we observe a high rate of cache misses in practice.
3398  * Prefetching this data results in improved performance.
3399  */
3400 static inline void prefetch_curr_exec_start(struct task_struct *p)
3401 {
3402 #ifdef CONFIG_FAIR_GROUP_SCHED
3403         struct sched_entity *curr = (&p->se)->cfs_rq->curr;
3404 #else
3405         struct sched_entity *curr = (&task_rq(p)->cfs)->curr;
3406 #endif
3407         prefetch(curr);
3408         prefetch(&curr->exec_start);
3409 }
3410 
3411 /*
3412  * Return accounted runtime for the task.
3413  * In case the task is currently running, return the runtime plus current's
3414  * pending runtime that have not been accounted yet.
3415  */
3416 unsigned long long task_sched_runtime(struct task_struct *p)
3417 {
3418         struct rq_flags rf;
3419         struct rq *rq;
3420         u64 ns;
3421 
3422 #if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
3423         /*
3424          * 64-bit doesn't need locks to atomically read a 64-bit value.
3425          * So we have a optimization chance when the task's delta_exec is 0.
3426          * Reading ->on_cpu is racy, but this is ok.
3427          *
3428          * If we race with it leaving CPU, we'll take a lock. So we're correct.
3429          * If we race with it entering CPU, unaccounted time is 0. This is
3430          * indistinguishable from the read occurring a few cycles earlier.
3431          * If we see ->on_cpu without ->on_rq, the task is leaving, and has
3432          * been accounted, so we're correct here as well.
3433          */
3434         if (!p->on_cpu || !task_on_rq_queued(p))
3435                 return p->se.sum_exec_runtime;
3436 #endif
3437 
3438         rq = task_rq_lock(p, &rf);
3439         /*
3440          * Must be ->curr _and_ ->on_rq.  If dequeued, we would
3441          * project cycles that may never be accounted to this
3442          * thread, breaking clock_gettime().
3443          */
3444         if (task_current(rq, p) && task_on_rq_queued(p)) {
3445                 prefetch_curr_exec_start(p);
3446                 update_rq_clock(rq);
3447                 p->sched_class->update_curr(rq);
3448         }
3449         ns = p->se.sum_exec_runtime;
3450         task_rq_unlock(rq, p, &rf);
3451 
3452         return ns;
3453 }
3454 
3455 /*
3456  * This function gets called by the timer code, with HZ frequency.
3457  * We call it with interrupts disabled.
3458  */
3459 void scheduler_tick(void)
3460 {
3461         int cpu = smp_processor_id();
3462         struct rq *rq = cpu_rq(cpu);
3463         struct task_struct *curr = rq->curr;
3464         struct rq_flags rf;
3465 
3466         sched_clock_tick();
3467 
3468         rq_lock(rq, &rf);
3469 
3470         update_rq_clock(rq);
3471         curr->sched_class->task_tick(rq, curr, 0);
3472         calc_global_load_tick(rq);
3473         psi_task_tick(rq);
3474 
3475         rq_unlock(rq, &rf);
3476 
3477         perf_event_task_tick();
3478 
3479 #ifdef CONFIG_SMP
3480         rq->idle_balance = idle_cpu(cpu);
3481         trigger_load_balance(rq);
3482 #endif
3483 }
3484 
3485 #ifdef CONFIG_NO_HZ_FULL
3486 
3487 struct tick_work {
3488         int                     cpu;
3489         atomic_t                state;
3490         struct delayed_work     work;
3491 };
3492 /* Values for ->state, see diagram below. */
3493 #define TICK_SCHED_REMOTE_OFFLINE       0
3494 #define TICK_SCHED_REMOTE_OFFLINING     1
3495 #define TICK_SCHED_REMOTE_RUNNING       2
3496 
3497 /*
3498  * State diagram for ->state:
3499  *
3500  *
3501  *          TICK_SCHED_REMOTE_OFFLINE
3502  *                    |   ^
3503  *                    |   |
3504  *                    |   | sched_tick_remote()
3505  *                    |   |
3506  *                    |   |
3507  *                    +--TICK_SCHED_REMOTE_OFFLINING
3508  *                    |   ^
3509  *                    |   |
3510  * sched_tick_start() |   | sched_tick_stop()
3511  *                    |   |
3512  *                    V   |
3513  *          TICK_SCHED_REMOTE_RUNNING
3514  *
3515  *
3516  * Other transitions get WARN_ON_ONCE(), except that sched_tick_remote()
3517  * and sched_tick_start() are happy to leave the state in RUNNING.
3518  */
3519 
3520 static struct tick_work __percpu *tick_work_cpu;
3521 
3522 static void sched_tick_remote(struct work_struct *work)
3523 {
3524         struct delayed_work *dwork = to_delayed_work(work);
3525         struct tick_work *twork = container_of(dwork, struct tick_work, work);
3526         int cpu = twork->cpu;
3527         struct rq *rq = cpu_rq(cpu);
3528         struct task_struct *curr;
3529         struct rq_flags rf;
3530         u64 delta;
3531         int os;
3532 
3533         /*
3534          * Handle the tick only if it appears the remote CPU is running in full
3535          * dynticks mode. The check is racy by nature, but missing a tick or
3536          * having one too much is no big deal because the scheduler tick updates
3537          * statistics and checks timeslices in a time-independent way, regardless
3538          * of when exactly it is running.
3539          */
3540         if (idle_cpu(cpu) || !tick_nohz_tick_stopped_cpu(cpu))
3541                 goto out_requeue;
3542 
3543         rq_lock_irq(rq, &rf);
3544         curr = rq->curr;
3545         if (is_idle_task(curr) || cpu_is_offline(cpu))
3546                 goto out_unlock;
3547 
3548         update_rq_clock(rq);
3549         delta = rq_clock_task(rq) - curr->se.exec_start;
3550 
3551         /*
3552          * Make sure the next tick runs within a reasonable
3553          * amount of time.
3554          */
3555         WARN_ON_ONCE(delta > (u64)NSEC_PER_SEC * 3);
3556         curr->sched_class->task_tick(rq, curr, 0);
3557 
3558 out_unlock:
3559         rq_unlock_irq(rq, &rf);
3560 
3561 out_requeue:
3562         /*
3563          * Run the remote tick once per second (1Hz). This arbitrary
3564          * frequency is large enough to avoid overload but short enough
3565          * to keep scheduler internal stats reasonably up to date.  But
3566          * first update state to reflect hotplug activity if required.
3567          */
3568         os = atomic_fetch_add_unless(&twork->state, -1, TICK_SCHED_REMOTE_RUNNING);
3569         WARN_ON_ONCE(os == TICK_SCHED_REMOTE_OFFLINE);
3570         if (os == TICK_SCHED_REMOTE_RUNNING)
3571                 queue_delayed_work(system_unbound_wq, dwork, HZ);
3572 }
3573 
3574 static void sched_tick_start(int cpu)
3575 {
3576         int os;
3577         struct tick_work *twork;
3578 
3579         if (housekeeping_cpu(cpu, HK_FLAG_TICK))
3580                 return;
3581 
3582         WARN_ON_ONCE(!tick_work_cpu);
3583 
3584         twork = per_cpu_ptr(tick_work_cpu, cpu);
3585         os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_RUNNING);
3586         WARN_ON_ONCE(os == TICK_SCHED_REMOTE_RUNNING);
3587         if (os == TICK_SCHED_REMOTE_OFFLINE) {
3588                 twork->cpu = cpu;
3589                 INIT_DELAYED_WORK(&twork->work, sched_tick_remote);
3590                 queue_delayed_work(system_unbound_wq, &twork->work, HZ);
3591         }
3592 }
3593 
3594 #ifdef CONFIG_HOTPLUG_CPU
3595 static void sched_tick_stop(int cpu)
3596 {
3597         struct tick_work *twork;
3598         int os;
3599 
3600         if (housekeeping_cpu(cpu, HK_FLAG_TICK))
3601                 return;
3602 
3603         WARN_ON_ONCE(!tick_work_cpu);
3604 
3605         twork = per_cpu_ptr(tick_work_cpu, cpu);
3606         /* There cannot be competing actions, but don't rely on stop-machine. */
3607         os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_OFFLINING);
3608         WARN_ON_ONCE(os != TICK_SCHED_REMOTE_RUNNING);
3609         /* Don't cancel, as this would mess up the state machine. */
3610 }
3611 #endif /* CONFIG_HOTPLUG_CPU */
3612 
3613 int __init sched_tick_offload_init(void)
3614 {
3615         tick_work_cpu = alloc_percpu(struct tick_work);
3616         BUG_ON(!tick_work_cpu);
3617         return 0;
3618 }
3619 
3620 #else /* !CONFIG_NO_HZ_FULL */
3621 static inline void sched_tick_start(int cpu) { }
3622 static inline void sched_tick_stop(int cpu) { }
3623 #endif
3624 
3625 #if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
3626                                 defined(CONFIG_TRACE_PREEMPT_TOGGLE))
3627 /*
3628  * If the value passed in is equal to the current preempt count
3629  * then we just disabled preemption. Start timing the latency.
3630  */
3631 static inline void preempt_latency_start(int val)
3632 {
3633         if (preempt_count() == val) {
3634                 unsigned long ip = get_lock_parent_ip();
3635 #ifdef CONFIG_DEBUG_PREEMPT
3636                 current->preempt_disable_ip = ip;
3637 #endif
3638                 trace_preempt_off(CALLER_ADDR0, ip);
3639         }
3640 }
3641 
3642 void preempt_count_add(int val)
3643 {
3644 #ifdef CONFIG_DEBUG_PREEMPT
3645         /*
3646          * Underflow?
3647          */
3648         if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
3649                 return;
3650 #endif
3651         __preempt_count_add(val);
3652 #ifdef CONFIG_DEBUG_PREEMPT
3653         /*
3654          * Spinlock count overflowing soon?
3655          */
3656         DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
3657                                 PREEMPT_MASK - 10);
3658 #endif
3659         preempt_latency_start(val);
3660 }
3661 EXPORT_SYMBOL(preempt_count_add);
3662 NOKPROBE_SYMBOL(preempt_count_add);
3663 
3664 /*
3665  * If the value passed in equals to the current preempt count
3666  * then we just enabled preemption. Stop timing the latency.
3667  */
3668 static inline void preempt_latency_stop(int val)
3669 {
3670         if (preempt_count() == val)
3671                 trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip());
3672 }
3673 
3674 void preempt_count_sub(int val)
3675 {
3676 #ifdef CONFIG_DEBUG_PREEMPT
3677         /*
3678          * Underflow?
3679          */
3680         if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
3681                 return;
3682         /*
3683          * Is the spinlock portion underflowing?
3684          */
3685         if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
3686                         !(preempt_count() & PREEMPT_MASK)))
3687                 return;
3688 #endif
3689 
3690         preempt_latency_stop(val);
3691         __preempt_count_sub(val);
3692 }
3693 EXPORT_SYMBOL(preempt_count_sub);
3694 NOKPROBE_SYMBOL(preempt_count_sub);
3695 
3696 #else
3697 static inline void preempt_latency_start(int val) { }
3698 static inline void preempt_latency_stop(int val) { }
3699 #endif
3700 
3701 static inline unsigned long get_preempt_disable_ip(struct task_struct *p)
3702 {
3703 #ifdef CONFIG_DEBUG_PREEMPT
3704         return p->preempt_disable_ip;
3705 #else
3706         return 0;
3707 #endif
3708 }
3709 
3710 /*
3711  * Print scheduling while atomic bug:
3712  */
3713 static noinline void __schedule_bug(struct task_struct *prev)
3714 {
3715         /* Save this before calling printk(), since that will clobber it */
3716         unsigned long preempt_disable_ip = get_preempt_disable_ip(current);
3717 
3718         if (oops_in_progress)
3719                 return;
3720 
3721         printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
3722                 prev->comm, prev->pid, preempt_count());
3723 
3724         debug_show_held_locks(prev);
3725         print_modules();
3726         if (irqs_disabled())
3727                 print_irqtrace_events(prev);
3728         if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)
3729             && in_atomic_preempt_off()) {
3730                 pr_err("Preemption disabled at:");
3731                 print_ip_sym(preempt_disable_ip);
3732                 pr_cont("\n");
3733         }
3734         if (panic_on_warn)
3735                 panic("scheduling while atomic\n");
3736 
3737         dump_stack();
3738         add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
3739 }
3740 
3741 /*
3742  * Various schedule()-time debugging checks and statistics:
3743  */
3744 static inline void schedule_debug(struct task_struct *prev)
3745 {
3746 #ifdef CONFIG_SCHED_STACK_END_CHECK
3747         if (task_stack_end_corrupted(prev))
3748                 panic("corrupted stack end detected inside scheduler\n");
3749 #endif
3750 
3751         if (unlikely(in_atomic_preempt_off())) {
3752                 __schedule_bug(prev);
3753                 preempt_count_set(PREEMPT_DISABLED);
3754         }
3755         rcu_sleep_check();
3756 
3757         profile_hit(SCHED_PROFILING, __builtin_return_address(0));
3758 
3759         schedstat_inc(this_rq()->sched_count);
3760 }
3761 
3762 /*
3763  * Pick up the highest-prio task:
3764  */
3765 static inline struct task_struct *
3766 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
3767 {
3768         const struct sched_class *class;
3769         struct task_struct *p;
3770 
3771         /*
3772          * Optimization: we know that if all tasks are in the fair class we can
3773          * call that function directly, but only if the @prev task wasn't of a
3774          * higher scheduling class, because otherwise those loose the
3775          * opportunity to pull in more work from other CPUs.
3776          */
3777         if (likely((prev->sched_class == &idle_sched_class ||
3778                     prev->sched_class == &fair_sched_class) &&
3779                    rq->nr_running == rq->cfs.h_nr_running)) {
3780 
3781                 p = fair_sched_class.pick_next_task(rq, prev, rf);
3782                 if (unlikely(p == RETRY_TASK))
3783                         goto again;
3784 
3785                 /* Assumes fair_sched_class->next == idle_sched_class */
3786                 if (unlikely(!p))
3787                         p = idle_sched_class.pick_next_task(rq, prev, rf);
3788 
3789                 return p;
3790         }
3791 
3792 again:
3793         for_each_class(class) {
3794                 p = class->pick_next_task(rq, prev, rf);
3795                 if (p) {
3796                         if (unlikely(p == RETRY_TASK))
3797                                 goto again;
3798                         return p;
3799                 }
3800         }
3801 
3802         /* The idle class should always have a runnable task: */
3803         BUG();
3804 }
3805 
3806 /*
3807  * __schedule() is the main scheduler function.
3808  *
3809  * The main means of driving the scheduler and thus entering this function are:
3810  *
3811  *   1. Explicit blocking: mutex, semaphore, waitqueue, etc.
3812  *
3813  *   2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return
3814  *      paths. For example, see arch/x86/entry_64.S.
3815  *
3816  *      To drive preemption between tasks, the scheduler sets the flag in timer
3817  *      interrupt handler scheduler_tick().
3818  *
3819  *   3. Wakeups don't really cause entry into schedule(). They add a
3820  *      task to the run-queue and that's it.
3821  *
3822  *      Now, if the new task added to the run-queue preempts the current
3823  *      task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets
3824  *      called on the nearest possible occasion:
3825  *
3826  *       - If the kernel is preemptible (CONFIG_PREEMPT=y):
3827  *
3828  *         - in syscall or exception context, at the next outmost
3829  *           preempt_enable(). (this might be as soon as the wake_up()'s
3830  *           spin_unlock()!)
3831  *
3832  *         - in IRQ context, return from interrupt-handler to
3833  *           preemptible context
3834  *
3835  *       - If the kernel is not preemptible (CONFIG_PREEMPT is not set)
3836  *         then at the next:
3837  *
3838  *          - cond_resched() call
3839  *          - explicit schedule() call
3840  *          - return from syscall or exception to user-space
3841  *          - return from interrupt-handler to user-space
3842  *
3843  * WARNING: must be called with preemption disabled!
3844  */
3845 static void __sched notrace __schedule(bool preempt)
3846 {
3847         struct task_struct *prev, *next;
3848         unsigned long *switch_count;
3849         struct rq_flags rf;
3850         struct rq *rq;
3851         int cpu;
3852 
3853         cpu = smp_processor_id();
3854         rq = cpu_rq(cpu);
3855         prev = rq->curr;
3856 
3857         schedule_debug(prev);
3858 
3859         if (sched_feat(HRTICK))
3860                 hrtick_clear(rq);
3861 
3862         local_irq_disable();
3863         rcu_note_context_switch(preempt);
3864 
3865         /*
3866          * Make sure that signal_pending_state()->signal_pending() below
3867          * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE)
3868          * done by the caller to avoid the race with signal_wake_up().
3869          *
3870          * The membarrier system call requires a full memory barrier
3871          * after coming from user-space, before storing to rq->curr.
3872          */
3873         rq_lock(rq, &rf);
3874         smp_mb__after_spinlock();
3875 
3876         /* Promote REQ to ACT */
3877         rq->clock_update_flags <<= 1;
3878         update_rq_clock(rq);
3879 
3880         switch_count = &prev->nivcsw;
3881         if (!preempt && prev->state) {
3882                 if (signal_pending_state(prev->state, prev)) {
3883                         prev->state = TASK_RUNNING;
3884                 } else {
3885                         deactivate_task(rq, prev, DEQUEUE_SLEEP | DEQUEUE_NOCLOCK);
3886 
3887                         if (prev->in_iowait) {
3888                                 atomic_inc(&rq->nr_iowait);
3889                                 delayacct_blkio_start();
3890                         }
3891                 }
3892                 switch_count = &prev->nvcsw;
3893         }
3894 
3895         next = pick_next_task(rq, prev, &rf);
3896         clear_tsk_need_resched(prev);
3897         clear_preempt_need_resched();
3898 
3899         if (likely(prev != next)) {
3900                 rq->nr_switches++;
3901                 rq->curr = next;
3902                 /*
3903                  * The membarrier system call requires each architecture
3904                  * to have a full memory barrier after updating
3905                  * rq->curr, before returning to user-space.
3906                  *
3907                  * Here are the schemes providing that barrier on the
3908                  * various architectures:
3909                  * - mm ? switch_mm() : mmdrop() for x86, s390, sparc, PowerPC.
3910                  *   switch_mm() rely on membarrier_arch_switch_mm() on PowerPC.
3911                  * - finish_lock_switch() for weakly-ordered
3912                  *   architectures where spin_unlock is a full barrier,
3913                  * - switch_to() for arm64 (weakly-ordered, spin_unlock
3914                  *   is a RELEASE barrier),
3915                  */
3916                 ++*switch_count;
3917 
3918                 trace_sched_switch(preempt, prev, next);
3919 
3920                 /* Also unlocks the rq: */
3921                 rq = context_switch(rq, prev, next, &rf);
3922         } else {
3923                 rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP);
3924                 rq_unlock_irq(rq, &rf);
3925         }
3926 
3927         balance_callback(rq);
3928 }
3929 
3930 void __noreturn do_task_dead(void)
3931 {
3932         /* Causes final put_task_struct in finish_task_switch(): */
3933         set_special_state(TASK_DEAD);
3934 
3935         /* Tell freezer to ignore us: */
3936         current->flags |= PF_NOFREEZE;
3937 
3938         __schedule(false);
3939         BUG();
3940 
3941         /* Avoid "noreturn function does return" - but don't continue if BUG() is a NOP: */
3942         for (;;)
3943                 cpu_relax();
3944 }
3945 
3946 static inline void sched_submit_work(struct task_struct *tsk)
3947 {
3948         if (!tsk->state)
3949                 return;
3950 
3951         /*
3952          * If a worker went to sleep, notify and ask workqueue whether
3953          * it wants to wake up a task to maintain concurrency.
3954          * As this function is called inside the schedule() context,
3955          * we disable preemption to avoid it calling schedule() again
3956          * in the possible wakeup of a kworker.
3957          */
3958         if (tsk->flags & PF_WQ_WORKER) {
3959                 preempt_disable();
3960                 wq_worker_sleeping(tsk);
3961                 preempt_enable_no_resched();
3962         }
3963 
3964         if (tsk_is_pi_blocked(tsk))
3965                 return;
3966 
3967         /*
3968          * If we are going to sleep and we have plugged IO queued,
3969          * make sure to submit it to avoid deadlocks.
3970          */
3971         if (blk_needs_flush_plug(tsk))
3972                 blk_schedule_flush_plug(tsk);
3973 }
3974 
3975 static void sched_update_worker(struct task_struct *tsk)
3976 {
3977         if (tsk->flags & PF_WQ_WORKER)
3978                 wq_worker_running(tsk);
3979 }
3980 
3981 asmlinkage __visible void __sched schedule(void)
3982 {
3983         struct task_struct *tsk = current;
3984 
3985         sched_submit_work(tsk);
3986         do {
3987                 preempt_disable();
3988                 __schedule(false);
3989                 sched_preempt_enable_no_resched();
3990         } while (need_resched());
3991         sched_update_worker(tsk);
3992 }
3993 EXPORT_SYMBOL(schedule);
3994 
3995 /*
3996  * synchronize_rcu_tasks() makes sure that no task is stuck in preempted
3997  * state (have scheduled out non-voluntarily) by making sure that all
3998  * tasks have either left the run queue or have gone into user space.
3999  * As idle tasks do not do either, they must not ever be preempted
4000  * (schedule out non-voluntarily).
4001  *
4002  * schedule_idle() is similar to schedule_preempt_disable() except that it
4003  * never enables preemption because it does not call sched_submit_work().
4004  */
4005 void __sched schedule_idle(void)
4006 {
4007         /*
4008          * As this skips calling sched_submit_work(), which the idle task does
4009          * regardless because that function is a nop when the task is in a
4010          * TASK_RUNNING state, make sure this isn't used someplace that the
4011          * current task can be in any other state. Note, idle is always in the
4012          * TASK_RUNNING state.
4013          */
4014         WARN_ON_ONCE(current->state);
4015         do {
4016                 __schedule(false);
4017         } while (need_resched());
4018 }
4019 
4020 #ifdef CONFIG_CONTEXT_TRACKING
4021 asmlinkage __visible void __sched schedule_user(void)
4022 {
4023         /*
4024          * If we come here after a random call to set_need_resched(),
4025          * or we have been woken up remotely but the IPI has not yet arrived,
4026          * we haven't yet exited the RCU idle mode. Do it here manually until
4027          * we find a better solution.
4028          *
4029          * NB: There are buggy callers of this function.  Ideally we
4030          * should warn if prev_state != CONTEXT_USER, but that will trigger
4031          * too frequently to make sense yet.
4032          */
4033         enum ctx_state prev_state = exception_enter();
4034         schedule();
4035         exception_exit(prev_state);
4036 }
4037 #endif
4038 
4039 /**
4040  * schedule_preempt_disabled - called with preemption disabled
4041  *
4042  * Returns with preemption disabled. Note: preempt_count must be 1
4043  */
4044 void __sched schedule_preempt_disabled(void)
4045 {
4046         sched_preempt_enable_no_resched();
4047         schedule();
4048         preempt_disable();
4049 }
4050 
4051 static void __sched notrace preempt_schedule_common(void)
4052 {
4053         do {
4054                 /*
4055                  * Because the function tracer can trace preempt_count_sub()
4056                  * and it also uses preempt_enable/disable_notrace(), if
4057                  * NEED_RESCHED is set, the preempt_enable_notrace() called
4058                  * by the function tracer will call this function again and
4059                  * cause infinite recursion.
4060                  *
4061                  * Preemption must be disabled here before the function
4062                  * tracer can trace. Break up preempt_disable() into two
4063                  * calls. One to disable preemption without fear of being
4064                  * traced. The other to still record the preemption latency,
4065                  * which can also be traced by the function tracer.
4066                  */
4067                 preempt_disable_notrace();
4068                 preempt_latency_start(1);
4069                 __schedule(true);
4070                 preempt_latency_stop(1);
4071                 preempt_enable_no_resched_notrace();
4072 
4073                 /*
4074                  * Check again in case we missed a preemption opportunity
4075                  * between schedule and now.
4076                  */
4077         } while (need_resched());
4078 }
4079 
4080 #ifdef CONFIG_PREEMPT
4081 /*
4082  * this is the entry point to schedule() from in-kernel preemption
4083  * off of preempt_enable. Kernel preemptions off return from interrupt
4084  * occur there and call schedule directly.
4085  */
4086 asmlinkage __visible void __sched notrace preempt_schedule(void)
4087 {
4088         /*
4089          * If there is a non-zero preempt_count or interrupts are disabled,
4090          * we do not want to preempt the current task. Just return..
4091          */
4092         if (likely(!preemptible()))
4093                 return;
4094 
4095         preempt_schedule_common();
4096 }
4097 NOKPROBE_SYMBOL(preempt_schedule);
4098 EXPORT_SYMBOL(preempt_schedule);
4099 
4100 /**
4101  * preempt_schedule_notrace - preempt_schedule called by tracing
4102  *
4103  * The tracing infrastructure uses preempt_enable_notrace to prevent
4104  * recursion and tracing preempt enabling caused by the tracing
4105  * infrastructure itself. But as tracing can happen in areas coming
4106  * from userspace or just about to enter userspace, a preempt enable
4107  * can occur before user_exit() is called. This will cause the scheduler
4108  * to be called when the system is still in usermode.
4109  *
4110  * To prevent this, the preempt_enable_notrace will use this function
4111  * instead of preempt_schedule() to exit user context if needed before
4112  * calling the scheduler.
4113  */
4114 asmlinkage __visible void __sched notrace preempt_schedule_notrace(void)
4115 {
4116         enum ctx_state prev_ctx;
4117 
4118         if (likely(!preemptible()))
4119                 return;
4120 
4121         do {
4122                 /*
4123                  * Because the function tracer can trace preempt_count_sub()
4124                  * and it also uses preempt_enable/disable_notrace(), if
4125                  * NEED_RESCHED is set, the preempt_enable_notrace() called
4126                  * by the function tracer will call this function again and
4127                  * cause infinite recursion.
4128                  *
4129                  * Preemption must be disabled here before the function
4130                  * tracer can trace. Break up preempt_disable() into two
4131                  * calls. One to disable preemption without fear of being
4132                  * traced. The other to still record the preemption latency,
4133                  * which can also be traced by the function tracer.
4134                  */
4135                 preempt_disable_notrace();
4136                 preempt_latency_start(1);
4137                 /*
4138                  * Needs preempt disabled in case user_exit() is traced
4139                  * and the tracer calls preempt_enable_notrace() causing
4140                  * an infinite recursion.
4141                  */
4142                 prev_ctx = exception_enter();
4143                 __schedule(true);
4144                 exception_exit(prev_ctx);
4145 
4146                 preempt_latency_stop(1);
4147                 preempt_enable_no_resched_notrace();
4148         } while (need_resched());
4149 }
4150 EXPORT_SYMBOL_GPL(preempt_schedule_notrace);
4151 
4152 #endif /* CONFIG_PREEMPT */
4153 
4154 /*
4155  * this is the entry point to schedule() from kernel preemption
4156  * off of irq context.
4157  * Note, that this is called and return with irqs disabled. This will
4158  * protect us against recursive calling from irq.
4159  */
4160 asmlinkage __visible void __sched preempt_schedule_irq(void)
4161 {
4162         enum ctx_state prev_state;
4163 
4164         /* Catch callers which need to be fixed */
4165         BUG_ON(preempt_count() || !irqs_disabled());
4166 
4167         prev_state = exception_enter();
4168 
4169         do {
4170                 preempt_disable();
4171                 local_irq_enable();
4172                 __schedule(true);
4173                 local_irq_disable();
4174                 sched_preempt_enable_no_resched();
4175         } while (need_resched());
4176 
4177         exception_exit(prev_state);
4178 }
4179 
4180 int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags,
4181                           void *key)
4182 {
4183         return try_to_wake_up(curr->private, mode, wake_flags);
4184 }
4185 EXPORT_SYMBOL(default_wake_function);
4186 
4187 #ifdef CONFIG_RT_MUTEXES
4188 
4189 static inline int __rt_effective_prio(struct task_struct *pi_task, int prio)
4190 {
4191         if (pi_task)
4192                 prio = min(prio, pi_task->prio);
4193 
4194         return prio;
4195 }
4196 
4197 static inline int rt_effective_prio(struct task_struct *p, int prio)
4198 {
4199         struct task_struct *pi_task = rt_mutex_get_top_task(p);
4200 
4201         return __rt_effective_prio(pi_task, prio);
4202 }
4203 
4204 /*
4205  * rt_mutex_setprio - set the current priority of a task
4206  * @p: task to boost
4207  * @pi_task: donor task
4208  *
4209  * This function changes the 'effective' priority of a task. It does
4210  * not touch ->normal_prio like __setscheduler().
4211  *
4212  * Used by the rt_mutex code to implement priority inheritance
4213  * logic. Call site only calls if the priority of the task changed.
4214  */
4215 void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task)
4216 {
4217         int prio, oldprio, queued, running, queue_flag =
4218                 DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
4219         const struct sched_class *prev_class;
4220         struct rq_flags rf;
4221         struct rq *rq;
4222 
4223         /* XXX used to be waiter->prio, not waiter->task->prio */
4224         prio = __rt_effective_prio(pi_task, p->normal_prio);
4225 
4226         /*
4227          * If nothing changed; bail early.
4228          */
4229         if (p->pi_top_task == pi_task && prio == p->prio && !dl_prio(prio))
4230                 return;
4231 
4232         rq = __task_rq_lock(p, &rf);
4233         update_rq_clock(rq);
4234         /*
4235          * Set under pi_lock && rq->lock, such that the value can be used under
4236          * either lock.
4237          *
4238          * Note that there is loads of tricky to make this pointer cache work
4239          * right. rt_mutex_slowunlock()+rt_mutex_postunlock() work together to
4240          * ensure a task is de-boosted (pi_task is set to NULL) before the
4241          * task is allowed to run again (and can exit). This ensures the pointer
4242          * points to a blocked task -- which guaratees the task is present.
4243          */
4244         p->pi_top_task = pi_task;
4245 
4246         /*
4247          * For FIFO/RR we only need to set prio, if that matches we're done.
4248          */
4249         if (prio == p->prio && !dl_prio(prio))
4250                 goto out_unlock;
4251 
4252         /*
4253          * Idle task boosting is a nono in general. There is one
4254          * exception, when PREEMPT_RT and NOHZ is active:
4255          *
4256          * The idle task calls get_next_timer_interrupt() and holds
4257          * the timer wheel base->lock on the CPU and another CPU wants
4258          * to access the timer (probably to cancel it). We can safely
4259          * ignore the boosting request, as the idle CPU runs this code
4260          * with interrupts disabled and will complete the lock
4261          * protected section without being interrupted. So there is no
4262          * real need to boost.
4263          */
4264         if (unlikely(p == rq->idle)) {
4265                 WARN_ON(p != rq->curr);
4266                 WARN_ON(p->pi_blocked_on);
4267                 goto out_unlock;
4268         }
4269 
4270         trace_sched_pi_setprio(p, pi_task);
4271         oldprio = p->prio;
4272 
4273         if (oldprio == prio)
4274                 queue_flag &= ~DEQUEUE_MOVE;
4275 
4276         prev_class = p->sched_class;
4277         queued = task_on_rq_queued(p);
4278         running = task_current(rq, p);
4279         if (queued)
4280                 dequeue_task(rq, p, queue_flag);
4281         if (running)
4282                 put_prev_task(rq, p);
4283 
4284         /*
4285          * Boosting condition are:
4286          * 1. -rt task is running and holds mutex A
4287          *      --> -dl task blocks on mutex A
4288          *
4289          * 2. -dl task is running and holds mutex A
4290          *      --> -dl task blocks on mutex A and could preempt the
4291          *          running task
4292          */
4293         if (dl_prio(prio)) {
4294                 if (!dl_prio(p->normal_prio) ||
4295                     (pi_task && dl_entity_preempt(&pi_task->dl, &p->dl))) {
4296                         p->dl.dl_boosted = 1;
4297                         queue_flag |= ENQUEUE_REPLENISH;
4298                 } else
4299                         p->dl.dl_boosted = 0;
4300                 p->sched_class = &dl_sched_class;
4301         } else if (rt_prio(prio)) {
4302                 if (dl_prio(oldprio))
4303                         p->dl.dl_boosted = 0;
4304                 if (oldprio < prio)
4305                         queue_flag |= ENQUEUE_HEAD;
4306                 p->sched_class = &rt_sched_class;
4307         } else {
4308                 if (dl_prio(oldprio))
4309                         p->dl.dl_boosted = 0;
4310                 if (rt_prio(oldprio))
4311                         p->rt.timeout = 0;
4312                 p->sched_class = &fair_sched_class;
4313         }
4314 
4315         p->prio = prio;
4316 
4317         if (queued)
4318                 enqueue_task(rq, p, queue_flag);
4319         if (running)
4320                 set_curr_task(rq, p);
4321 
4322         check_class_changed(rq, p, prev_class, oldprio);
4323 out_unlock:
4324         /* Avoid rq from going away on us: */
4325         preempt_disable();
4326         __task_rq_unlock(rq, &rf);
4327 
4328         balance_callback(rq);
4329         preempt_enable();
4330 }
4331 #else
4332 static inline int rt_effective_prio(struct task_struct *p, int prio)
4333 {
4334         return prio;
4335 }
4336 #endif
4337 
4338 void set_user_nice(struct task_struct *p, long nice)
4339 {
4340         bool queued, running;
4341         int old_prio, delta;
4342         struct rq_flags rf;
4343         struct rq *rq;
4344 
4345         if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE)
4346                 return;
4347         /*
4348          * We have to be careful, if called from sys_setpriority(),
4349          * the task might be in the middle of scheduling on another CPU.
4350          */
4351         rq = task_rq_lock(p, &rf);
4352         update_rq_clock(rq);
4353 
4354         /*
4355          * The RT priorities are set via sched_setscheduler(), but we still
4356          * allow the 'normal' nice value to be set - but as expected
4357          * it wont have any effect on scheduling until the task is
4358          * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR:
4359          */
4360         if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
4361                 p->static_prio = NICE_TO_PRIO(nice);
4362                 goto out_unlock;
4363         }
4364         queued = task_on_rq_queued(p);
4365         running = task_current(rq, p);
4366         if (queued)
4367                 dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK);
4368         if (running)
4369                 put_prev_task(rq, p);
4370 
4371         p->static_prio = NICE_TO_PRIO(nice);
4372         set_load_weight(p, true);
4373         old_prio = p->prio;
4374         p->prio = effective_prio(p);
4375         delta = p->prio - old_prio;
4376 
4377         if (queued) {
4378                 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
4379                 /*
4380                  * If the task increased its priority or is running and
4381                  * lowered its priority, then reschedule its CPU:
4382                  */
4383                 if (delta < 0 || (delta > 0 && task_running(rq, p)))
4384                         resched_curr(rq);
4385         }
4386         if (running)
4387                 set_curr_task(rq, p);
4388 out_unlock:
4389         task_rq_unlock(rq, p, &rf);
4390 }
4391 EXPORT_SYMBOL(set_user_nice);
4392 
4393 /*
4394  * can_nice - check if a task can reduce its nice value
4395  * @p: task
4396  * @nice: nice value
4397  */
4398 int can_nice(const struct task_struct *p, const int nice)
4399 {
4400         /* Convert nice value [19,-20] to rlimit style value [1,40]: */
4401         int nice_rlim = nice_to_rlimit(nice);
4402 
4403         return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
4404                 capable(CAP_SYS_NICE));
4405 }
4406 
4407 #ifdef __ARCH_WANT_SYS_NICE
4408 
4409 /*
4410  * sys_nice - change the priority of the current process.
4411  * @increment: priority increment
4412  *
4413  * sys_setpriority is a more generic, but much slower function that
4414  * does similar things.
4415  */
4416 SYSCALL_DEFINE1(nice, int, increment)
4417 {
4418         long nice, retval;
4419         if (!ccs_capable(CCS_SYS_NICE))
4420                 return -EPERM;
4421 
4422         /*
4423          * Setpriority might change our priority at the same moment.
4424          * We don't have to worry. Conceptually one call occurs first
4425          * and we have a single winner.
4426          */
4427         increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH);
4428         nice = task_nice(current) + increment;
4429 
4430         nice = clamp_val(nice, MIN_NICE, MAX_NICE);
4431         if (increment < 0 && !can_nice(current, nice))
4432                 return -EPERM;
4433 
4434         retval = security_task_setnice(current, nice);
4435         if (retval)
4436                 return retval;
4437 
4438         set_user_nice(current, nice);
4439         return 0;
4440 }
4441 
4442 #endif
4443 
4444 /**
4445  * task_prio - return the priority value of a given task.
4446  * @p: the task in question.
4447  *
4448  * Return: The priority value as seen by users in /proc.
4449  * RT tasks are offset by -200. Normal tasks are centered
4450  * around 0, value goes from -16 to +15.
4451  */
4452 int task_prio(const struct task_struct *p)
4453 {
4454         return p->prio - MAX_RT_PRIO;
4455 }
4456 
4457 /**
4458  * idle_cpu - is a given CPU idle currently?
4459  * @cpu: the processor in question.
4460  *
4461  * Return: 1 if the CPU is currently idle. 0 otherwise.
4462  */
4463 int idle_cpu(int cpu)
4464 {
4465         struct rq *rq = cpu_rq(cpu);
4466 
4467         if (rq->curr != rq->idle)
4468                 return 0;
4469 
4470         if (rq->nr_running)
4471                 return 0;
4472 
4473 #ifdef CONFIG_SMP
4474         if (!llist_empty(&rq->wake_list))
4475                 return 0;
4476 #endif
4477 
4478         return 1;
4479 }
4480 
4481 /**
4482  * available_idle_cpu - is a given CPU idle for enqueuing work.
4483  * @cpu: the CPU in question.
4484  *
4485  * Return: 1 if the CPU is currently idle. 0 otherwise.
4486  */
4487 int available_idle_cpu(int cpu)
4488 {
4489         if (!idle_cpu(cpu))
4490                 return 0;
4491 
4492         if (vcpu_is_preempted(cpu))
4493                 return 0;
4494 
4495         return 1;
4496 }
4497 
4498 /**
4499  * idle_task - return the idle task for a given CPU.
4500  * @cpu: the processor in question.
4501  *
4502  * Return: The idle task for the CPU @cpu.
4503  */
4504 struct task_struct *idle_task(int cpu)
4505 {
4506         return cpu_rq(cpu)->idle;
4507 }
4508 
4509 /**
4510  * find_process_by_pid - find a process with a matching PID value.
4511  * @pid: the pid in question.
4512  *
4513  * The task of @pid, if found. %NULL otherwise.
4514  */
4515 static struct task_struct *find_process_by_pid(pid_t pid)
4516 {
4517         return pid ? find_task_by_vpid(pid) : current;
4518 }
4519 
4520 /*
4521  * sched_setparam() passes in -1 for its policy, to let the functions
4522  * it calls know not to change it.
4523  */
4524 #define SETPARAM_POLICY -1
4525 
4526 static void __setscheduler_params(struct task_struct *p,
4527                 const struct sched_attr *attr)
4528 {
4529         int policy = attr->sched_policy;
4530 
4531         if (policy == SETPARAM_POLICY)
4532                 policy = p->policy;
4533 
4534         p->policy = policy;
4535 
4536         if (dl_policy(policy))
4537                 __setparam_dl(p, attr);
4538         else if (fair_policy(policy))
4539                 p->static_prio = NICE_TO_PRIO(attr->sched_nice);
4540 
4541         /*
4542          * __sched_setscheduler() ensures attr->sched_priority == 0 when
4543          * !rt_policy. Always setting this ensures that things like
4544          * getparam()/getattr() don't report silly values for !rt tasks.
4545          */
4546         p->rt_priority = attr->sched_priority;
4547         p->normal_prio = normal_prio(p);
4548         set_load_weight(p, true);
4549 }
4550 
4551 /* Actually do priority change: must hold pi & rq lock. */
4552 static void __setscheduler(struct rq *rq, struct task_struct *p,
4553                            const struct sched_attr *attr, bool keep_boost)
4554 {
4555         /*
4556          * If params can't change scheduling class changes aren't allowed
4557          * either.
4558          */
4559         if (attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)
4560                 return;
4561 
4562         __setscheduler_params(p, attr);
4563 
4564         /*
4565          * Keep a potential priority boosting if called from
4566          * sched_setscheduler().
4567          */
4568         p->prio = normal_prio(p);
4569         if (keep_boost)
4570                 p->prio = rt_effective_prio(p, p->prio);
4571 
4572         if (dl_prio(p->prio))
4573                 p->sched_class = &dl_sched_class;
4574         else if (rt_prio(p->prio))
4575                 p->sched_class = &rt_sched_class;
4576         else
4577                 p->sched_class = &fair_sched_class;
4578 }
4579 
4580 /*
4581  * Check the target process has a UID that matches the current process's:
4582  */
4583 static bool check_same_owner(struct task_struct *p)
4584 {
4585         const struct cred *cred = current_cred(), *pcred;
4586         bool match;
4587 
4588         rcu_read_lock();
4589         pcred = __task_cred(p);
4590         match = (uid_eq(cred->euid, pcred->euid) ||
4591                  uid_eq(cred->euid, pcred->uid));
4592         rcu_read_unlock();
4593         return match;
4594 }
4595 
4596 static int __sched_setscheduler(struct task_struct *p,
4597                                 const struct sched_attr *attr,
4598                                 bool user, bool pi)
4599 {
4600         int newprio = dl_policy(attr->sched_policy) ? MAX_DL_PRIO - 1 :
4601                       MAX_RT_PRIO - 1 - attr->sched_priority;
4602         int retval, oldprio, oldpolicy = -1, queued, running;
4603         int new_effective_prio, policy = attr->sched_policy;
4604         const struct sched_class *prev_class;
4605         struct rq_flags rf;
4606         int reset_on_fork;
4607         int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
4608         struct rq *rq;
4609 
4610         /* The pi code expects interrupts enabled */
4611         BUG_ON(pi && in_interrupt());
4612 recheck:
4613         /* Double check policy once rq lock held: */
4614         if (policy < 0) {
4615                 reset_on_fork = p->sched_reset_on_fork;
4616                 policy = oldpolicy = p->policy;
4617         } else {
4618                 reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK);
4619 
4620                 if (!valid_policy(policy))
4621                         return -EINVAL;
4622         }
4623 
4624         if (attr->sched_flags & ~(SCHED_FLAG_ALL | SCHED_FLAG_SUGOV))
4625                 return -EINVAL;
4626 
4627         /*
4628          * Valid priorities for SCHED_FIFO and SCHED_RR are
4629          * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
4630          * SCHED_BATCH and SCHED_IDLE is 0.
4631          */
4632         if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) ||
4633             (!p->mm && attr->sched_priority > MAX_RT_PRIO-1))
4634                 return -EINVAL;
4635         if ((dl_policy(policy) && !__checkparam_dl(attr)) ||
4636             (rt_policy(policy) != (attr->sched_priority != 0)))
4637                 return -EINVAL;
4638 
4639         /*
4640          * Allow unprivileged RT tasks to decrease priority:
4641          */
4642         if (user && !capable(CAP_SYS_NICE)) {
4643                 if (fair_policy(policy)) {
4644                         if (attr->sched_nice < task_nice(p) &&
4645                             !can_nice(p, attr->sched_nice))
4646                                 return -EPERM;
4647                 }
4648 
4649                 if (rt_policy(policy)) {
4650                         unsigned long rlim_rtprio =
4651                                         task_rlimit(p, RLIMIT_RTPRIO);
4652 
4653                         /* Can't set/change the rt policy: */
4654                         if (policy != p->policy && !rlim_rtprio)
4655                                 return -EPERM;
4656 
4657                         /* Can't increase priority: */
4658                         if (attr->sched_priority > p->rt_priority &&
4659                             attr->sched_priority > rlim_rtprio)
4660                                 return -EPERM;
4661                 }
4662 
4663                  /*
4664                   * Can't set/change SCHED_DEADLINE policy at all for now
4665                   * (safest behavior); in the future we would like to allow
4666                   * unprivileged DL tasks to increase their relative deadline
4667                   * or reduce their runtime (both ways reducing utilization)
4668                   */
4669                 if (dl_policy(policy))
4670                         return -EPERM;
4671 
4672                 /*
4673                  * Treat SCHED_IDLE as nice 20. Only allow a switch to
4674                  * SCHED_NORMAL if the RLIMIT_NICE would normally permit it.
4675                  */
4676                 if (task_has_idle_policy(p) && !idle_policy(policy)) {
4677                         if (!can_nice(p, task_nice(p)))
4678                                 return -EPERM;
4679                 }
4680 
4681                 /* Can't change other user's priorities: */
4682                 if (!check_same_owner(p))
4683                         return -EPERM;
4684 
4685                 /* Normal users shall not reset the sched_reset_on_fork flag: */
4686                 if (p->sched_reset_on_fork && !reset_on_fork)
4687                         return -EPERM;
4688         }
4689 
4690         if (user) {
4691                 if (attr->sched_flags & SCHED_FLAG_SUGOV)
4692                         return -EINVAL;
4693 
4694                 retval = security_task_setscheduler(p);
4695                 if (retval)
4696                         return retval;
4697         }
4698 
4699         /* Update task specific "requested" clamps */
4700         if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) {
4701                 retval = uclamp_validate(p, attr);
4702                 if (retval)
4703                         return retval;
4704         }
4705 
4706         /*
4707          * Make sure no PI-waiters arrive (or leave) while we are
4708          * changing the priority of the task:
4709          *
4710          * To be able to change p->policy safely, the appropriate
4711          * runqueue lock must be held.
4712          */
4713         rq = task_rq_lock(p, &rf);
4714         update_rq_clock(rq);
4715 
4716         /*
4717          * Changing the policy of the stop threads its a very bad idea:
4718          */
4719         if (p == rq->stop) {
4720                 task_rq_unlock(rq, p, &rf);
4721                 return -EINVAL;
4722         }
4723 
4724         /*
4725          * If not changing anything there's no need to proceed further,
4726          * but store a possible modification of reset_on_fork.
4727          */
4728         if (unlikely(policy == p->policy)) {
4729                 if (fair_policy(policy) && attr->sched_nice != task_nice(p))
4730                         goto change;
4731                 if (rt_policy(policy) && attr->sched_priority != p->rt_priority)
4732                         goto change;
4733                 if (dl_policy(policy) && dl_param_changed(p, attr))
4734                         goto change;
4735                 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP)
4736                         goto change;
4737 
4738                 p->sched_reset_on_fork = reset_on_fork;
4739                 task_rq_unlock(rq, p, &rf);
4740                 return 0;
4741         }
4742 change:
4743 
4744         if (user) {
4745 #ifdef CONFIG_RT_GROUP_SCHED
4746                 /*
4747                  * Do not allow realtime tasks into groups that have no runtime
4748                  * assigned.
4749                  */
4750                 if (rt_bandwidth_enabled() && rt_policy(policy) &&
4751                                 task_group(p)->rt_bandwidth.rt_runtime == 0 &&
4752                                 !task_group_is_autogroup(task_group(p))) {
4753                         task_rq_unlock(rq, p, &rf);
4754                         return -EPERM;
4755                 }
4756 #endif
4757 #ifdef CONFIG_SMP
4758                 if (dl_bandwidth_enabled() && dl_policy(policy) &&
4759                                 !(attr->sched_flags & SCHED_FLAG_SUGOV)) {
4760                         cpumask_t *span = rq->rd->span;
4761 
4762                         /*
4763                          * Don't allow tasks with an affinity mask smaller than
4764                          * the entire root_domain to become SCHED_DEADLINE. We
4765                          * will also fail if there's no bandwidth available.
4766                          */
4767                         if (!cpumask_subset(span, p->cpus_ptr) ||
4768                             rq->rd->dl_bw.bw == 0) {
4769                                 task_rq_unlock(rq, p, &rf);
4770                                 return -EPERM;
4771                         }
4772                 }
4773 #endif
4774         }
4775 
4776         /* Re-check policy now with rq lock held: */
4777         if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
4778                 policy = oldpolicy = -1;
4779                 task_rq_unlock(rq, p, &rf);
4780                 goto recheck;
4781         }
4782 
4783         /*
4784          * If setscheduling to SCHED_DEADLINE (or changing the parameters
4785          * of a SCHED_DEADLINE task) we need to check if enough bandwidth
4786          * is available.
4787          */
4788         if ((dl_policy(policy) || dl_task(p)) && sched_dl_overflow(p, policy, attr)) {
4789                 task_rq_unlock(rq, p, &rf);
4790                 return -EBUSY;
4791         }
4792 
4793         p->sched_reset_on_fork = reset_on_fork;
4794         oldprio = p->prio;
4795 
4796         if (pi) {
4797                 /*
4798                  * Take priority boosted tasks into account. If the new
4799                  * effective priority is unchanged, we just store the new
4800                  * normal parameters and do not touch the scheduler class and
4801                  * the runqueue. This will be done when the task deboost
4802                  * itself.
4803                  */
4804                 new_effective_prio = rt_effective_prio(p, newprio);
4805                 if (new_effective_prio == oldprio)
4806                         queue_flags &= ~DEQUEUE_MOVE;
4807         }
4808 
4809         queued = task_on_rq_queued(p);
4810         running = task_current(rq, p);
4811         if (queued)
4812                 dequeue_task(rq, p, queue_flags);
4813         if (running)
4814                 put_prev_task(rq, p);
4815 
4816         prev_class = p->sched_class;
4817 
4818         __setscheduler(rq, p, attr, pi);
4819         __setscheduler_uclamp(p, attr);
4820 
4821         if (queued) {
4822                 /*
4823                  * We enqueue to tail when the priority of a task is
4824                  * increased (user space view).
4825                  */
4826                 if (oldprio < p->prio)
4827                         queue_flags |= ENQUEUE_HEAD;
4828 
4829                 enqueue_task(rq, p, queue_flags);
4830         }
4831         if (running)
4832                 set_curr_task(rq, p);
4833 
4834         check_class_changed(rq, p, prev_class, oldprio);
4835 
4836         /* Avoid rq from going away on us: */
4837         preempt_disable();
4838         task_rq_unlock(rq, p, &rf);
4839 
4840         if (pi)
4841                 rt_mutex_adjust_pi(p);
4842 
4843         /* Run balance callbacks after we've adjusted the PI chain: */
4844         balance_callback(rq);
4845         preempt_enable();
4846 
4847         return 0;
4848 }
4849 
4850 static int _sched_setscheduler(struct task_struct *p, int policy,
4851                                const struct sched_param *param, bool check)
4852 {
4853         struct sched_attr attr = {
4854                 .sched_policy   = policy,
4855                 .sched_priority = param->sched_priority,
4856                 .sched_nice     = PRIO_TO_NICE(p->static_prio),
4857         };
4858 
4859         /* Fixup the legacy SCHED_RESET_ON_FORK hack. */
4860         if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) {
4861                 attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
4862                 policy &= ~SCHED_RESET_ON_FORK;
4863                 attr.sched_policy = policy;
4864         }
4865 
4866         return __sched_setscheduler(p, &attr, check, true);
4867 }
4868 /**
4869  * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
4870  * @p: the task in question.
4871  * @policy: new policy.
4872  * @param: structure containing the new RT priority.
4873  *
4874  * Return: 0 on success. An error code otherwise.
4875  *
4876  * NOTE that the task may be already dead.
4877  */
4878 int sched_setscheduler(struct task_struct *p, int policy,
4879                        const struct sched_param *param)
4880 {
4881         return _sched_setscheduler(p, policy, param, true);
4882 }
4883 EXPORT_SYMBOL_GPL(sched_setscheduler);
4884 
4885 int sched_setattr(struct task_struct *p, const struct sched_attr *attr)
4886 {
4887         return __sched_setscheduler(p, attr, true, true);
4888 }
4889 EXPORT_SYMBOL_GPL(sched_setattr);
4890 
4891 int sched_setattr_nocheck(struct task_struct *p, const struct sched_attr *attr)
4892 {
4893         return __sched_setscheduler(p, attr, false, true);
4894 }
4895 
4896 /**
4897  * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
4898  * @p: the task in question.
4899  * @policy: new policy.
4900  * @param: structure containing the new RT priority.
4901  *
4902  * Just like sched_setscheduler, only don't bother checking if the
4903  * current context has permission.  For example, this is needed in
4904  * stop_machine(): we create temporary high priority worker threads,
4905  * but our caller might not have that capability.
4906  *
4907  * Return: 0 on success. An error code otherwise.
4908  */
4909 int sched_setscheduler_nocheck(struct task_struct *p, int policy,
4910                                const struct sched_param *param)
4911 {
4912         return _sched_setscheduler(p, policy, param, false);
4913 }
4914 EXPORT_SYMBOL_GPL(sched_setscheduler_nocheck);
4915 
4916 static int
4917 do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
4918 {
4919         struct sched_param lparam;
4920         struct task_struct *p;
4921         int retval;
4922 
4923         if (!param || pid < 0)
4924                 return -EINVAL;
4925         if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
4926                 return -EFAULT;
4927 
4928         rcu_read_lock();
4929         retval = -ESRCH;
4930         p = find_process_by_pid(pid);
4931         if (p != NULL)
4932                 retval = sched_setscheduler(p, policy, &lparam);
4933         rcu_read_unlock();
4934 
4935         return retval;
4936 }
4937 
4938 /*
4939  * Mimics kernel/events/core.c perf_copy_attr().
4940  */
4941 static int sched_copy_attr(struct sched_attr __user *uattr, struct sched_attr *attr)
4942 {
4943         u32 size;
4944         int ret;
4945 
4946         if (!access_ok(uattr, SCHED_ATTR_SIZE_VER0))
4947                 return -EFAULT;
4948 
4949         /* Zero the full structure, so that a short copy will be nice: */
4950         memset(attr, 0, sizeof(*attr));
4951 
4952         ret = get_user(size, &uattr->size);
4953         if (ret)
4954                 return ret;
4955 
4956         /* Bail out on silly large: */
4957         if (size > PAGE_SIZE)
4958                 goto err_size;
4959 
4960         /* ABI compatibility quirk: */
4961         if (!size)
4962                 size = SCHED_ATTR_SIZE_VER0;
4963 
4964         if (size < SCHED_ATTR_SIZE_VER0)
4965                 goto err_size;
4966 
4967         /*
4968          * If we're handed a bigger struct than we know of,
4969          * ensure all the unknown bits are 0 - i.e. new
4970          * user-space does not rely on any kernel feature
4971          * extensions we dont know about yet.
4972          */
4973         if (size > sizeof(*attr)) {
4974                 unsigned char __user *addr;
4975                 unsigned char __user *end;
4976                 unsigned char val;
4977 
4978                 addr = (void __user *)uattr + sizeof(*attr);
4979                 end  = (void __user *)uattr + size;
4980 
4981                 for (; addr < end; addr++) {
4982                         ret = get_user(val, addr);
4983                         if (ret)
4984                                 return ret;
4985                         if (val)
4986                                 goto err_size;
4987                 }
4988                 size = sizeof(*attr);
4989         }
4990 
4991         ret = copy_from_user(attr, uattr, size);
4992         if (ret)
4993                 return -EFAULT;
4994 
4995         if ((attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) &&
4996             size < SCHED_ATTR_SIZE_VER1)
4997                 return -EINVAL;
4998 
4999         /*
5000          * XXX: Do we want to be lenient like existing syscalls; or do we want
5001          * to be strict and return an error on out-of-bounds values?
5002          */
5003         attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE);
5004 
5005         return 0;
5006 
5007 err_size:
5008         put_user(sizeof(*attr), &uattr->size);
5009         return -E2BIG;
5010 }
5011 
5012 /**
5013  * sys_sched_setscheduler - set/change the scheduler policy and RT priority
5014  * @pid: the pid in question.
5015  * @policy: new policy.
5016  * @param: structure containing the new RT priority.
5017  *
5018  * Return: 0 on success. An error code otherwise.
5019  */
5020 SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy, struct sched_param __user *, param)
5021 {
5022         if (policy < 0)
5023                 return -EINVAL;
5024 
5025         return do_sched_setscheduler(pid, policy, param);
5026 }
5027 
5028 /**
5029  * sys_sched_setparam - set/change the RT priority of a thread
5030  * @pid: the pid in question.
5031  * @param: structure containing the new RT priority.
5032  *
5033  * Return: 0 on success. An error code otherwise.
5034  */
5035 SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
5036 {
5037         return do_sched_setscheduler(pid, SETPARAM_POLICY, param);
5038 }
5039 
5040 /**
5041  * sys_sched_setattr - same as above, but with extended sched_attr
5042  * @pid: the pid in question.
5043  * @uattr: structure containing the extended parameters.
5044  * @flags: for future extension.
5045  */
5046 SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr,
5047                                unsigned int, flags)
5048 {
5049         struct sched_attr attr;
5050         struct task_struct *p;
5051         int retval;
5052 
5053         if (!uattr || pid < 0 || flags)
5054                 return -EINVAL;
5055 
5056         retval = sched_copy_attr(uattr, &attr);
5057         if (retval)
5058                 return retval;
5059 
5060         if ((int)attr.sched_policy < 0)
5061                 return -EINVAL;
5062         if (attr.sched_flags & SCHED_FLAG_KEEP_POLICY)
5063                 attr.sched_policy = SETPARAM_POLICY;
5064 
5065         rcu_read_lock();
5066         retval = -ESRCH;
5067         p = find_process_by_pid(pid);
5068         if (likely(p))
5069                 get_task_struct(p);
5070         rcu_read_unlock();
5071 
5072         if (likely(p)) {
5073                 retval = sched_setattr(p, &attr);
5074                 put_task_struct(p);
5075         }
5076 
5077         return retval;
5078 }
5079 
5080 /**
5081  * sys_sched_getscheduler - get the policy (scheduling class) of a thread
5082  * @pid: the pid in question.
5083  *
5084  * Return: On success, the policy of the thread. Otherwise, a negative error
5085  * code.
5086  */
5087 SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
5088 {
5089         struct task_struct *p;
5090         int retval;
5091 
5092         if (pid < 0)
5093                 return -EINVAL;
5094 
5095         retval = -ESRCH;
5096         rcu_read_lock();
5097         p = find_process_by_pid(pid);
5098         if (p) {
5099                 retval = security_task_getscheduler(p);
5100                 if (!retval)
5101                         retval = p->policy
5102                                 | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
5103         }
5104         rcu_read_unlock();
5105         return retval;
5106 }
5107 
5108 /**
5109  * sys_sched_getparam - get the RT priority of a thread
5110  * @pid: the pid in question.
5111  * @param: structure containing the RT priority.
5112  *
5113  * Return: On success, 0 and the RT priority is in @param. Otherwise, an error
5114  * code.
5115  */
5116 SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
5117 {
5118         struct sched_param lp = { .sched_priority = 0 };
5119         struct task_struct *p;
5120         int retval;
5121 
5122         if (!param || pid < 0)
5123                 return -EINVAL;
5124 
5125         rcu_read_lock();
5126         p = find_process_by_pid(pid);
5127         retval = -ESRCH;
5128         if (!p)
5129                 goto out_unlock;
5130 
5131         retval = security_task_getscheduler(p);
5132         if (retval)
5133                 goto out_unlock;
5134 
5135         if (task_has_rt_policy(p))
5136                 lp.sched_priority = p->rt_priority;
5137         rcu_read_unlock();
5138 
5139         /*
5140          * This one might sleep, we cannot do it with a spinlock held ...
5141          */
5142         retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;
5143 
5144         return retval;
5145 
5146 out_unlock:
5147         rcu_read_unlock();
5148         return retval;
5149 }
5150 
5151 /*
5152  * Copy the kernel size attribute structure (which might be larger
5153  * than what user-space knows about) to user-space.
5154  *
5155  * Note that all cases are valid: user-space buffer can be larger or
5156  * smaller than the kernel-space buffer. The usual case is that both
5157  * have the same size.
5158  */
5159 static int
5160 sched_attr_copy_to_user(struct sched_attr __user *uattr,
5161                         struct sched_attr *kattr,
5162                         unsigned int usize)
5163 {
5164         unsigned int ksize = sizeof(*kattr);
5165 
5166         if (!access_ok(uattr, usize))
5167                 return -EFAULT;
5168 
5169         /*
5170          * sched_getattr() ABI forwards and backwards compatibility:
5171          *
5172          * If usize == ksize then we just copy everything to user-space and all is good.
5173          *
5174          * If usize < ksize then we only copy as much as user-space has space for,
5175          * this keeps ABI compatibility as well. We skip the rest.
5176          *
5177          * If usize > ksize then user-space is using a newer version of the ABI,
5178          * which part the kernel doesn't know about. Just ignore it - tooling can
5179          * detect the kernel's knowledge of attributes from the attr->size value
5180          * which is set to ksize in this case.
5181          */
5182         kattr->size = min(usize, ksize);
5183 
5184         if (copy_to_user(uattr, kattr, kattr->size))
5185                 return -EFAULT;
5186 
5187         return 0;
5188 }
5189 
5190 /**
5191  * sys_sched_getattr - similar to sched_getparam, but with sched_attr
5192  * @pid: the pid in question.
5193  * @uattr: structure containing the extended parameters.
5194  * @usize: sizeof(attr) that user-space knows about, for forwards and backwards compatibility.
5195  * @flags: for future extension.
5196  */
5197 SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
5198                 unsigned int, usize, unsigned int, flags)
5199 {
5200         struct sched_attr kattr = { };
5201         struct task_struct *p;
5202         int retval;
5203 
5204         if (!uattr || pid < 0 || usize > PAGE_SIZE ||
5205             usize < SCHED_ATTR_SIZE_VER0 || flags)
5206                 return -EINVAL;
5207 
5208         rcu_read_lock();
5209         p = find_process_by_pid(pid);
5210         retval = -ESRCH;
5211         if (!p)
5212                 goto out_unlock;
5213 
5214         retval = security_task_getscheduler(p);
5215         if (retval)
5216                 goto out_unlock;
5217 
5218         kattr.sched_policy = p->policy;
5219         if (p->sched_reset_on_fork)
5220                 kattr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
5221         if (task_has_dl_policy(p))
5222                 __getparam_dl(p, &kattr);
5223         else if (task_has_rt_policy(p))
5224                 kattr.sched_priority = p->rt_priority;
5225         else
5226                 kattr.sched_nice = task_nice(p);
5227 
5228 #ifdef CONFIG_UCLAMP_TASK
5229         kattr.sched_util_min = p->uclamp_req[UCLAMP_MIN].value;
5230         kattr.sched_util_max = p->uclamp_req[UCLAMP_MAX].value;
5231 #endif
5232 
5233         rcu_read_unlock();
5234 
5235         return sched_attr_copy_to_user(uattr, &kattr, usize);
5236 
5237 out_unlock:
5238         rcu_read_unlock();
5239         return retval;
5240 }
5241 
5242 long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
5243 {
5244         cpumask_var_t cpus_allowed, new_mask;
5245         struct task_struct *p;
5246         int retval;
5247 
5248         rcu_read_lock();
5249 
5250         p = find_process_by_pid(pid);
5251         if (!p) {
5252                 rcu_read_unlock();
5253                 return -ESRCH;
5254         }
5255 
5256         /* Prevent p going away */
5257         get_task_struct(p);
5258         rcu_read_unlock();
5259 
5260         if (p->flags & PF_NO_SETAFFINITY) {
5261                 retval = -EINVAL;
5262                 goto out_put_task;
5263         }
5264         if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
5265                 retval = -ENOMEM;
5266                 goto out_put_task;
5267         }
5268         if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
5269                 retval = -ENOMEM;
5270                 goto out_free_cpus_allowed;
5271         }
5272         retval = -EPERM;
5273         if (!check_same_owner(p)) {
5274                 rcu_read_lock();
5275                 if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) {
5276                         rcu_read_unlock();
5277                         goto out_free_new_mask;
5278                 }
5279                 rcu_read_unlock();
5280         }
5281 
5282         retval = security_task_setscheduler(p);
5283         if (retval)
5284                 goto out_free_new_mask;
5285 
5286 
5287         cpuset_cpus_allowed(p, cpus_allowed);
5288         cpumask_and(new_mask, in_mask, cpus_allowed);
5289 
5290         /*
5291          * Since bandwidth control happens on root_domain basis,
5292          * if admission test is enabled, we only admit -deadline
5293          * tasks allowed to run on all the CPUs in the task's
5294          * root_domain.
5295          */
5296 #ifdef CONFIG_SMP
5297         if (task_has_dl_policy(p) && dl_bandwidth_enabled()) {
5298                 rcu_read_lock();
5299                 if (!cpumask_subset(task_rq(p)->rd->span, new_mask)) {
5300                         retval = -EBUSY;
5301                         rcu_read_unlock();
5302                         goto out_free_new_mask;
5303                 }
5304                 rcu_read_unlock();
5305         }
5306 #endif
5307 again:
5308         retval = __set_cpus_allowed_ptr(p, new_mask, true);
5309 
5310         if (!retval) {
5311                 cpuset_cpus_allowed(p, cpus_allowed);
5312                 if (!cpumask_subset(new_mask, cpus_allowed)) {
5313                         /*
5314                          * We must have raced with a concurrent cpuset
5315                          * update. Just reset the cpus_allowed to the
5316                          * cpuset's cpus_allowed
5317                          */
5318                         cpumask_copy(new_mask, cpus_allowed);
5319                         goto again;
5320                 }
5321         }
5322 out_free_new_mask:
5323         free_cpumask_var(new_mask);
5324 out_free_cpus_allowed:
5325         free_cpumask_var(cpus_allowed);
5326 out_put_task:
5327         put_task_struct(p);
5328         return retval;
5329 }
5330 
5331 static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
5332                              struct cpumask *new_mask)
5333 {
5334         if (len < cpumask_size())
5335                 cpumask_clear(new_mask);
5336         else if (len > cpumask_size())
5337                 len = cpumask_size();
5338 
5339         return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
5340 }
5341 
5342 /**
5343  * sys_sched_setaffinity - set the CPU affinity of a process
5344  * @pid: pid of the process
5345  * @len: length in bytes of the bitmask pointed to by user_mask_ptr
5346  * @user_mask_ptr: user-space pointer to the new CPU mask
5347  *
5348  * Return: 0 on success. An error code otherwise.
5349  */
5350 SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
5351                 unsigned long __user *, user_mask_ptr)
5352 {
5353         cpumask_var_t new_mask;
5354         int retval;
5355 
5356         if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
5357                 return -ENOMEM;
5358 
5359         retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
5360         if (retval == 0)
5361                 retval = sched_setaffinity(pid, new_mask);
5362         free_cpumask_var(new_mask);
5363         return retval;
5364 }
5365 
5366 long sched_getaffinity(pid_t pid, struct cpumask *mask)
5367 {
5368         struct task_struct *p;
5369         unsigned long flags;
5370         int retval;
5371 
5372         rcu_read_lock();
5373 
5374         retval = -ESRCH;
5375         p = find_process_by_pid(pid);
5376         if (!p)
5377                 goto out_unlock;
5378 
5379         retval = security_task_getscheduler(p);
5380         if (retval)
5381                 goto out_unlock;
5382 
5383         raw_spin_lock_irqsave(&p->pi_lock, flags);
5384         cpumask_and(mask, &p->cpus_mask, cpu_active_mask);
5385         raw_spin_unlock_irqrestore(&p->pi_lock, flags);
5386 
5387 out_unlock:
5388         rcu_read_unlock();
5389 
5390         return retval;
5391 }
5392 
5393 /**
5394  * sys_sched_getaffinity - get the CPU affinity of a process
5395  * @pid: pid of the process
5396  * @len: length in bytes of the bitmask pointed to by user_mask_ptr
5397  * @user_mask_ptr: user-space pointer to hold the current CPU mask
5398  *
5399  * Return: size of CPU mask copied to user_mask_ptr on success. An
5400  * error code otherwise.
5401  */
5402 SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
5403                 unsigned long __user *, user_mask_ptr)
5404 {
5405         int ret;
5406         cpumask_var_t mask;
5407 
5408         if ((len * BITS_PER_BYTE) < nr_cpu_ids)
5409                 return -EINVAL;
5410         if (len & (sizeof(unsigned long)-1))
5411                 return -EINVAL;
5412 
5413         if (!alloc_cpumask_var(&mask, GFP_KERNEL))
5414                 return -ENOMEM;
5415 
5416         ret = sched_getaffinity(pid, mask);
5417         if (ret == 0) {
5418                 unsigned int retlen = min(len, cpumask_size());
5419 
5420                 if (copy_to_user(user_mask_ptr, mask, retlen))
5421                         ret = -EFAULT;
5422                 else
5423                         ret = retlen;
5424         }
5425         free_cpumask_var(mask);
5426 
5427         return ret;
5428 }
5429 
5430 /**
5431  * sys_sched_yield - yield the current processor to other threads.
5432  *
5433  * This function yields the current CPU to other tasks. If there are no
5434  * other threads running on this CPU then this function will return.
5435  *
5436  * Return: 0.
5437  */
5438 static void do_sched_yield(void)
5439 {
5440         struct rq_flags rf;
5441         struct rq *rq;
5442 
5443         rq = this_rq_lock_irq(&rf);
5444 
5445         schedstat_inc(rq->yld_count);
5446         current->sched_class->yield_task(rq);
5447 
5448         /*
5449          * Since we are going to call schedule() anyway, there's
5450          * no need to preempt or enable interrupts:
5451          */
5452         preempt_disable();
5453         rq_unlock(rq, &rf);
5454         sched_preempt_enable_no_resched();
5455 
5456         schedule();
5457 }
5458 
5459 SYSCALL_DEFINE0(sched_yield)
5460 {
5461         do_sched_yield();
5462         return 0;
5463 }
5464 
5465 #ifndef CONFIG_PREEMPT
5466 int __sched _cond_resched(void)
5467 {
5468         if (should_resched(0)) {
5469                 preempt_schedule_common();
5470                 return 1;
5471         }
5472         rcu_all_qs();
5473         return 0;
5474 }
5475 EXPORT_SYMBOL(_cond_resched);
5476 #endif
5477 
5478 /*
5479  * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
5480  * call schedule, and on return reacquire the lock.
5481  *
5482  * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
5483  * operations here to prevent schedule() from being called twice (once via
5484  * spin_unlock(), once by hand).
5485  */
5486 int __cond_resched_lock(spinlock_t *lock)
5487 {
5488         int resched = should_resched(PREEMPT_LOCK_OFFSET);
5489         int ret = 0;
5490 
5491         lockdep_assert_held(lock);
5492 
5493         if (spin_needbreak(lock) || resched) {
5494                 spin_unlock(lock);
5495                 if (resched)
5496                         preempt_schedule_common();
5497                 else
5498                         cpu_relax();
5499                 ret = 1;
5500                 spin_lock(lock);
5501         }
5502         return ret;
5503 }
5504 EXPORT_SYMBOL(__cond_resched_lock);
5505 
5506 /**
5507  * yield - yield the current processor to other threads.
5508  *
5509  * Do not ever use this function, there's a 99% chance you're doing it wrong.
5510  *
5511  * The scheduler is at all times free to pick the calling task as the most
5512  * eligible task to run, if removing the yield() call from your code breaks
5513  * it, its already broken.
5514  *
5515  * Typical broken usage is:
5516  *
5517  * while (!event)
5518  *      yield();
5519  *
5520  * where one assumes that yield() will let 'the other' process run that will
5521  * make event true. If the current task is a SCHED_FIFO task that will never
5522  * happen. Never use yield() as a progress guarantee!!
5523  *
5524  * If you want to use yield() to wait for something, use wait_event().
5525  * If you want to use yield() to be 'nice' for others, use cond_resched().
5526  * If you still want to use yield(), do not!
5527  */
5528 void __sched yield(void)
5529 {
5530         set_current_state(TASK_RUNNING);
5531         do_sched_yield();
5532 }
5533 EXPORT_SYMBOL(yield);
5534 
5535 /**
5536  * yield_to - yield the current processor to another thread in
5537  * your thread group, or accelerate that thread toward the
5538  * processor it's on.
5539  * @p: target task
5540  * @preempt: whether task preemption is allowed or not
5541  *
5542  * It's the caller's job to ensure that the target task struct
5543  * can't go away on us before we can do any checks.
5544  *
5545  * Return:
5546  *      true (>0) if we indeed boosted the target task.
5547  *      false (0) if we failed to boost the target.
5548  *      -ESRCH if there's no task to yield to.
5549  */
5550 int __sched yield_to(struct task_struct *p, bool preempt)
5551 {
5552         struct task_struct *curr = current;
5553         struct rq *rq, *p_rq;
5554         unsigned long flags;
5555         int yielded = 0;
5556 
5557         local_irq_save(flags);
5558         rq = this_rq();
5559 
5560 again:
5561         p_rq = task_rq(p);
5562         /*
5563          * If we're the only runnable task on the rq and target rq also
5564          * has only one task, there's absolutely no point in yielding.
5565          */
5566         if (rq->nr_running == 1 && p_rq->nr_running == 1) {
5567                 yielded = -ESRCH;
5568                 goto out_irq;
5569         }
5570 
5571         double_rq_lock(rq, p_rq);
5572         if (task_rq(p) != p_rq) {
5573                 double_rq_unlock(rq, p_rq);
5574                 goto again;
5575         }
5576 
5577         if (!curr->sched_class->yield_to_task)
5578                 goto out_unlock;
5579 
5580         if (curr->sched_class != p->sched_class)
5581                 goto out_unlock;
5582 
5583         if (task_running(p_rq, p) || p->state)
5584                 goto out_unlock;
5585 
5586         yielded = curr->sched_class->yield_to_task(rq, p, preempt);
5587         if (yielded) {
5588                 schedstat_inc(rq->yld_count);
5589                 /*
5590                  * Make p's CPU reschedule; pick_next_entity takes care of
5591                  * fairness.
5592                  */
5593                 if (preempt && rq != p_rq)
5594                         resched_curr(p_rq);
5595         }
5596 
5597 out_unlock:
5598         double_rq_unlock(rq, p_rq);
5599 out_irq:
5600         local_irq_restore(flags);
5601 
5602         if (yielded > 0)
5603                 schedule();
5604 
5605         return yielded;
5606 }
5607 EXPORT_SYMBOL_GPL(yield_to);
5608 
5609 int io_schedule_prepare(void)
5610 {
5611         int old_iowait = current->in_iowait;
5612 
5613         current->in_iowait = 1;
5614         blk_schedule_flush_plug(current);
5615 
5616         return old_iowait;
5617 }
5618 
5619 void io_schedule_finish(int token)
5620 {
5621         current->in_iowait = token;
5622 }
5623 
5624 /*
5625  * This task is about to go to sleep on IO. Increment rq->nr_iowait so
5626  * that process accounting knows that this is a task in IO wait state.
5627  */
5628 long __sched io_schedule_timeout(long timeout)
5629 {
5630         int token;
5631         long ret;
5632 
5633         token = io_schedule_prepare();
5634         ret = schedule_timeout(timeout);
5635         io_schedule_finish(token);
5636 
5637         return ret;
5638 }
5639 EXPORT_SYMBOL(io_schedule_timeout);
5640 
5641 void __sched io_schedule(void)
5642 {
5643         int token;
5644 
5645         token = io_schedule_prepare();
5646         schedule();
5647         io_schedule_finish(token);
5648 }
5649 EXPORT_SYMBOL(io_schedule);
5650 
5651 /**
5652  * sys_sched_get_priority_max - return maximum RT priority.
5653  * @policy: scheduling class.
5654  *
5655  * Return: On success, this syscall returns the maximum
5656  * rt_priority that can be used by a given scheduling class.
5657  * On failure, a negative error code is returned.
5658  */
5659 SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
5660 {
5661         int ret = -EINVAL;
5662 
5663         switch (policy) {
5664         case SCHED_FIFO:
5665         case SCHED_RR:
5666                 ret = MAX_USER_RT_PRIO-1;
5667                 break;
5668         case SCHED_DEADLINE:
5669         case SCHED_NORMAL:
5670         case SCHED_BATCH:
5671         case SCHED_IDLE:
5672                 ret = 0;
5673                 break;
5674         }
5675         return ret;
5676 }
5677 
5678 /**
5679  * sys_sched_get_priority_min - return minimum RT priority.
5680  * @policy: scheduling class.
5681  *
5682  * Return: On success, this syscall returns the minimum
5683  * rt_priority that can be used by a given scheduling class.
5684  * On failure, a negative error code is returned.
5685  */
5686 SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
5687 {
5688         int ret = -EINVAL;
5689 
5690         switch (policy) {
5691         case SCHED_FIFO:
5692         case SCHED_RR:
5693                 ret = 1;
5694                 break;
5695         case SCHED_DEADLINE:
5696         case SCHED_NORMAL:
5697         case SCHED_BATCH:
5698         case SCHED_IDLE:
5699                 ret = 0;
5700         }
5701         return ret;
5702 }
5703 
5704 static int sched_rr_get_interval(pid_t pid, struct timespec64 *t)
5705 {
5706         struct task_struct *p;
5707         unsigned int time_slice;
5708         struct rq_flags rf;
5709         struct rq *rq;
5710         int retval;
5711 
5712         if (pid < 0)
5713                 return -EINVAL;
5714 
5715         retval = -ESRCH;
5716         rcu_read_lock();
5717         p = find_process_by_pid(pid);
5718         if (!p)
5719                 goto out_unlock;
5720 
5721         retval = security_task_getscheduler(p);
5722         if (retval)
5723                 goto out_unlock;
5724 
5725         rq = task_rq_lock(p, &rf);
5726         time_slice = 0;
5727         if (p->sched_class->get_rr_interval)
5728                 time_slice = p->sched_class->get_rr_interval(rq, p);
5729         task_rq_unlock(rq, p, &rf);
5730 
5731         rcu_read_unlock();
5732         jiffies_to_timespec64(time_slice, t);
5733         return 0;
5734 
5735 out_unlock:
5736         rcu_read_unlock();
5737         return retval;
5738 }
5739 
5740 /**
5741  * sys_sched_rr_get_interval - return the default timeslice of a process.
5742  * @pid: pid of the process.
5743  * @interval: userspace pointer to the timeslice value.
5744  *
5745  * this syscall writes the default timeslice value of a given process
5746  * into the user-space timespec buffer. A value of '' means infinity.
5747  *
5748  * Return: On success, 0 and the timeslice is in @interval. Otherwise,
5749  * an error code.
5750  */
5751 SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
5752                 struct __kernel_timespec __user *, interval)
5753 {
5754         struct timespec64 t;
5755         int retval = sched_rr_get_interval(pid, &t);
5756 
5757         if (retval == 0)
5758                 retval = put_timespec64(&t, interval);
5759 
5760         return retval;
5761 }
5762 
5763 #ifdef CONFIG_COMPAT_32BIT_TIME
5764 SYSCALL_DEFINE2(sched_rr_get_interval_time32, pid_t, pid,
5765                 struct old_timespec32 __user *, interval)
5766 {
5767         struct timespec64 t;
5768         int retval = sched_rr_get_interval(pid, &t);
5769 
5770         if (retval == 0)
5771                 retval = put_old_timespec32(&t, interval);
5772         return retval;
5773 }
5774 #endif
5775 
5776 void sched_show_task(struct task_struct *p)
5777 {
5778         unsigned long free = 0;
5779         int ppid;
5780 
5781         if (!try_get_task_stack(p))
5782                 return;
5783 
5784         printk(KERN_INFO "%-15.15s %c", p->comm, task_state_to_char(p));
5785 
5786         if (p->state == TASK_RUNNING)
5787                 printk(KERN_CONT "  running task    ");
5788 #ifdef CONFIG_DEBUG_STACK_USAGE
5789         free = stack_not_used(p);
5790 #endif
5791         ppid = 0;
5792         rcu_read_lock();
5793         if (pid_alive(p))
5794                 ppid = task_pid_nr(rcu_dereference(p->real_parent));
5795         rcu_read_unlock();
5796         printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
5797                 task_pid_nr(p), ppid,
5798                 (unsigned long)task_thread_info(p)->flags);
5799 
5800         print_worker_info(KERN_INFO, p);
5801         show_stack(p, NULL);
5802         put_task_stack(p);
5803 }
5804 EXPORT_SYMBOL_GPL(sched_show_task);
5805 
5806 static inline bool
5807 state_filter_match(unsigned long state_filter, struct task_struct *p)
5808 {
5809         /* no filter, everything matches */
5810         if (!state_filter)
5811                 return true;
5812 
5813         /* filter, but doesn't match */
5814         if (!(p->state & state_filter))
5815                 return false;
5816 
5817         /*
5818          * When looking for TASK_UNINTERRUPTIBLE skip TASK_IDLE (allows
5819          * TASK_KILLABLE).
5820          */
5821         if (state_filter == TASK_UNINTERRUPTIBLE && p->state == TASK_IDLE)
5822                 return false;
5823 
5824         return true;
5825 }
5826 
5827 
5828 void show_state_filter(unsigned long state_filter)
5829 {
5830         struct task_struct *g, *p;
5831 
5832 #if BITS_PER_LONG == 32
5833         printk(KERN_INFO
5834                 "  task                PC stack   pid father\n");
5835 #else
5836         printk(KERN_INFO
5837                 "  task                        PC stack   pid father\n");
5838 #endif
5839         rcu_read_lock();
5840         for_each_process_thread(g, p) {
5841                 /*
5842                  * reset the NMI-timeout, listing all files on a slow
5843                  * console might take a lot of time:
5844                  * Also, reset softlockup watchdogs on all CPUs, because
5845                  * another CPU might be blocked waiting for us to process
5846                  * an IPI.
5847                  */
5848                 touch_nmi_watchdog();
5849                 touch_all_softlockup_watchdogs();
5850                 if (state_filter_match(state_filter, p))
5851                         sched_show_task(p);
5852         }
5853 
5854 #ifdef CONFIG_SCHED_DEBUG
5855         if (!state_filter)
5856                 sysrq_sched_debug_show();
5857 #endif
5858         rcu_read_unlock();
5859         /*
5860          * Only show locks if all tasks are dumped:
5861          */
5862         if (!state_filter)
5863                 debug_show_all_locks();
5864 }
5865 
5866 /**
5867  * init_idle - set up an idle thread for a given CPU
5868  * @idle: task in question
5869  * @cpu: CPU the idle task belongs to
5870  *
5871  * NOTE: this function does not set the idle thread's NEED_RESCHED
5872  * flag, to make booting more robust.
5873  */
5874 void init_idle(struct task_struct *idle, int cpu)
5875 {
5876         struct rq *rq = cpu_rq(cpu);
5877         unsigned long flags;
5878 
5879         __sched_fork(0, idle);
5880 
5881         raw_spin_lock_irqsave(&idle->pi_lock, flags);
5882         raw_spin_lock(&rq->lock);
5883 
5884         idle->state = TASK_RUNNING;
5885         idle->se.exec_start = sched_clock();
5886         idle->flags |= PF_IDLE;
5887 
5888         kasan_unpoison_task_stack(idle);
5889 
5890 #ifdef CONFIG_SMP
5891         /*
5892          * Its possible that init_idle() gets called multiple times on a task,
5893          * in that case do_set_cpus_allowed() will not do the right thing.
5894          *
5895          * And since this is boot we can forgo the serialization.
5896          */
5897         set_cpus_allowed_common(idle, cpumask_of(cpu));
5898 #endif
5899         /*
5900          * We're having a chicken and egg problem, even though we are
5901          * holding rq->lock, the CPU isn't yet set to this CPU so the
5902          * lockdep check in task_group() will fail.
5903          *
5904          * Similar case to sched_fork(). / Alternatively we could
5905          * use task_rq_lock() here and obtain the other rq->lock.
5906          *
5907          * Silence PROVE_RCU
5908          */
5909         rcu_read_lock();
5910         __set_task_cpu(idle, cpu);
5911         rcu_read_unlock();
5912 
5913         rq->curr = rq->idle = idle;
5914         idle->on_rq = TASK_ON_RQ_QUEUED;
5915 #ifdef CONFIG_SMP
5916         idle->on_cpu = 1;
5917 #endif
5918         raw_spin_unlock(&rq->lock);
5919         raw_spin_unlock_irqrestore(&idle->pi_lock, flags);
5920 
5921         /* Set the preempt count _outside_ the spinlocks! */
5922         init_idle_preempt_count(idle, cpu);
5923 
5924         /*
5925          * The idle tasks have their own, simple scheduling class:
5926          */
5927         idle->sched_class = &idle_sched_class;
5928         ftrace_graph_init_idle_task(idle, cpu);
5929         vtime_init_idle(idle, cpu);
5930 #ifdef CONFIG_SMP
5931         sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu);
5932 #endif
5933 }
5934 
5935 #ifdef CONFIG_SMP
5936 
5937 int cpuset_cpumask_can_shrink(const struct cpumask *cur,
5938                               const struct cpumask *trial)
5939 {
5940         int ret = 1;
5941 
5942         if (!cpumask_weight(cur))
5943                 return ret;
5944 
5945         ret = dl_cpuset_cpumask_can_shrink(cur, trial);
5946 
5947         return ret;
5948 }
5949 
5950 int task_can_attach(struct task_struct *p,
5951                     const struct cpumask *cs_cpus_allowed)
5952 {
5953         int ret = 0;
5954 
5955         /*
5956          * Kthreads which disallow setaffinity shouldn't be moved
5957          * to a new cpuset; we don't want to change their CPU
5958          * affinity and isolating such threads by their set of
5959          * allowed nodes is unnecessary.  Thus, cpusets are not
5960          * applicable for such threads.  This prevents checking for
5961          * success of set_cpus_allowed_ptr() on all attached tasks
5962          * before cpus_mask may be changed.
5963          */
5964         if (p->flags & PF_NO_SETAFFINITY) {
5965                 ret = -EINVAL;
5966                 goto out;
5967         }
5968 
5969         if (dl_task(p) && !cpumask_intersects(task_rq(p)->rd->span,
5970                                               cs_cpus_allowed))
5971                 ret = dl_task_can_attach(p, cs_cpus_allowed);
5972 
5973 out:
5974         return ret;
5975 }
5976 
5977 bool sched_smp_initialized __read_mostly;
5978 
5979 #ifdef CONFIG_NUMA_BALANCING
5980 /* Migrate current task p to target_cpu */
5981 int migrate_task_to(struct task_struct *p, int target_cpu)
5982 {
5983         struct migration_arg arg = { p, target_cpu };
5984         int curr_cpu = task_cpu(p);
5985 
5986         if (curr_cpu == target_cpu)
5987                 return 0;
5988 
5989         if (!cpumask_test_cpu(target_cpu, p->cpus_ptr))
5990                 return -EINVAL;
5991 
5992         /* TODO: This is not properly updating schedstats */
5993 
5994         trace_sched_move_numa(p, curr_cpu, target_cpu);
5995         return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg);
5996 }
5997 
5998 /*
5999  * Requeue a task on a given node and accurately track the number of NUMA
6000  * tasks on the runqueues
6001  */
6002 void sched_setnuma(struct task_struct *p, int nid)
6003 {
6004         bool queued, running;
6005         struct rq_flags rf;
6006         struct rq *rq;
6007 
6008         rq = task_rq_lock(p, &rf);
6009         queued = task_on_rq_queued(p);
6010         running = task_current(rq, p);
6011 
6012         if (queued)
6013                 dequeue_task(rq, p, DEQUEUE_SAVE);
6014         if (running)
6015                 put_prev_task(rq, p);
6016 
6017         p->numa_preferred_nid = nid;
6018 
6019         if (queued)
6020                 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
6021         if (running)
6022                 set_curr_task(rq, p);
6023         task_rq_unlock(rq, p, &rf);
6024 }
6025 #endif /* CONFIG_NUMA_BALANCING */
6026 
6027 #ifdef CONFIG_HOTPLUG_CPU
6028 /*
6029  * Ensure that the idle task is using init_mm right before its CPU goes
6030  * offline.
6031  */
6032 void idle_task_exit(void)
6033 {
6034         struct mm_struct *mm = current->active_mm;
6035 
6036         BUG_ON(cpu_online(smp_processor_id()));
6037 
6038         if (mm != &init_mm) {
6039                 switch_mm(mm, &init_mm, current);
6040                 current->active_mm = &init_mm;
6041                 finish_arch_post_lock_switch();
6042         }
6043         mmdrop(mm);
6044 }
6045 
6046 /*
6047  * Since this CPU is going 'away' for a while, fold any nr_active delta
6048  * we might have. Assumes we're called after migrate_tasks() so that the
6049  * nr_active count is stable. We need to take the teardown thread which
6050  * is calling this into account, so we hand in adjust = 1 to the load
6051  * calculation.
6052  *
6053  * Also see the comment "Global load-average calculations".
6054  */
6055 static void calc_load_migrate(struct rq *rq)
6056 {
6057         long delta = calc_load_fold_active(rq, 1);
6058         if (delta)
6059                 atomic_long_add(delta, &calc_load_tasks);
6060 }
6061 
6062 static void put_prev_task_fake(struct rq *rq, struct task_struct *prev)
6063 {
6064 }
6065 
6066 static const struct sched_class fake_sched_class = {
6067         .put_prev_task = put_prev_task_fake,
6068 };
6069 
6070 static struct task_struct fake_task = {
6071         /*
6072          * Avoid pull_{rt,dl}_task()
6073          */
6074         .prio = MAX_PRIO + 1,
6075         .sched_class = &fake_sched_class,
6076 };
6077 
6078 /*
6079  * Migrate all tasks from the rq, sleeping tasks will be migrated by
6080  * try_to_wake_up()->select_task_rq().
6081  *
6082  * Called with rq->lock held even though we'er in stop_machine() and
6083  * there's no concurrency possible, we hold the required locks anyway
6084  * because of lock validation efforts.
6085  */
6086 static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf)
6087 {
6088         struct rq *rq = dead_rq;
6089         struct task_struct *next, *stop = rq->stop;
6090         struct rq_flags orf = *rf;
6091         int dest_cpu;
6092 
6093         /*
6094          * Fudge the rq selection such that the below task selection loop
6095          * doesn't get stuck on the currently eligible stop task.
6096          *
6097          * We're currently inside stop_machine() and the rq is either stuck
6098          * in the stop_machine_cpu_stop() loop, or we're executing this code,
6099          * either way we should never end up calling schedule() until we're
6100          * done here.
6101          */
6102         rq->stop = NULL;
6103 
6104         /*
6105          * put_prev_task() and pick_next_task() sched
6106          * class method both need to have an up-to-date
6107          * value of rq->clock[_task]
6108          */
6109         update_rq_clock(rq);
6110 
6111         for (;;) {
6112                 /*
6113                  * There's this thread running, bail when that's the only
6114                  * remaining thread:
6115                  */
6116                 if (rq->nr_running == 1)
6117                         break;
6118 
6119                 /*
6120                  * pick_next_task() assumes pinned rq->lock:
6121                  */
6122                 next = pick_next_task(rq, &fake_task, rf);
6123                 BUG_ON(!next);
6124                 put_prev_task(rq, next);
6125 
6126                 /*
6127                  * Rules for changing task_struct::cpus_mask are holding
6128                  * both pi_lock and rq->lock, such that holding either
6129                  * stabilizes the mask.
6130                  *
6131                  * Drop rq->lock is not quite as disastrous as it usually is
6132                  * because !cpu_active at this point, which means load-balance
6133                  * will not interfere. Also, stop-machine.
6134                  */
6135                 rq_unlock(rq, rf);
6136                 raw_spin_lock(&next->pi_lock);
6137                 rq_relock(rq, rf);
6138 
6139                 /*
6140                  * Since we're inside stop-machine, _nothing_ should have
6141                  * changed the task, WARN if weird stuff happened, because in
6142                  * that case the above rq->lock drop is a fail too.
6143                  */
6144                 if (WARN_ON(task_rq(next) != rq || !task_on_rq_queued(next))) {
6145                         raw_spin_unlock(&next->pi_lock);
6146                         continue;
6147                 }
6148 
6149                 /* Find suitable destination for @next, with force if needed. */
6150                 dest_cpu = select_fallback_rq(dead_rq->cpu, next);
6151                 rq = __migrate_task(rq, rf, next, dest_cpu);
6152                 if (rq != dead_rq) {
6153                         rq_unlock(rq, rf);
6154                         rq = dead_rq;
6155                         *rf = orf;
6156                         rq_relock(rq, rf);
6157                 }
6158                 raw_spin_unlock(&next->pi_lock);
6159         }
6160 
6161         rq->stop = stop;
6162 }
6163 #endif /* CONFIG_HOTPLUG_CPU */
6164 
6165 void set_rq_online(struct rq *rq)
6166 {
6167         if (!rq->online) {
6168                 const struct sched_class *class;
6169 
6170                 cpumask_set_cpu(rq->cpu, rq->rd->online);
6171                 rq->online = 1;
6172 
6173                 for_each_class(class) {
6174                         if (class->rq_online)
6175                                 class->rq_online(rq);
6176                 }
6177         }
6178 }
6179 
6180 void set_rq_offline(struct rq *rq)
6181 {
6182         if (rq->online) {
6183                 const struct sched_class *class;
6184 
6185                 for_each_class(class) {
6186                         if (class->rq_offline)
6187                                 class->rq_offline(rq);
6188                 }
6189 
6190                 cpumask_clear_cpu(rq->cpu, rq->rd->online);
6191                 rq->online = 0;
6192         }
6193 }
6194 
6195 /*
6196  * used to mark begin/end of suspend/resume:
6197  */
6198 static int num_cpus_frozen;
6199 
6200 /*
6201  * Update cpusets according to cpu_active mask.  If cpusets are
6202  * disabled, cpuset_update_active_cpus() becomes a simple wrapper
6203  * around partition_sched_domains().
6204  *
6205  * If we come here as part of a suspend/resume, don't touch cpusets because we
6206  * want to restore it back to its original state upon resume anyway.
6207  */
6208 static void cpuset_cpu_active(void)
6209 {
6210         if (cpuhp_tasks_frozen) {
6211                 /*
6212                  * num_cpus_frozen tracks how many CPUs are involved in suspend
6213                  * resume sequence. As long as this is not the last online
6214                  * operation in the resume sequence, just build a single sched
6215                  * domain, ignoring cpusets.
6216                  */
6217                 partition_sched_domains(1, NULL, NULL);
6218                 if (--num_cpus_frozen)
6219                         return;
6220                 /*
6221                  * This is the last CPU online operation. So fall through and
6222                  * restore the original sched domains by considering the
6223                  * cpuset configurations.
6224                  */
6225                 cpuset_force_rebuild();
6226         }
6227         cpuset_update_active_cpus();
6228 }
6229 
6230 static int cpuset_cpu_inactive(unsigned int cpu)
6231 {
6232         if (!cpuhp_tasks_frozen) {
6233                 if (dl_cpu_busy(cpu))
6234                         return -EBUSY;
6235                 cpuset_update_active_cpus();
6236         } else {
6237                 num_cpus_frozen++;
6238                 partition_sched_domains(1, NULL, NULL);
6239         }
6240         return 0;
6241 }
6242 
6243 int sched_cpu_activate(unsigned int cpu)
6244 {
6245         struct rq *rq = cpu_rq(cpu);
6246         struct rq_flags rf;
6247 
6248 #ifdef CONFIG_SCHED_SMT
6249         /*
6250          * When going up, increment the number of cores with SMT present.
6251          */
6252         if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
6253                 static_branch_inc_cpuslocked(&sched_smt_present);
6254 #endif
6255         set_cpu_active(cpu, true);
6256 
6257         if (sched_smp_initialized) {
6258                 sched_domains_numa_masks_set(cpu);
6259                 cpuset_cpu_active();
6260         }
6261 
6262         /*
6263          * Put the rq online, if not already. This happens:
6264          *
6265          * 1) In the early boot process, because we build the real domains
6266          *    after all CPUs have been brought up.
6267          *
6268          * 2) At runtime, if cpuset_cpu_active() fails to rebuild the
6269          *    domains.
6270          */
6271         rq_lock_irqsave(rq, &rf);
6272         if (rq->rd) {
6273                 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6274                 set_rq_online(rq);
6275         }
6276         rq_unlock_irqrestore(rq, &rf);
6277 
6278         update_max_interval();
6279 
6280         return 0;
6281 }
6282 
6283 int sched_cpu_deactivate(unsigned int cpu)
6284 {
6285         int ret;
6286 
6287         set_cpu_active(cpu, false);
6288         /*
6289          * We've cleared cpu_active_mask, wait for all preempt-disabled and RCU
6290          * users of this state to go away such that all new such users will
6291          * observe it.
6292          *
6293          * Do sync before park smpboot threads to take care the rcu boost case.
6294          */
6295         synchronize_rcu();
6296 
6297 #ifdef CONFIG_SCHED_SMT
6298         /*
6299          * When going down, decrement the number of cores with SMT present.
6300          */
6301         if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
6302                 static_branch_dec_cpuslocked(&sched_smt_present);
6303 #endif
6304 
6305         if (!sched_smp_initialized)
6306                 return 0;
6307 
6308         ret = cpuset_cpu_inactive(cpu);
6309         if (ret) {
6310                 set_cpu_active(cpu, true);
6311                 return ret;
6312         }
6313         sched_domains_numa_masks_clear(cpu);
6314         return 0;
6315 }
6316 
6317 static void sched_rq_cpu_starting(unsigned int cpu)
6318 {
6319         struct rq *rq = cpu_rq(cpu);
6320 
6321         rq->calc_load_update = calc_load_update;
6322         update_max_interval();
6323 }
6324 
6325 int sched_cpu_starting(unsigned int cpu)
6326 {
6327         sched_rq_cpu_starting(cpu);
6328         sched_tick_start(cpu);
6329         return 0;
6330 }
6331 
6332 #ifdef CONFIG_HOTPLUG_CPU
6333 int sched_cpu_dying(unsigned int cpu)
6334 {
6335         struct rq *rq = cpu_rq(cpu);
6336         struct rq_flags rf;
6337 
6338         /* Handle pending wakeups and then migrate everything off */
6339         sched_ttwu_pending();
6340         sched_tick_stop(cpu);
6341 
6342         rq_lock_irqsave(rq, &rf);
6343         if (rq->rd) {
6344                 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6345                 set_rq_offline(rq);
6346         }
6347         migrate_tasks(rq, &rf);
6348         BUG_ON(rq->nr_running != 1);
6349         rq_unlock_irqrestore(rq, &rf);
6350 
6351         calc_load_migrate(rq);
6352         update_max_interval();
6353         nohz_balance_exit_idle(rq);
6354         hrtick_clear(rq);
6355         return 0;
6356 }
6357 #endif
6358 
6359 void __init sched_init_smp(void)
6360 {
6361         sched_init_numa();
6362 
6363         /*
6364          * There's no userspace yet to cause hotplug operations; hence all the
6365          * CPU masks are stable and all blatant races in the below code cannot
6366          * happen.
6367          */
6368         mutex_lock(&sched_domains_mutex);
6369         sched_init_domains(cpu_active_mask);
6370         mutex_unlock(&sched_domains_mutex);
6371 
6372         /* Move init over to a non-isolated CPU */
6373         if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_FLAG_DOMAIN)) < 0)
6374                 BUG();
6375         sched_init_granularity();
6376 
6377         init_sched_rt_class();
6378         init_sched_dl_class();
6379 
6380         sched_smp_initialized = true;
6381 }
6382 
6383 static int __init migration_init(void)
6384 {
6385         sched_cpu_starting(smp_processor_id());
6386         return 0;
6387 }
6388 early_initcall(migration_init);
6389 
6390 #else
6391 void __init sched_init_smp(void)
6392 {
6393         sched_init_granularity();
6394 }
6395 #endif /* CONFIG_SMP */
6396 
6397 int in_sched_functions(unsigned long addr)
6398 {
6399         return in_lock_functions(addr) ||
6400                 (addr >= (unsigned long)__sched_text_start
6401                 && addr < (unsigned long)__sched_text_end);
6402 }
6403 
6404 #ifdef CONFIG_CGROUP_SCHED
6405 /*
6406  * Default task group.
6407  * Every task in system belongs to this group at bootup.
6408  */
6409 struct task_group root_task_group;
6410 LIST_HEAD(task_groups);
6411 
6412 /* Cacheline aligned slab cache for task_group */
6413 static struct kmem_cache *task_group_cache __read_mostly;
6414 #endif
6415 
6416 DECLARE_PER_CPU(cpumask_var_t, load_balance_mask);
6417 DECLARE_PER_CPU(cpumask_var_t, select_idle_mask);
6418 
6419 void __init sched_init(void)
6420 {
6421         unsigned long alloc_size = 0, ptr;
6422         int i;
6423 
6424         wait_bit_init();
6425 
6426 #ifdef CONFIG_FAIR_GROUP_SCHED
6427         alloc_size += 2 * nr_cpu_ids * sizeof(void **);
6428 #endif
6429 #ifdef CONFIG_RT_GROUP_SCHED
6430         alloc_size += 2 * nr_cpu_ids * sizeof(void **);
6431 #endif
6432         if (alloc_size) {
6433                 ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);
6434 
6435 #ifdef CONFIG_FAIR_GROUP_SCHED
6436                 root_task_group.se = (struct sched_entity **)ptr;
6437                 ptr += nr_cpu_ids * sizeof(void **);
6438 
6439                 root_task_group.cfs_rq = (struct cfs_rq **)ptr;
6440                 ptr += nr_cpu_ids * sizeof(void **);
6441 
6442 #endif /* CONFIG_FAIR_GROUP_SCHED */
6443 #ifdef CONFIG_RT_GROUP_SCHED
6444                 root_task_group.rt_se = (struct sched_rt_entity **)ptr;