~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/sched/fair.c

Version: ~ [ linux-5.13-rc7 ] ~ [ linux-5.12.12 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.45 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.127 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.195 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.237 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.273 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.273 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  * Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH)
  4  *
  5  *  Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
  6  *
  7  *  Interactivity improvements by Mike Galbraith
  8  *  (C) 2007 Mike Galbraith <efault@gmx.de>
  9  *
 10  *  Various enhancements by Dmitry Adamushko.
 11  *  (C) 2007 Dmitry Adamushko <dmitry.adamushko@gmail.com>
 12  *
 13  *  Group scheduling enhancements by Srivatsa Vaddagiri
 14  *  Copyright IBM Corporation, 2007
 15  *  Author: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
 16  *
 17  *  Scaled math optimizations by Thomas Gleixner
 18  *  Copyright (C) 2007, Thomas Gleixner <tglx@linutronix.de>
 19  *
 20  *  Adaptive scheduling granularity, math enhancements by Peter Zijlstra
 21  *  Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra
 22  */
 23 
 24 #include <linux/sched/mm.h>
 25 #include <linux/sched/topology.h>
 26 
 27 #include <linux/latencytop.h>
 28 #include <linux/cpumask.h>
 29 #include <linux/cpuidle.h>
 30 #include <linux/slab.h>
 31 #include <linux/profile.h>
 32 #include <linux/interrupt.h>
 33 #include <linux/mempolicy.h>
 34 #include <linux/migrate.h>
 35 #include <linux/task_work.h>
 36 #include <linux/sched/isolation.h>
 37 
 38 #include <trace/events/sched.h>
 39 
 40 #include "sched.h"
 41 
 42 /*
 43  * Targeted preemption latency for CPU-bound tasks:
 44  *
 45  * NOTE: this latency value is not the same as the concept of
 46  * 'timeslice length' - timeslices in CFS are of variable length
 47  * and have no persistent notion like in traditional, time-slice
 48  * based scheduling concepts.
 49  *
 50  * (to see the precise effective timeslice length of your workload,
 51  *  run vmstat and monitor the context-switches (cs) field)
 52  *
 53  * (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds)
 54  */
 55 unsigned int sysctl_sched_latency                       = 6000000ULL;
 56 unsigned int normalized_sysctl_sched_latency            = 6000000ULL;
 57 
 58 /*
 59  * The initial- and re-scaling of tunables is configurable
 60  *
 61  * Options are:
 62  *
 63  *   SCHED_TUNABLESCALING_NONE - unscaled, always *1
 64  *   SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus)
 65  *   SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus
 66  *
 67  * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus))
 68  */
 69 enum sched_tunable_scaling sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG;
 70 
 71 /*
 72  * Minimal preemption granularity for CPU-bound tasks:
 73  *
 74  * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds)
 75  */
 76 unsigned int sysctl_sched_min_granularity               = 750000ULL;
 77 unsigned int normalized_sysctl_sched_min_granularity    = 750000ULL;
 78 
 79 /*
 80  * This value is kept at sysctl_sched_latency/sysctl_sched_min_granularity
 81  */
 82 static unsigned int sched_nr_latency = 8;
 83 
 84 /*
 85  * After fork, child runs first. If set to 0 (default) then
 86  * parent will (try to) run first.
 87  */
 88 unsigned int sysctl_sched_child_runs_first __read_mostly;
 89 
 90 /*
 91  * SCHED_OTHER wake-up granularity.
 92  *
 93  * This option delays the preemption effects of decoupled workloads
 94  * and reduces their over-scheduling. Synchronous workloads will still
 95  * have immediate wakeup/sleep latencies.
 96  *
 97  * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds)
 98  */
 99 unsigned int sysctl_sched_wakeup_granularity            = 1000000UL;
100 unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL;
101 
102 const_debug unsigned int sysctl_sched_migration_cost    = 500000UL;
103 
104 #ifdef CONFIG_SMP
105 /*
106  * For asym packing, by default the lower numbered cpu has higher priority.
107  */
108 int __weak arch_asym_cpu_priority(int cpu)
109 {
110         return -cpu;
111 }
112 #endif
113 
114 #ifdef CONFIG_CFS_BANDWIDTH
115 /*
116  * Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool
117  * each time a cfs_rq requests quota.
118  *
119  * Note: in the case that the slice exceeds the runtime remaining (either due
120  * to consumption or the quota being specified to be smaller than the slice)
121  * we will always only issue the remaining available time.
122  *
123  * (default: 5 msec, units: microseconds)
124  */
125 unsigned int sysctl_sched_cfs_bandwidth_slice           = 5000UL;
126 #endif
127 
128 /*
129  * The margin used when comparing utilization with CPU capacity:
130  * util * margin < capacity * 1024
131  *
132  * (default: ~20%)
133  */
134 unsigned int capacity_margin                            = 1280;
135 
136 static inline void update_load_add(struct load_weight *lw, unsigned long inc)
137 {
138         lw->weight += inc;
139         lw->inv_weight = 0;
140 }
141 
142 static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
143 {
144         lw->weight -= dec;
145         lw->inv_weight = 0;
146 }
147 
148 static inline void update_load_set(struct load_weight *lw, unsigned long w)
149 {
150         lw->weight = w;
151         lw->inv_weight = 0;
152 }
153 
154 /*
155  * Increase the granularity value when there are more CPUs,
156  * because with more CPUs the 'effective latency' as visible
157  * to users decreases. But the relationship is not linear,
158  * so pick a second-best guess by going with the log2 of the
159  * number of CPUs.
160  *
161  * This idea comes from the SD scheduler of Con Kolivas:
162  */
163 static unsigned int get_update_sysctl_factor(void)
164 {
165         unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8);
166         unsigned int factor;
167 
168         switch (sysctl_sched_tunable_scaling) {
169         case SCHED_TUNABLESCALING_NONE:
170                 factor = 1;
171                 break;
172         case SCHED_TUNABLESCALING_LINEAR:
173                 factor = cpus;
174                 break;
175         case SCHED_TUNABLESCALING_LOG:
176         default:
177                 factor = 1 + ilog2(cpus);
178                 break;
179         }
180 
181         return factor;
182 }
183 
184 static void update_sysctl(void)
185 {
186         unsigned int factor = get_update_sysctl_factor();
187 
188 #define SET_SYSCTL(name) \
189         (sysctl_##name = (factor) * normalized_sysctl_##name)
190         SET_SYSCTL(sched_min_granularity);
191         SET_SYSCTL(sched_latency);
192         SET_SYSCTL(sched_wakeup_granularity);
193 #undef SET_SYSCTL
194 }
195 
196 void sched_init_granularity(void)
197 {
198         update_sysctl();
199 }
200 
201 #define WMULT_CONST     (~0U)
202 #define WMULT_SHIFT     32
203 
204 static void __update_inv_weight(struct load_weight *lw)
205 {
206         unsigned long w;
207 
208         if (likely(lw->inv_weight))
209                 return;
210 
211         w = scale_load_down(lw->weight);
212 
213         if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST))
214                 lw->inv_weight = 1;
215         else if (unlikely(!w))
216                 lw->inv_weight = WMULT_CONST;
217         else
218                 lw->inv_weight = WMULT_CONST / w;
219 }
220 
221 /*
222  * delta_exec * weight / lw.weight
223  *   OR
224  * (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT
225  *
226  * Either weight := NICE_0_LOAD and lw \e sched_prio_to_wmult[], in which case
227  * we're guaranteed shift stays positive because inv_weight is guaranteed to
228  * fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22.
229  *
230  * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus
231  * weight/lw.weight <= 1, and therefore our shift will also be positive.
232  */
233 static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw)
234 {
235         u64 fact = scale_load_down(weight);
236         int shift = WMULT_SHIFT;
237 
238         __update_inv_weight(lw);
239 
240         if (unlikely(fact >> 32)) {
241                 while (fact >> 32) {
242                         fact >>= 1;
243                         shift--;
244                 }
245         }
246 
247         /* hint to use a 32x32->64 mul */
248         fact = (u64)(u32)fact * lw->inv_weight;
249 
250         while (fact >> 32) {
251                 fact >>= 1;
252                 shift--;
253         }
254 
255         return mul_u64_u32_shr(delta_exec, fact, shift);
256 }
257 
258 
259 const struct sched_class fair_sched_class;
260 
261 /**************************************************************
262  * CFS operations on generic schedulable entities:
263  */
264 
265 #ifdef CONFIG_FAIR_GROUP_SCHED
266 
267 /* cpu runqueue to which this cfs_rq is attached */
268 static inline struct rq *rq_of(struct cfs_rq *cfs_rq)
269 {
270         return cfs_rq->rq;
271 }
272 
273 /* An entity is a task if it doesn't "own" a runqueue */
274 #define entity_is_task(se)      (!se->my_q)
275 
276 static inline struct task_struct *task_of(struct sched_entity *se)
277 {
278         SCHED_WARN_ON(!entity_is_task(se));
279         return container_of(se, struct task_struct, se);
280 }
281 
282 /* Walk up scheduling entities hierarchy */
283 #define for_each_sched_entity(se) \
284                 for (; se; se = se->parent)
285 
286 static inline struct cfs_rq *task_cfs_rq(struct task_struct *p)
287 {
288         return p->se.cfs_rq;
289 }
290 
291 /* runqueue on which this entity is (to be) queued */
292 static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se)
293 {
294         return se->cfs_rq;
295 }
296 
297 /* runqueue "owned" by this group */
298 static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp)
299 {
300         return grp->my_q;
301 }
302 
303 static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
304 {
305         if (!cfs_rq->on_list) {
306                 struct rq *rq = rq_of(cfs_rq);
307                 int cpu = cpu_of(rq);
308                 /*
309                  * Ensure we either appear before our parent (if already
310                  * enqueued) or force our parent to appear after us when it is
311                  * enqueued. The fact that we always enqueue bottom-up
312                  * reduces this to two cases and a special case for the root
313                  * cfs_rq. Furthermore, it also means that we will always reset
314                  * tmp_alone_branch either when the branch is connected
315                  * to a tree or when we reach the beg of the tree
316                  */
317                 if (cfs_rq->tg->parent &&
318                     cfs_rq->tg->parent->cfs_rq[cpu]->on_list) {
319                         /*
320                          * If parent is already on the list, we add the child
321                          * just before. Thanks to circular linked property of
322                          * the list, this means to put the child at the tail
323                          * of the list that starts by parent.
324                          */
325                         list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list,
326                                 &(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list));
327                         /*
328                          * The branch is now connected to its tree so we can
329                          * reset tmp_alone_branch to the beginning of the
330                          * list.
331                          */
332                         rq->tmp_alone_branch = &rq->leaf_cfs_rq_list;
333                 } else if (!cfs_rq->tg->parent) {
334                         /*
335                          * cfs rq without parent should be put
336                          * at the tail of the list.
337                          */
338                         list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list,
339                                 &rq->leaf_cfs_rq_list);
340                         /*
341                          * We have reach the beg of a tree so we can reset
342                          * tmp_alone_branch to the beginning of the list.
343                          */
344                         rq->tmp_alone_branch = &rq->leaf_cfs_rq_list;
345                 } else {
346                         /*
347                          * The parent has not already been added so we want to
348                          * make sure that it will be put after us.
349                          * tmp_alone_branch points to the beg of the branch
350                          * where we will add parent.
351                          */
352                         list_add_rcu(&cfs_rq->leaf_cfs_rq_list,
353                                 rq->tmp_alone_branch);
354                         /*
355                          * update tmp_alone_branch to points to the new beg
356                          * of the branch
357                          */
358                         rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list;
359                 }
360 
361                 cfs_rq->on_list = 1;
362         }
363 }
364 
365 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq)
366 {
367         if (cfs_rq->on_list) {
368                 list_del_rcu(&cfs_rq->leaf_cfs_rq_list);
369                 cfs_rq->on_list = 0;
370         }
371 }
372 
373 /* Iterate thr' all leaf cfs_rq's on a runqueue */
374 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos)                      \
375         list_for_each_entry_safe(cfs_rq, pos, &rq->leaf_cfs_rq_list,    \
376                                  leaf_cfs_rq_list)
377 
378 /* Do the two (enqueued) entities belong to the same group ? */
379 static inline struct cfs_rq *
380 is_same_group(struct sched_entity *se, struct sched_entity *pse)
381 {
382         if (se->cfs_rq == pse->cfs_rq)
383                 return se->cfs_rq;
384 
385         return NULL;
386 }
387 
388 static inline struct sched_entity *parent_entity(struct sched_entity *se)
389 {
390         return se->parent;
391 }
392 
393 static void
394 find_matching_se(struct sched_entity **se, struct sched_entity **pse)
395 {
396         int se_depth, pse_depth;
397 
398         /*
399          * preemption test can be made between sibling entities who are in the
400          * same cfs_rq i.e who have a common parent. Walk up the hierarchy of
401          * both tasks until we find their ancestors who are siblings of common
402          * parent.
403          */
404 
405         /* First walk up until both entities are at same depth */
406         se_depth = (*se)->depth;
407         pse_depth = (*pse)->depth;
408 
409         while (se_depth > pse_depth) {
410                 se_depth--;
411                 *se = parent_entity(*se);
412         }
413 
414         while (pse_depth > se_depth) {
415                 pse_depth--;
416                 *pse = parent_entity(*pse);
417         }
418 
419         while (!is_same_group(*se, *pse)) {
420                 *se = parent_entity(*se);
421                 *pse = parent_entity(*pse);
422         }
423 }
424 
425 #else   /* !CONFIG_FAIR_GROUP_SCHED */
426 
427 static inline struct task_struct *task_of(struct sched_entity *se)
428 {
429         return container_of(se, struct task_struct, se);
430 }
431 
432 static inline struct rq *rq_of(struct cfs_rq *cfs_rq)
433 {
434         return container_of(cfs_rq, struct rq, cfs);
435 }
436 
437 #define entity_is_task(se)      1
438 
439 #define for_each_sched_entity(se) \
440                 for (; se; se = NULL)
441 
442 static inline struct cfs_rq *task_cfs_rq(struct task_struct *p)
443 {
444         return &task_rq(p)->cfs;
445 }
446 
447 static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se)
448 {
449         struct task_struct *p = task_of(se);
450         struct rq *rq = task_rq(p);
451 
452         return &rq->cfs;
453 }
454 
455 /* runqueue "owned" by this group */
456 static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp)
457 {
458         return NULL;
459 }
460 
461 static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
462 {
463 }
464 
465 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq)
466 {
467 }
468 
469 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos)      \
470                 for (cfs_rq = &rq->cfs, pos = NULL; cfs_rq; cfs_rq = pos)
471 
472 static inline struct sched_entity *parent_entity(struct sched_entity *se)
473 {
474         return NULL;
475 }
476 
477 static inline void
478 find_matching_se(struct sched_entity **se, struct sched_entity **pse)
479 {
480 }
481 
482 #endif  /* CONFIG_FAIR_GROUP_SCHED */
483 
484 static __always_inline
485 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec);
486 
487 /**************************************************************
488  * Scheduling class tree data structure manipulation methods:
489  */
490 
491 static inline u64 max_vruntime(u64 max_vruntime, u64 vruntime)
492 {
493         s64 delta = (s64)(vruntime - max_vruntime);
494         if (delta > 0)
495                 max_vruntime = vruntime;
496 
497         return max_vruntime;
498 }
499 
500 static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime)
501 {
502         s64 delta = (s64)(vruntime - min_vruntime);
503         if (delta < 0)
504                 min_vruntime = vruntime;
505 
506         return min_vruntime;
507 }
508 
509 static inline int entity_before(struct sched_entity *a,
510                                 struct sched_entity *b)
511 {
512         return (s64)(a->vruntime - b->vruntime) < 0;
513 }
514 
515 static void update_min_vruntime(struct cfs_rq *cfs_rq)
516 {
517         struct sched_entity *curr = cfs_rq->curr;
518         struct rb_node *leftmost = rb_first_cached(&cfs_rq->tasks_timeline);
519 
520         u64 vruntime = cfs_rq->min_vruntime;
521 
522         if (curr) {
523                 if (curr->on_rq)
524                         vruntime = curr->vruntime;
525                 else
526                         curr = NULL;
527         }
528 
529         if (leftmost) { /* non-empty tree */
530                 struct sched_entity *se;
531                 se = rb_entry(leftmost, struct sched_entity, run_node);
532 
533                 if (!curr)
534                         vruntime = se->vruntime;
535                 else
536                         vruntime = min_vruntime(vruntime, se->vruntime);
537         }
538 
539         /* ensure we never gain time by being placed backwards. */
540         cfs_rq->min_vruntime = max_vruntime(cfs_rq->min_vruntime, vruntime);
541 #ifndef CONFIG_64BIT
542         smp_wmb();
543         cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime;
544 #endif
545 }
546 
547 /*
548  * Enqueue an entity into the rb-tree:
549  */
550 static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
551 {
552         struct rb_node **link = &cfs_rq->tasks_timeline.rb_root.rb_node;
553         struct rb_node *parent = NULL;
554         struct sched_entity *entry;
555         bool leftmost = true;
556 
557         /*
558          * Find the right place in the rbtree:
559          */
560         while (*link) {
561                 parent = *link;
562                 entry = rb_entry(parent, struct sched_entity, run_node);
563                 /*
564                  * We dont care about collisions. Nodes with
565                  * the same key stay together.
566                  */
567                 if (entity_before(se, entry)) {
568                         link = &parent->rb_left;
569                 } else {
570                         link = &parent->rb_right;
571                         leftmost = false;
572                 }
573         }
574 
575         rb_link_node(&se->run_node, parent, link);
576         rb_insert_color_cached(&se->run_node,
577                                &cfs_rq->tasks_timeline, leftmost);
578 }
579 
580 static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
581 {
582         rb_erase_cached(&se->run_node, &cfs_rq->tasks_timeline);
583 }
584 
585 struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq)
586 {
587         struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline);
588 
589         if (!left)
590                 return NULL;
591 
592         return rb_entry(left, struct sched_entity, run_node);
593 }
594 
595 static struct sched_entity *__pick_next_entity(struct sched_entity *se)
596 {
597         struct rb_node *next = rb_next(&se->run_node);
598 
599         if (!next)
600                 return NULL;
601 
602         return rb_entry(next, struct sched_entity, run_node);
603 }
604 
605 #ifdef CONFIG_SCHED_DEBUG
606 struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq)
607 {
608         struct rb_node *last = rb_last(&cfs_rq->tasks_timeline.rb_root);
609 
610         if (!last)
611                 return NULL;
612 
613         return rb_entry(last, struct sched_entity, run_node);
614 }
615 
616 /**************************************************************
617  * Scheduling class statistics methods:
618  */
619 
620 int sched_proc_update_handler(struct ctl_table *table, int write,
621                 void __user *buffer, size_t *lenp,
622                 loff_t *ppos)
623 {
624         int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
625         unsigned int factor = get_update_sysctl_factor();
626 
627         if (ret || !write)
628                 return ret;
629 
630         sched_nr_latency = DIV_ROUND_UP(sysctl_sched_latency,
631                                         sysctl_sched_min_granularity);
632 
633 #define WRT_SYSCTL(name) \
634         (normalized_sysctl_##name = sysctl_##name / (factor))
635         WRT_SYSCTL(sched_min_granularity);
636         WRT_SYSCTL(sched_latency);
637         WRT_SYSCTL(sched_wakeup_granularity);
638 #undef WRT_SYSCTL
639 
640         return 0;
641 }
642 #endif
643 
644 /*
645  * delta /= w
646  */
647 static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se)
648 {
649         if (unlikely(se->load.weight != NICE_0_LOAD))
650                 delta = __calc_delta(delta, NICE_0_LOAD, &se->load);
651 
652         return delta;
653 }
654 
655 /*
656  * The idea is to set a period in which each task runs once.
657  *
658  * When there are too many tasks (sched_nr_latency) we have to stretch
659  * this period because otherwise the slices get too small.
660  *
661  * p = (nr <= nl) ? l : l*nr/nl
662  */
663 static u64 __sched_period(unsigned long nr_running)
664 {
665         if (unlikely(nr_running > sched_nr_latency))
666                 return nr_running * sysctl_sched_min_granularity;
667         else
668                 return sysctl_sched_latency;
669 }
670 
671 /*
672  * We calculate the wall-time slice from the period by taking a part
673  * proportional to the weight.
674  *
675  * s = p*P[w/rw]
676  */
677 static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se)
678 {
679         u64 slice = __sched_period(cfs_rq->nr_running + !se->on_rq);
680 
681         for_each_sched_entity(se) {
682                 struct load_weight *load;
683                 struct load_weight lw;
684 
685                 cfs_rq = cfs_rq_of(se);
686                 load = &cfs_rq->load;
687 
688                 if (unlikely(!se->on_rq)) {
689                         lw = cfs_rq->load;
690 
691                         update_load_add(&lw, se->load.weight);
692                         load = &lw;
693                 }
694                 slice = __calc_delta(slice, se->load.weight, load);
695         }
696         return slice;
697 }
698 
699 /*
700  * We calculate the vruntime slice of a to-be-inserted task.
701  *
702  * vs = s/w
703  */
704 static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se)
705 {
706         return calc_delta_fair(sched_slice(cfs_rq, se), se);
707 }
708 
709 #ifdef CONFIG_SMP
710 
711 #include "sched-pelt.h"
712 
713 static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu);
714 static unsigned long task_h_load(struct task_struct *p);
715 
716 /* Give new sched_entity start runnable values to heavy its load in infant time */
717 void init_entity_runnable_average(struct sched_entity *se)
718 {
719         struct sched_avg *sa = &se->avg;
720 
721         memset(sa, 0, sizeof(*sa));
722 
723         /*
724          * Tasks are intialized with full load to be seen as heavy tasks until
725          * they get a chance to stabilize to their real load level.
726          * Group entities are intialized with zero load to reflect the fact that
727          * nothing has been attached to the task group yet.
728          */
729         if (entity_is_task(se))
730                 sa->runnable_load_avg = sa->load_avg = scale_load_down(se->load.weight);
731 
732         se->runnable_weight = se->load.weight;
733 
734         /* when this task enqueue'ed, it will contribute to its cfs_rq's load_avg */
735 }
736 
737 static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq);
738 static void attach_entity_cfs_rq(struct sched_entity *se);
739 
740 /*
741  * With new tasks being created, their initial util_avgs are extrapolated
742  * based on the cfs_rq's current util_avg:
743  *
744  *   util_avg = cfs_rq->util_avg / (cfs_rq->load_avg + 1) * se.load.weight
745  *
746  * However, in many cases, the above util_avg does not give a desired
747  * value. Moreover, the sum of the util_avgs may be divergent, such
748  * as when the series is a harmonic series.
749  *
750  * To solve this problem, we also cap the util_avg of successive tasks to
751  * only 1/2 of the left utilization budget:
752  *
753  *   util_avg_cap = (1024 - cfs_rq->avg.util_avg) / 2^n
754  *
755  * where n denotes the nth task.
756  *
757  * For example, a simplest series from the beginning would be like:
758  *
759  *  task  util_avg: 512, 256, 128,  64,  32,   16,    8, ...
760  * cfs_rq util_avg: 512, 768, 896, 960, 992, 1008, 1016, ...
761  *
762  * Finally, that extrapolated util_avg is clamped to the cap (util_avg_cap)
763  * if util_avg > util_avg_cap.
764  */
765 void post_init_entity_util_avg(struct sched_entity *se)
766 {
767         struct cfs_rq *cfs_rq = cfs_rq_of(se);
768         struct sched_avg *sa = &se->avg;
769         long cap = (long)(SCHED_CAPACITY_SCALE - cfs_rq->avg.util_avg) / 2;
770 
771         if (cap > 0) {
772                 if (cfs_rq->avg.util_avg != 0) {
773                         sa->util_avg  = cfs_rq->avg.util_avg * se->load.weight;
774                         sa->util_avg /= (cfs_rq->avg.load_avg + 1);
775 
776                         if (sa->util_avg > cap)
777                                 sa->util_avg = cap;
778                 } else {
779                         sa->util_avg = cap;
780                 }
781         }
782 
783         if (entity_is_task(se)) {
784                 struct task_struct *p = task_of(se);
785                 if (p->sched_class != &fair_sched_class) {
786                         /*
787                          * For !fair tasks do:
788                          *
789                         update_cfs_rq_load_avg(now, cfs_rq);
790                         attach_entity_load_avg(cfs_rq, se);
791                         switched_from_fair(rq, p);
792                          *
793                          * such that the next switched_to_fair() has the
794                          * expected state.
795                          */
796                         se->avg.last_update_time = cfs_rq_clock_task(cfs_rq);
797                         return;
798                 }
799         }
800 
801         attach_entity_cfs_rq(se);
802 }
803 
804 #else /* !CONFIG_SMP */
805 void init_entity_runnable_average(struct sched_entity *se)
806 {
807 }
808 void post_init_entity_util_avg(struct sched_entity *se)
809 {
810 }
811 static void update_tg_load_avg(struct cfs_rq *cfs_rq, int force)
812 {
813 }
814 #endif /* CONFIG_SMP */
815 
816 /*
817  * Update the current task's runtime statistics.
818  */
819 static void update_curr(struct cfs_rq *cfs_rq)
820 {
821         struct sched_entity *curr = cfs_rq->curr;
822         u64 now = rq_clock_task(rq_of(cfs_rq));
823         u64 delta_exec;
824 
825         if (unlikely(!curr))
826                 return;
827 
828         delta_exec = now - curr->exec_start;
829         if (unlikely((s64)delta_exec <= 0))
830                 return;
831 
832         curr->exec_start = now;
833 
834         schedstat_set(curr->statistics.exec_max,
835                       max(delta_exec, curr->statistics.exec_max));
836 
837         curr->sum_exec_runtime += delta_exec;
838         schedstat_add(cfs_rq->exec_clock, delta_exec);
839 
840         curr->vruntime += calc_delta_fair(delta_exec, curr);
841         update_min_vruntime(cfs_rq);
842 
843         if (entity_is_task(curr)) {
844                 struct task_struct *curtask = task_of(curr);
845 
846                 trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime);
847                 cgroup_account_cputime(curtask, delta_exec);
848                 account_group_exec_runtime(curtask, delta_exec);
849         }
850 
851         account_cfs_rq_runtime(cfs_rq, delta_exec);
852 }
853 
854 static void update_curr_fair(struct rq *rq)
855 {
856         update_curr(cfs_rq_of(&rq->curr->se));
857 }
858 
859 static inline void
860 update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
861 {
862         u64 wait_start, prev_wait_start;
863 
864         if (!schedstat_enabled())
865                 return;
866 
867         wait_start = rq_clock(rq_of(cfs_rq));
868         prev_wait_start = schedstat_val(se->statistics.wait_start);
869 
870         if (entity_is_task(se) && task_on_rq_migrating(task_of(se)) &&
871             likely(wait_start > prev_wait_start))
872                 wait_start -= prev_wait_start;
873 
874         schedstat_set(se->statistics.wait_start, wait_start);
875 }
876 
877 static inline void
878 update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
879 {
880         struct task_struct *p;
881         u64 delta;
882 
883         if (!schedstat_enabled())
884                 return;
885 
886         delta = rq_clock(rq_of(cfs_rq)) - schedstat_val(se->statistics.wait_start);
887 
888         if (entity_is_task(se)) {
889                 p = task_of(se);
890                 if (task_on_rq_migrating(p)) {
891                         /*
892                          * Preserve migrating task's wait time so wait_start
893                          * time stamp can be adjusted to accumulate wait time
894                          * prior to migration.
895                          */
896                         schedstat_set(se->statistics.wait_start, delta);
897                         return;
898                 }
899                 trace_sched_stat_wait(p, delta);
900         }
901 
902         schedstat_set(se->statistics.wait_max,
903                       max(schedstat_val(se->statistics.wait_max), delta));
904         schedstat_inc(se->statistics.wait_count);
905         schedstat_add(se->statistics.wait_sum, delta);
906         schedstat_set(se->statistics.wait_start, 0);
907 }
908 
909 static inline void
910 update_stats_enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
911 {
912         struct task_struct *tsk = NULL;
913         u64 sleep_start, block_start;
914 
915         if (!schedstat_enabled())
916                 return;
917 
918         sleep_start = schedstat_val(se->statistics.sleep_start);
919         block_start = schedstat_val(se->statistics.block_start);
920 
921         if (entity_is_task(se))
922                 tsk = task_of(se);
923 
924         if (sleep_start) {
925                 u64 delta = rq_clock(rq_of(cfs_rq)) - sleep_start;
926 
927                 if ((s64)delta < 0)
928                         delta = 0;
929 
930                 if (unlikely(delta > schedstat_val(se->statistics.sleep_max)))
931                         schedstat_set(se->statistics.sleep_max, delta);
932 
933                 schedstat_set(se->statistics.sleep_start, 0);
934                 schedstat_add(se->statistics.sum_sleep_runtime, delta);
935 
936                 if (tsk) {
937                         account_scheduler_latency(tsk, delta >> 10, 1);
938                         trace_sched_stat_sleep(tsk, delta);
939                 }
940         }
941         if (block_start) {
942                 u64 delta = rq_clock(rq_of(cfs_rq)) - block_start;
943 
944                 if ((s64)delta < 0)
945                         delta = 0;
946 
947                 if (unlikely(delta > schedstat_val(se->statistics.block_max)))
948                         schedstat_set(se->statistics.block_max, delta);
949 
950                 schedstat_set(se->statistics.block_start, 0);
951                 schedstat_add(se->statistics.sum_sleep_runtime, delta);
952 
953                 if (tsk) {
954                         if (tsk->in_iowait) {
955                                 schedstat_add(se->statistics.iowait_sum, delta);
956                                 schedstat_inc(se->statistics.iowait_count);
957                                 trace_sched_stat_iowait(tsk, delta);
958                         }
959 
960                         trace_sched_stat_blocked(tsk, delta);
961 
962                         /*
963                          * Blocking time is in units of nanosecs, so shift by
964                          * 20 to get a milliseconds-range estimation of the
965                          * amount of time that the task spent sleeping:
966                          */
967                         if (unlikely(prof_on == SLEEP_PROFILING)) {
968                                 profile_hits(SLEEP_PROFILING,
969                                                 (void *)get_wchan(tsk),
970                                                 delta >> 20);
971                         }
972                         account_scheduler_latency(tsk, delta >> 10, 0);
973                 }
974         }
975 }
976 
977 /*
978  * Task is being enqueued - update stats:
979  */
980 static inline void
981 update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
982 {
983         if (!schedstat_enabled())
984                 return;
985 
986         /*
987          * Are we enqueueing a waiting task? (for current tasks
988          * a dequeue/enqueue event is a NOP)
989          */
990         if (se != cfs_rq->curr)
991                 update_stats_wait_start(cfs_rq, se);
992 
993         if (flags & ENQUEUE_WAKEUP)
994                 update_stats_enqueue_sleeper(cfs_rq, se);
995 }
996 
997 static inline void
998 update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
999 {
1000 
1001         if (!schedstat_enabled())
1002                 return;
1003 
1004         /*
1005          * Mark the end of the wait period if dequeueing a
1006          * waiting task:
1007          */
1008         if (se != cfs_rq->curr)
1009                 update_stats_wait_end(cfs_rq, se);
1010 
1011         if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) {
1012                 struct task_struct *tsk = task_of(se);
1013 
1014                 if (tsk->state & TASK_INTERRUPTIBLE)
1015                         schedstat_set(se->statistics.sleep_start,
1016                                       rq_clock(rq_of(cfs_rq)));
1017                 if (tsk->state & TASK_UNINTERRUPTIBLE)
1018                         schedstat_set(se->statistics.block_start,
1019                                       rq_clock(rq_of(cfs_rq)));
1020         }
1021 }
1022 
1023 /*
1024  * We are picking a new current task - update its stats:
1025  */
1026 static inline void
1027 update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
1028 {
1029         /*
1030          * We are starting a new run period:
1031          */
1032         se->exec_start = rq_clock_task(rq_of(cfs_rq));
1033 }
1034 
1035 /**************************************************
1036  * Scheduling class queueing methods:
1037  */
1038 
1039 #ifdef CONFIG_NUMA_BALANCING
1040 /*
1041  * Approximate time to scan a full NUMA task in ms. The task scan period is
1042  * calculated based on the tasks virtual memory size and
1043  * numa_balancing_scan_size.
1044  */
1045 unsigned int sysctl_numa_balancing_scan_period_min = 1000;
1046 unsigned int sysctl_numa_balancing_scan_period_max = 60000;
1047 
1048 /* Portion of address space to scan in MB */
1049 unsigned int sysctl_numa_balancing_scan_size = 256;
1050 
1051 /* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */
1052 unsigned int sysctl_numa_balancing_scan_delay = 1000;
1053 
1054 struct numa_group {
1055         atomic_t refcount;
1056 
1057         spinlock_t lock; /* nr_tasks, tasks */
1058         int nr_tasks;
1059         pid_t gid;
1060         int active_nodes;
1061 
1062         struct rcu_head rcu;
1063         unsigned long total_faults;
1064         unsigned long max_faults_cpu;
1065         /*
1066          * Faults_cpu is used to decide whether memory should move
1067          * towards the CPU. As a consequence, these stats are weighted
1068          * more by CPU use than by memory faults.
1069          */
1070         unsigned long *faults_cpu;
1071         unsigned long faults[0];
1072 };
1073 
1074 static inline unsigned long group_faults_priv(struct numa_group *ng);
1075 static inline unsigned long group_faults_shared(struct numa_group *ng);
1076 
1077 static unsigned int task_nr_scan_windows(struct task_struct *p)
1078 {
1079         unsigned long rss = 0;
1080         unsigned long nr_scan_pages;
1081 
1082         /*
1083          * Calculations based on RSS as non-present and empty pages are skipped
1084          * by the PTE scanner and NUMA hinting faults should be trapped based
1085          * on resident pages
1086          */
1087         nr_scan_pages = sysctl_numa_balancing_scan_size << (20 - PAGE_SHIFT);
1088         rss = get_mm_rss(p->mm);
1089         if (!rss)
1090                 rss = nr_scan_pages;
1091 
1092         rss = round_up(rss, nr_scan_pages);
1093         return rss / nr_scan_pages;
1094 }
1095 
1096 /* For sanitys sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */
1097 #define MAX_SCAN_WINDOW 2560
1098 
1099 static unsigned int task_scan_min(struct task_struct *p)
1100 {
1101         unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size);
1102         unsigned int scan, floor;
1103         unsigned int windows = 1;
1104 
1105         if (scan_size < MAX_SCAN_WINDOW)
1106                 windows = MAX_SCAN_WINDOW / scan_size;
1107         floor = 1000 / windows;
1108 
1109         scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p);
1110         return max_t(unsigned int, floor, scan);
1111 }
1112 
1113 static unsigned int task_scan_start(struct task_struct *p)
1114 {
1115         unsigned long smin = task_scan_min(p);
1116         unsigned long period = smin;
1117 
1118         /* Scale the maximum scan period with the amount of shared memory. */
1119         if (p->numa_group) {
1120                 struct numa_group *ng = p->numa_group;
1121                 unsigned long shared = group_faults_shared(ng);
1122                 unsigned long private = group_faults_priv(ng);
1123 
1124                 period *= atomic_read(&ng->refcount);
1125                 period *= shared + 1;
1126                 period /= private + shared + 1;
1127         }
1128 
1129         return max(smin, period);
1130 }
1131 
1132 static unsigned int task_scan_max(struct task_struct *p)
1133 {
1134         unsigned long smin = task_scan_min(p);
1135         unsigned long smax;
1136 
1137         /* Watch for min being lower than max due to floor calculations */
1138         smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p);
1139 
1140         /* Scale the maximum scan period with the amount of shared memory. */
1141         if (p->numa_group) {
1142                 struct numa_group *ng = p->numa_group;
1143                 unsigned long shared = group_faults_shared(ng);
1144                 unsigned long private = group_faults_priv(ng);
1145                 unsigned long period = smax;
1146 
1147                 period *= atomic_read(&ng->refcount);
1148                 period *= shared + 1;
1149                 period /= private + shared + 1;
1150 
1151                 smax = max(smax, period);
1152         }
1153 
1154         return max(smin, smax);
1155 }
1156 
1157 static void account_numa_enqueue(struct rq *rq, struct task_struct *p)
1158 {
1159         rq->nr_numa_running += (p->numa_preferred_nid != -1);
1160         rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p));
1161 }
1162 
1163 static void account_numa_dequeue(struct rq *rq, struct task_struct *p)
1164 {
1165         rq->nr_numa_running -= (p->numa_preferred_nid != -1);
1166         rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p));
1167 }
1168 
1169 /* Shared or private faults. */
1170 #define NR_NUMA_HINT_FAULT_TYPES 2
1171 
1172 /* Memory and CPU locality */
1173 #define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2)
1174 
1175 /* Averaged statistics, and temporary buffers. */
1176 #define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2)
1177 
1178 pid_t task_numa_group_id(struct task_struct *p)
1179 {
1180         return p->numa_group ? p->numa_group->gid : 0;
1181 }
1182 
1183 /*
1184  * The averaged statistics, shared & private, memory & cpu,
1185  * occupy the first half of the array. The second half of the
1186  * array is for current counters, which are averaged into the
1187  * first set by task_numa_placement.
1188  */
1189 static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv)
1190 {
1191         return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv;
1192 }
1193 
1194 static inline unsigned long task_faults(struct task_struct *p, int nid)
1195 {
1196         if (!p->numa_faults)
1197                 return 0;
1198 
1199         return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] +
1200                 p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)];
1201 }
1202 
1203 static inline unsigned long group_faults(struct task_struct *p, int nid)
1204 {
1205         if (!p->numa_group)
1206                 return 0;
1207 
1208         return p->numa_group->faults[task_faults_idx(NUMA_MEM, nid, 0)] +
1209                 p->numa_group->faults[task_faults_idx(NUMA_MEM, nid, 1)];
1210 }
1211 
1212 static inline unsigned long group_faults_cpu(struct numa_group *group, int nid)
1213 {
1214         return group->faults_cpu[task_faults_idx(NUMA_MEM, nid, 0)] +
1215                 group->faults_cpu[task_faults_idx(NUMA_MEM, nid, 1)];
1216 }
1217 
1218 static inline unsigned long group_faults_priv(struct numa_group *ng)
1219 {
1220         unsigned long faults = 0;
1221         int node;
1222 
1223         for_each_online_node(node) {
1224                 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 1)];
1225         }
1226 
1227         return faults;
1228 }
1229 
1230 static inline unsigned long group_faults_shared(struct numa_group *ng)
1231 {
1232         unsigned long faults = 0;
1233         int node;
1234 
1235         for_each_online_node(node) {
1236                 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 0)];
1237         }
1238 
1239         return faults;
1240 }
1241 
1242 /*
1243  * A node triggering more than 1/3 as many NUMA faults as the maximum is
1244  * considered part of a numa group's pseudo-interleaving set. Migrations
1245  * between these nodes are slowed down, to allow things to settle down.
1246  */
1247 #define ACTIVE_NODE_FRACTION 3
1248 
1249 static bool numa_is_active_node(int nid, struct numa_group *ng)
1250 {
1251         return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu;
1252 }
1253 
1254 /* Handle placement on systems where not all nodes are directly connected. */
1255 static unsigned long score_nearby_nodes(struct task_struct *p, int nid,
1256                                         int maxdist, bool task)
1257 {
1258         unsigned long score = 0;
1259         int node;
1260 
1261         /*
1262          * All nodes are directly connected, and the same distance
1263          * from each other. No need for fancy placement algorithms.
1264          */
1265         if (sched_numa_topology_type == NUMA_DIRECT)
1266                 return 0;
1267 
1268         /*
1269          * This code is called for each node, introducing N^2 complexity,
1270          * which should be ok given the number of nodes rarely exceeds 8.
1271          */
1272         for_each_online_node(node) {
1273                 unsigned long faults;
1274                 int dist = node_distance(nid, node);
1275 
1276                 /*
1277                  * The furthest away nodes in the system are not interesting
1278                  * for placement; nid was already counted.
1279                  */
1280                 if (dist == sched_max_numa_distance || node == nid)
1281                         continue;
1282 
1283                 /*
1284                  * On systems with a backplane NUMA topology, compare groups
1285                  * of nodes, and move tasks towards the group with the most
1286                  * memory accesses. When comparing two nodes at distance
1287                  * "hoplimit", only nodes closer by than "hoplimit" are part
1288                  * of each group. Skip other nodes.
1289                  */
1290                 if (sched_numa_topology_type == NUMA_BACKPLANE &&
1291                                         dist > maxdist)
1292                         continue;
1293 
1294                 /* Add up the faults from nearby nodes. */
1295                 if (task)
1296                         faults = task_faults(p, node);
1297                 else
1298                         faults = group_faults(p, node);
1299 
1300                 /*
1301                  * On systems with a glueless mesh NUMA topology, there are
1302                  * no fixed "groups of nodes". Instead, nodes that are not
1303                  * directly connected bounce traffic through intermediate
1304                  * nodes; a numa_group can occupy any set of nodes.
1305                  * The further away a node is, the less the faults count.
1306                  * This seems to result in good task placement.
1307                  */
1308                 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) {
1309                         faults *= (sched_max_numa_distance - dist);
1310                         faults /= (sched_max_numa_distance - LOCAL_DISTANCE);
1311                 }
1312 
1313                 score += faults;
1314         }
1315 
1316         return score;
1317 }
1318 
1319 /*
1320  * These return the fraction of accesses done by a particular task, or
1321  * task group, on a particular numa node.  The group weight is given a
1322  * larger multiplier, in order to group tasks together that are almost
1323  * evenly spread out between numa nodes.
1324  */
1325 static inline unsigned long task_weight(struct task_struct *p, int nid,
1326                                         int dist)
1327 {
1328         unsigned long faults, total_faults;
1329 
1330         if (!p->numa_faults)
1331                 return 0;
1332 
1333         total_faults = p->total_numa_faults;
1334 
1335         if (!total_faults)
1336                 return 0;
1337 
1338         faults = task_faults(p, nid);
1339         faults += score_nearby_nodes(p, nid, dist, true);
1340 
1341         return 1000 * faults / total_faults;
1342 }
1343 
1344 static inline unsigned long group_weight(struct task_struct *p, int nid,
1345                                          int dist)
1346 {
1347         unsigned long faults, total_faults;
1348 
1349         if (!p->numa_group)
1350                 return 0;
1351 
1352         total_faults = p->numa_group->total_faults;
1353 
1354         if (!total_faults)
1355                 return 0;
1356 
1357         faults = group_faults(p, nid);
1358         faults += score_nearby_nodes(p, nid, dist, false);
1359 
1360         return 1000 * faults / total_faults;
1361 }
1362 
1363 bool should_numa_migrate_memory(struct task_struct *p, struct page * page,
1364                                 int src_nid, int dst_cpu)
1365 {
1366         struct numa_group *ng = p->numa_group;
1367         int dst_nid = cpu_to_node(dst_cpu);
1368         int last_cpupid, this_cpupid;
1369 
1370         this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid);
1371 
1372         /*
1373          * Multi-stage node selection is used in conjunction with a periodic
1374          * migration fault to build a temporal task<->page relation. By using
1375          * a two-stage filter we remove short/unlikely relations.
1376          *
1377          * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate
1378          * a task's usage of a particular page (n_p) per total usage of this
1379          * page (n_t) (in a given time-span) to a probability.
1380          *
1381          * Our periodic faults will sample this probability and getting the
1382          * same result twice in a row, given these samples are fully
1383          * independent, is then given by P(n)^2, provided our sample period
1384          * is sufficiently short compared to the usage pattern.
1385          *
1386          * This quadric squishes small probabilities, making it less likely we
1387          * act on an unlikely task<->page relation.
1388          */
1389         last_cpupid = page_cpupid_xchg_last(page, this_cpupid);
1390         if (!cpupid_pid_unset(last_cpupid) &&
1391                                 cpupid_to_nid(last_cpupid) != dst_nid)
1392                 return false;
1393 
1394         /* Always allow migrate on private faults */
1395         if (cpupid_match_pid(p, last_cpupid))
1396                 return true;
1397 
1398         /* A shared fault, but p->numa_group has not been set up yet. */
1399         if (!ng)
1400                 return true;
1401 
1402         /*
1403          * Destination node is much more heavily used than the source
1404          * node? Allow migration.
1405          */
1406         if (group_faults_cpu(ng, dst_nid) > group_faults_cpu(ng, src_nid) *
1407                                         ACTIVE_NODE_FRACTION)
1408                 return true;
1409 
1410         /*
1411          * Distribute memory according to CPU & memory use on each node,
1412          * with 3/4 hysteresis to avoid unnecessary memory migrations:
1413          *
1414          * faults_cpu(dst)   3   faults_cpu(src)
1415          * --------------- * - > ---------------
1416          * faults_mem(dst)   4   faults_mem(src)
1417          */
1418         return group_faults_cpu(ng, dst_nid) * group_faults(p, src_nid) * 3 >
1419                group_faults_cpu(ng, src_nid) * group_faults(p, dst_nid) * 4;
1420 }
1421 
1422 static unsigned long weighted_cpuload(struct rq *rq);
1423 static unsigned long source_load(int cpu, int type);
1424 static unsigned long target_load(int cpu, int type);
1425 static unsigned long capacity_of(int cpu);
1426 
1427 /* Cached statistics for all CPUs within a node */
1428 struct numa_stats {
1429         unsigned long nr_running;
1430         unsigned long load;
1431 
1432         /* Total compute capacity of CPUs on a node */
1433         unsigned long compute_capacity;
1434 
1435         /* Approximate capacity in terms of runnable tasks on a node */
1436         unsigned long task_capacity;
1437         int has_free_capacity;
1438 };
1439 
1440 /*
1441  * XXX borrowed from update_sg_lb_stats
1442  */
1443 static void update_numa_stats(struct numa_stats *ns, int nid)
1444 {
1445         int smt, cpu, cpus = 0;
1446         unsigned long capacity;
1447 
1448         memset(ns, 0, sizeof(*ns));
1449         for_each_cpu(cpu, cpumask_of_node(nid)) {
1450                 struct rq *rq = cpu_rq(cpu);
1451 
1452                 ns->nr_running += rq->nr_running;
1453                 ns->load += weighted_cpuload(rq);
1454                 ns->compute_capacity += capacity_of(cpu);
1455 
1456                 cpus++;
1457         }
1458 
1459         /*
1460          * If we raced with hotplug and there are no CPUs left in our mask
1461          * the @ns structure is NULL'ed and task_numa_compare() will
1462          * not find this node attractive.
1463          *
1464          * We'll either bail at !has_free_capacity, or we'll detect a huge
1465          * imbalance and bail there.
1466          */
1467         if (!cpus)
1468                 return;
1469 
1470         /* smt := ceil(cpus / capacity), assumes: 1 < smt_power < 2 */
1471         smt = DIV_ROUND_UP(SCHED_CAPACITY_SCALE * cpus, ns->compute_capacity);
1472         capacity = cpus / smt; /* cores */
1473 
1474         ns->task_capacity = min_t(unsigned, capacity,
1475                 DIV_ROUND_CLOSEST(ns->compute_capacity, SCHED_CAPACITY_SCALE));
1476         ns->has_free_capacity = (ns->nr_running < ns->task_capacity);
1477 }
1478 
1479 struct task_numa_env {
1480         struct task_struct *p;
1481 
1482         int src_cpu, src_nid;
1483         int dst_cpu, dst_nid;
1484 
1485         struct numa_stats src_stats, dst_stats;
1486 
1487         int imbalance_pct;
1488         int dist;
1489 
1490         struct task_struct *best_task;
1491         long best_imp;
1492         int best_cpu;
1493 };
1494 
1495 static void task_numa_assign(struct task_numa_env *env,
1496                              struct task_struct *p, long imp)
1497 {
1498         if (env->best_task)
1499                 put_task_struct(env->best_task);
1500         if (p)
1501                 get_task_struct(p);
1502 
1503         env->best_task = p;
1504         env->best_imp = imp;
1505         env->best_cpu = env->dst_cpu;
1506 }
1507 
1508 static bool load_too_imbalanced(long src_load, long dst_load,
1509                                 struct task_numa_env *env)
1510 {
1511         long imb, old_imb;
1512         long orig_src_load, orig_dst_load;
1513         long src_capacity, dst_capacity;
1514 
1515         /*
1516          * The load is corrected for the CPU capacity available on each node.
1517          *
1518          * src_load        dst_load
1519          * ------------ vs ---------
1520          * src_capacity    dst_capacity
1521          */
1522         src_capacity = env->src_stats.compute_capacity;
1523         dst_capacity = env->dst_stats.compute_capacity;
1524 
1525         /* We care about the slope of the imbalance, not the direction. */
1526         if (dst_load < src_load)
1527                 swap(dst_load, src_load);
1528 
1529         /* Is the difference below the threshold? */
1530         imb = dst_load * src_capacity * 100 -
1531               src_load * dst_capacity * env->imbalance_pct;
1532         if (imb <= 0)
1533                 return false;
1534 
1535         /*
1536          * The imbalance is above the allowed threshold.
1537          * Compare it with the old imbalance.
1538          */
1539         orig_src_load = env->src_stats.load;
1540         orig_dst_load = env->dst_stats.load;
1541 
1542         if (orig_dst_load < orig_src_load)
1543                 swap(orig_dst_load, orig_src_load);
1544 
1545         old_imb = orig_dst_load * src_capacity * 100 -
1546                   orig_src_load * dst_capacity * env->imbalance_pct;
1547 
1548         /* Would this change make things worse? */
1549         return (imb > old_imb);
1550 }
1551 
1552 /*
1553  * This checks if the overall compute and NUMA accesses of the system would
1554  * be improved if the source tasks was migrated to the target dst_cpu taking
1555  * into account that it might be best if task running on the dst_cpu should
1556  * be exchanged with the source task
1557  */
1558 static void task_numa_compare(struct task_numa_env *env,
1559                               long taskimp, long groupimp)
1560 {
1561         struct rq *src_rq = cpu_rq(env->src_cpu);
1562         struct rq *dst_rq = cpu_rq(env->dst_cpu);
1563         struct task_struct *cur;
1564         long src_load, dst_load;
1565         long load;
1566         long imp = env->p->numa_group ? groupimp : taskimp;
1567         long moveimp = imp;
1568         int dist = env->dist;
1569 
1570         rcu_read_lock();
1571         cur = task_rcu_dereference(&dst_rq->curr);
1572         if (cur && ((cur->flags & PF_EXITING) || is_idle_task(cur)))
1573                 cur = NULL;
1574 
1575         /*
1576          * Because we have preemption enabled we can get migrated around and
1577          * end try selecting ourselves (current == env->p) as a swap candidate.
1578          */
1579         if (cur == env->p)
1580                 goto unlock;
1581 
1582         /*
1583          * "imp" is the fault differential for the source task between the
1584          * source and destination node. Calculate the total differential for
1585          * the source task and potential destination task. The more negative
1586          * the value is, the more rmeote accesses that would be expected to
1587          * be incurred if the tasks were swapped.
1588          */
1589         if (cur) {
1590                 /* Skip this swap candidate if cannot move to the source cpu */
1591                 if (!cpumask_test_cpu(env->src_cpu, &cur->cpus_allowed))
1592                         goto unlock;
1593 
1594                 /*
1595                  * If dst and source tasks are in the same NUMA group, or not
1596                  * in any group then look only at task weights.
1597                  */
1598                 if (cur->numa_group == env->p->numa_group) {
1599                         imp = taskimp + task_weight(cur, env->src_nid, dist) -
1600                               task_weight(cur, env->dst_nid, dist);
1601                         /*
1602                          * Add some hysteresis to prevent swapping the
1603                          * tasks within a group over tiny differences.
1604                          */
1605                         if (cur->numa_group)
1606                                 imp -= imp/16;
1607                 } else {
1608                         /*
1609                          * Compare the group weights. If a task is all by
1610                          * itself (not part of a group), use the task weight
1611                          * instead.
1612                          */
1613                         if (cur->numa_group)
1614                                 imp += group_weight(cur, env->src_nid, dist) -
1615                                        group_weight(cur, env->dst_nid, dist);
1616                         else
1617                                 imp += task_weight(cur, env->src_nid, dist) -
1618                                        task_weight(cur, env->dst_nid, dist);
1619                 }
1620         }
1621 
1622         if (imp <= env->best_imp && moveimp <= env->best_imp)
1623                 goto unlock;
1624 
1625         if (!cur) {
1626                 /* Is there capacity at our destination? */
1627                 if (env->src_stats.nr_running <= env->src_stats.task_capacity &&
1628                     !env->dst_stats.has_free_capacity)
1629                         goto unlock;
1630 
1631                 goto balance;
1632         }
1633 
1634         /* Balance doesn't matter much if we're running a task per cpu */
1635         if (imp > env->best_imp && src_rq->nr_running == 1 &&
1636                         dst_rq->nr_running == 1)
1637                 goto assign;
1638 
1639         /*
1640          * In the overloaded case, try and keep the load balanced.
1641          */
1642 balance:
1643         load = task_h_load(env->p);
1644         dst_load = env->dst_stats.load + load;
1645         src_load = env->src_stats.load - load;
1646 
1647         if (moveimp > imp && moveimp > env->best_imp) {
1648                 /*
1649                  * If the improvement from just moving env->p direction is
1650                  * better than swapping tasks around, check if a move is
1651                  * possible. Store a slightly smaller score than moveimp,
1652                  * so an actually idle CPU will win.
1653                  */
1654                 if (!load_too_imbalanced(src_load, dst_load, env)) {
1655                         imp = moveimp - 1;
1656                         cur = NULL;
1657                         goto assign;
1658                 }
1659         }
1660 
1661         if (imp <= env->best_imp)
1662                 goto unlock;
1663 
1664         if (cur) {
1665                 load = task_h_load(cur);
1666                 dst_load -= load;
1667                 src_load += load;
1668         }
1669 
1670         if (load_too_imbalanced(src_load, dst_load, env))
1671                 goto unlock;
1672 
1673         /*
1674          * One idle CPU per node is evaluated for a task numa move.
1675          * Call select_idle_sibling to maybe find a better one.
1676          */
1677         if (!cur) {
1678                 /*
1679                  * select_idle_siblings() uses an per-cpu cpumask that
1680                  * can be used from IRQ context.
1681                  */
1682                 local_irq_disable();
1683                 env->dst_cpu = select_idle_sibling(env->p, env->src_cpu,
1684                                                    env->dst_cpu);
1685                 local_irq_enable();
1686         }
1687 
1688 assign:
1689         task_numa_assign(env, cur, imp);
1690 unlock:
1691         rcu_read_unlock();
1692 }
1693 
1694 static void task_numa_find_cpu(struct task_numa_env *env,
1695                                 long taskimp, long groupimp)
1696 {
1697         int cpu;
1698 
1699         for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) {
1700                 /* Skip this CPU if the source task cannot migrate */
1701                 if (!cpumask_test_cpu(cpu, &env->p->cpus_allowed))
1702                         continue;
1703 
1704                 env->dst_cpu = cpu;
1705                 task_numa_compare(env, taskimp, groupimp);
1706         }
1707 }
1708 
1709 /* Only move tasks to a NUMA node less busy than the current node. */
1710 static bool numa_has_capacity(struct task_numa_env *env)
1711 {
1712         struct numa_stats *src = &env->src_stats;
1713         struct numa_stats *dst = &env->dst_stats;
1714 
1715         if (src->has_free_capacity && !dst->has_free_capacity)
1716                 return false;
1717 
1718         /*
1719          * Only consider a task move if the source has a higher load
1720          * than the destination, corrected for CPU capacity on each node.
1721          *
1722          *      src->load                dst->load
1723          * --------------------- vs ---------------------
1724          * src->compute_capacity    dst->compute_capacity
1725          */
1726         if (src->load * dst->compute_capacity * env->imbalance_pct >
1727 
1728             dst->load * src->compute_capacity * 100)
1729                 return true;
1730 
1731         return false;
1732 }
1733 
1734 static int task_numa_migrate(struct task_struct *p)
1735 {
1736         struct task_numa_env env = {
1737                 .p = p,
1738 
1739                 .src_cpu = task_cpu(p),
1740                 .src_nid = task_node(p),
1741 
1742                 .imbalance_pct = 112,
1743 
1744                 .best_task = NULL,
1745                 .best_imp = 0,
1746                 .best_cpu = -1,
1747         };
1748         struct sched_domain *sd;
1749         unsigned long taskweight, groupweight;
1750         int nid, ret, dist;
1751         long taskimp, groupimp;
1752 
1753         /*
1754          * Pick the lowest SD_NUMA domain, as that would have the smallest
1755          * imbalance and would be the first to start moving tasks about.
1756          *
1757          * And we want to avoid any moving of tasks about, as that would create
1758          * random movement of tasks -- counter the numa conditions we're trying
1759          * to satisfy here.
1760          */
1761         rcu_read_lock();
1762         sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu));
1763         if (sd)
1764                 env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2;
1765         rcu_read_unlock();
1766 
1767         /*
1768          * Cpusets can break the scheduler domain tree into smaller
1769          * balance domains, some of which do not cross NUMA boundaries.
1770          * Tasks that are "trapped" in such domains cannot be migrated
1771          * elsewhere, so there is no point in (re)trying.
1772          */
1773         if (unlikely(!sd)) {
1774                 p->numa_preferred_nid = task_node(p);
1775                 return -EINVAL;
1776         }
1777 
1778         env.dst_nid = p->numa_preferred_nid;
1779         dist = env.dist = node_distance(env.src_nid, env.dst_nid);
1780         taskweight = task_weight(p, env.src_nid, dist);
1781         groupweight = group_weight(p, env.src_nid, dist);
1782         update_numa_stats(&env.src_stats, env.src_nid);
1783         taskimp = task_weight(p, env.dst_nid, dist) - taskweight;
1784         groupimp = group_weight(p, env.dst_nid, dist) - groupweight;
1785         update_numa_stats(&env.dst_stats, env.dst_nid);
1786 
1787         /* Try to find a spot on the preferred nid. */
1788         if (numa_has_capacity(&env))
1789                 task_numa_find_cpu(&env, taskimp, groupimp);
1790 
1791         /*
1792          * Look at other nodes in these cases:
1793          * - there is no space available on the preferred_nid
1794          * - the task is part of a numa_group that is interleaved across
1795          *   multiple NUMA nodes; in order to better consolidate the group,
1796          *   we need to check other locations.
1797          */
1798         if (env.best_cpu == -1 || (p->numa_group && p->numa_group->active_nodes > 1)) {
1799                 for_each_online_node(nid) {
1800                         if (nid == env.src_nid || nid == p->numa_preferred_nid)
1801                                 continue;
1802 
1803                         dist = node_distance(env.src_nid, env.dst_nid);
1804                         if (sched_numa_topology_type == NUMA_BACKPLANE &&
1805                                                 dist != env.dist) {
1806                                 taskweight = task_weight(p, env.src_nid, dist);
1807                                 groupweight = group_weight(p, env.src_nid, dist);
1808                         }
1809 
1810                         /* Only consider nodes where both task and groups benefit */
1811                         taskimp = task_weight(p, nid, dist) - taskweight;
1812                         groupimp = group_weight(p, nid, dist) - groupweight;
1813                         if (taskimp < 0 && groupimp < 0)
1814                                 continue;
1815 
1816                         env.dist = dist;
1817                         env.dst_nid = nid;
1818                         update_numa_stats(&env.dst_stats, env.dst_nid);
1819                         if (numa_has_capacity(&env))
1820                                 task_numa_find_cpu(&env, taskimp, groupimp);
1821                 }
1822         }
1823 
1824         /*
1825          * If the task is part of a workload that spans multiple NUMA nodes,
1826          * and is migrating into one of the workload's active nodes, remember
1827          * this node as the task's preferred numa node, so the workload can
1828          * settle down.
1829          * A task that migrated to a second choice node will be better off
1830          * trying for a better one later. Do not set the preferred node here.
1831          */
1832         if (p->numa_group) {
1833                 struct numa_group *ng = p->numa_group;
1834 
1835                 if (env.best_cpu == -1)
1836                         nid = env.src_nid;
1837                 else
1838                         nid = env.dst_nid;
1839 
1840                 if (ng->active_nodes > 1 && numa_is_active_node(env.dst_nid, ng))
1841                         sched_setnuma(p, env.dst_nid);
1842         }
1843 
1844         /* No better CPU than the current one was found. */
1845         if (env.best_cpu == -1)
1846                 return -EAGAIN;
1847 
1848         /*
1849          * Reset the scan period if the task is being rescheduled on an
1850          * alternative node to recheck if the tasks is now properly placed.
1851          */
1852         p->numa_scan_period = task_scan_start(p);
1853 
1854         if (env.best_task == NULL) {
1855                 ret = migrate_task_to(p, env.best_cpu);
1856                 if (ret != 0)
1857                         trace_sched_stick_numa(p, env.src_cpu, env.best_cpu);
1858                 return ret;
1859         }
1860 
1861         ret = migrate_swap(p, env.best_task);
1862         if (ret != 0)
1863                 trace_sched_stick_numa(p, env.src_cpu, task_cpu(env.best_task));
1864         put_task_struct(env.best_task);
1865         return ret;
1866 }
1867 
1868 /* Attempt to migrate a task to a CPU on the preferred node. */
1869 static void numa_migrate_preferred(struct task_struct *p)
1870 {
1871         unsigned long interval = HZ;
1872 
1873         /* This task has no NUMA fault statistics yet */
1874         if (unlikely(p->numa_preferred_nid == -1 || !p->numa_faults))
1875                 return;
1876 
1877         /* Periodically retry migrating the task to the preferred node */
1878         interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16);
1879         p->numa_migrate_retry = jiffies + interval;
1880 
1881         /* Success if task is already running on preferred CPU */
1882         if (task_node(p) == p->numa_preferred_nid)
1883                 return;
1884 
1885         /* Otherwise, try migrate to a CPU on the preferred node */
1886         task_numa_migrate(p);
1887 }
1888 
1889 /*
1890  * Find out how many nodes on the workload is actively running on. Do this by
1891  * tracking the nodes from which NUMA hinting faults are triggered. This can
1892  * be different from the set of nodes where the workload's memory is currently
1893  * located.
1894  */
1895 static void numa_group_count_active_nodes(struct numa_group *numa_group)
1896 {
1897         unsigned long faults, max_faults = 0;
1898         int nid, active_nodes = 0;
1899 
1900         for_each_online_node(nid) {
1901                 faults = group_faults_cpu(numa_group, nid);
1902                 if (faults > max_faults)
1903                         max_faults = faults;
1904         }
1905 
1906         for_each_online_node(nid) {
1907                 faults = group_faults_cpu(numa_group, nid);
1908                 if (faults * ACTIVE_NODE_FRACTION > max_faults)
1909                         active_nodes++;
1910         }
1911 
1912         numa_group->max_faults_cpu = max_faults;
1913         numa_group->active_nodes = active_nodes;
1914 }
1915 
1916 /*
1917  * When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS
1918  * increments. The more local the fault statistics are, the higher the scan
1919  * period will be for the next scan window. If local/(local+remote) ratio is
1920  * below NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS)
1921  * the scan period will decrease. Aim for 70% local accesses.
1922  */
1923 #define NUMA_PERIOD_SLOTS 10
1924 #define NUMA_PERIOD_THRESHOLD 7
1925 
1926 /*
1927  * Increase the scan period (slow down scanning) if the majority of
1928  * our memory is already on our local node, or if the majority of
1929  * the page accesses are shared with other processes.
1930  * Otherwise, decrease the scan period.
1931  */
1932 static void update_task_scan_period(struct task_struct *p,
1933                         unsigned long shared, unsigned long private)
1934 {
1935         unsigned int period_slot;
1936         int lr_ratio, ps_ratio;
1937         int diff;
1938 
1939         unsigned long remote = p->numa_faults_locality[0];
1940         unsigned long local = p->numa_faults_locality[1];
1941 
1942         /*
1943          * If there were no record hinting faults then either the task is
1944          * completely idle or all activity is areas that are not of interest
1945          * to automatic numa balancing. Related to that, if there were failed
1946          * migration then it implies we are migrating too quickly or the local
1947          * node is overloaded. In either case, scan slower
1948          */
1949         if (local + shared == 0 || p->numa_faults_locality[2]) {
1950                 p->numa_scan_period = min(p->numa_scan_period_max,
1951                         p->numa_scan_period << 1);
1952 
1953                 p->mm->numa_next_scan = jiffies +
1954                         msecs_to_jiffies(p->numa_scan_period);
1955 
1956                 return;
1957         }
1958 
1959         /*
1960          * Prepare to scale scan period relative to the current period.
1961          *       == NUMA_PERIOD_THRESHOLD scan period stays the same
1962          *       <  NUMA_PERIOD_THRESHOLD scan period decreases (scan faster)
1963          *       >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower)
1964          */
1965         period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS);
1966         lr_ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote);
1967         ps_ratio = (private * NUMA_PERIOD_SLOTS) / (private + shared);
1968 
1969         if (ps_ratio >= NUMA_PERIOD_THRESHOLD) {
1970                 /*
1971                  * Most memory accesses are local. There is no need to
1972                  * do fast NUMA scanning, since memory is already local.
1973                  */
1974                 int slot = ps_ratio - NUMA_PERIOD_THRESHOLD;
1975                 if (!slot)
1976                         slot = 1;
1977                 diff = slot * period_slot;
1978         } else if (lr_ratio >= NUMA_PERIOD_THRESHOLD) {
1979                 /*
1980                  * Most memory accesses are shared with other tasks.
1981                  * There is no point in continuing fast NUMA scanning,
1982                  * since other tasks may just move the memory elsewhere.
1983                  */
1984                 int slot = lr_ratio - NUMA_PERIOD_THRESHOLD;
1985                 if (!slot)
1986                         slot = 1;
1987                 diff = slot * period_slot;
1988         } else {
1989                 /*
1990                  * Private memory faults exceed (SLOTS-THRESHOLD)/SLOTS,
1991                  * yet they are not on the local NUMA node. Speed up
1992                  * NUMA scanning to get the memory moved over.
1993                  */
1994                 int ratio = max(lr_ratio, ps_ratio);
1995                 diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot;
1996         }
1997 
1998         p->numa_scan_period = clamp(p->numa_scan_period + diff,
1999                         task_scan_min(p), task_scan_max(p));
2000         memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality));
2001 }
2002 
2003 /*
2004  * Get the fraction of time the task has been running since the last
2005  * NUMA placement cycle. The scheduler keeps similar statistics, but
2006  * decays those on a 32ms period, which is orders of magnitude off
2007  * from the dozens-of-seconds NUMA balancing period. Use the scheduler
2008  * stats only if the task is so new there are no NUMA statistics yet.
2009  */
2010 static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period)
2011 {
2012         u64 runtime, delta, now;
2013         /* Use the start of this time slice to avoid calculations. */
2014         now = p->se.exec_start;
2015         runtime = p->se.sum_exec_runtime;
2016 
2017         if (p->last_task_numa_placement) {
2018                 delta = runtime - p->last_sum_exec_runtime;
2019                 *period = now - p->last_task_numa_placement;
2020         } else {
2021                 delta = p->se.avg.load_sum;
2022                 *period = LOAD_AVG_MAX;
2023         }
2024 
2025         p->last_sum_exec_runtime = runtime;
2026         p->last_task_numa_placement = now;
2027 
2028         return delta;
2029 }
2030 
2031 /*
2032  * Determine the preferred nid for a task in a numa_group. This needs to
2033  * be done in a way that produces consistent results with group_weight,
2034  * otherwise workloads might not converge.
2035  */
2036 static int preferred_group_nid(struct task_struct *p, int nid)
2037 {
2038         nodemask_t nodes;
2039         int dist;
2040 
2041         /* Direct connections between all NUMA nodes. */
2042         if (sched_numa_topology_type == NUMA_DIRECT)
2043                 return nid;
2044 
2045         /*
2046          * On a system with glueless mesh NUMA topology, group_weight
2047          * scores nodes according to the number of NUMA hinting faults on
2048          * both the node itself, and on nearby nodes.
2049          */
2050         if (sched_numa_topology_type == NUMA_GLUELESS_MESH) {
2051                 unsigned long score, max_score = 0;
2052                 int node, max_node = nid;
2053 
2054                 dist = sched_max_numa_distance;
2055 
2056                 for_each_online_node(node) {
2057                         score = group_weight(p, node, dist);
2058                         if (score > max_score) {
2059                                 max_score = score;
2060                                 max_node = node;
2061                         }
2062                 }
2063                 return max_node;
2064         }
2065 
2066         /*
2067          * Finding the preferred nid in a system with NUMA backplane
2068          * interconnect topology is more involved. The goal is to locate
2069          * tasks from numa_groups near each other in the system, and
2070          * untangle workloads from different sides of the system. This requires
2071          * searching down the hierarchy of node groups, recursively searching
2072          * inside the highest scoring group of nodes. The nodemask tricks
2073          * keep the complexity of the search down.
2074          */
2075         nodes = node_online_map;
2076         for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) {
2077                 unsigned long max_faults = 0;
2078                 nodemask_t max_group = NODE_MASK_NONE;
2079                 int a, b;
2080 
2081                 /* Are there nodes at this distance from each other? */
2082                 if (!find_numa_distance(dist))
2083                         continue;
2084 
2085                 for_each_node_mask(a, nodes) {
2086                         unsigned long faults = 0;
2087                         nodemask_t this_group;
2088                         nodes_clear(this_group);
2089 
2090                         /* Sum group's NUMA faults; includes a==b case. */
2091                         for_each_node_mask(b, nodes) {
2092                                 if (node_distance(a, b) < dist) {
2093                                         faults += group_faults(p, b);
2094                                         node_set(b, this_group);
2095                                         node_clear(b, nodes);
2096                                 }
2097                         }
2098 
2099                         /* Remember the top group. */
2100                         if (faults > max_faults) {
2101                                 max_faults = faults;
2102                                 max_group = this_group;
2103                                 /*
2104                                  * subtle: at the smallest distance there is
2105                                  * just one node left in each "group", the
2106                                  * winner is the preferred nid.
2107                                  */
2108                                 nid = a;
2109                         }
2110                 }
2111                 /* Next round, evaluate the nodes within max_group. */
2112                 if (!max_faults)
2113                         break;
2114                 nodes = max_group;
2115         }
2116         return nid;
2117 }
2118 
2119 static void task_numa_placement(struct task_struct *p)
2120 {
2121         int seq, nid, max_nid = -1, max_group_nid = -1;
2122         unsigned long max_faults = 0, max_group_faults = 0;
2123         unsigned long fault_types[2] = { 0, 0 };
2124         unsigned long total_faults;
2125         u64 runtime, period;
2126         spinlock_t *group_lock = NULL;
2127 
2128         /*
2129          * The p->mm->numa_scan_seq field gets updated without
2130          * exclusive access. Use READ_ONCE() here to ensure
2131          * that the field is read in a single access:
2132          */
2133         seq = READ_ONCE(p->mm->numa_scan_seq);
2134         if (p->numa_scan_seq == seq)
2135                 return;
2136         p->numa_scan_seq = seq;
2137         p->numa_scan_period_max = task_scan_max(p);
2138 
2139         total_faults = p->numa_faults_locality[0] +
2140                        p->numa_faults_locality[1];
2141         runtime = numa_get_avg_runtime(p, &period);
2142 
2143         /* If the task is part of a group prevent parallel updates to group stats */
2144         if (p->numa_group) {
2145                 group_lock = &p->numa_group->lock;
2146                 spin_lock_irq(group_lock);
2147         }
2148 
2149         /* Find the node with the highest number of faults */
2150         for_each_online_node(nid) {
2151                 /* Keep track of the offsets in numa_faults array */
2152                 int mem_idx, membuf_idx, cpu_idx, cpubuf_idx;
2153                 unsigned long faults = 0, group_faults = 0;
2154                 int priv;
2155 
2156                 for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) {
2157                         long diff, f_diff, f_weight;
2158 
2159                         mem_idx = task_faults_idx(NUMA_MEM, nid, priv);
2160                         membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv);
2161                         cpu_idx = task_faults_idx(NUMA_CPU, nid, priv);
2162                         cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv);
2163 
2164                         /* Decay existing window, copy faults since last scan */
2165                         diff = p->numa_faults[membuf_idx] - p->numa_faults[mem_idx] / 2;
2166                         fault_types[priv] += p->numa_faults[membuf_idx];
2167                         p->numa_faults[membuf_idx] = 0;
2168 
2169                         /*
2170                          * Normalize the faults_from, so all tasks in a group
2171                          * count according to CPU use, instead of by the raw
2172                          * number of faults. Tasks with little runtime have
2173                          * little over-all impact on throughput, and thus their
2174                          * faults are less important.
2175                          */
2176                         f_weight = div64_u64(runtime << 16, period + 1);
2177                         f_weight = (f_weight * p->numa_faults[cpubuf_idx]) /
2178                                    (total_faults + 1);
2179                         f_diff = f_weight - p->numa_faults[cpu_idx] / 2;
2180                         p->numa_faults[cpubuf_idx] = 0;
2181 
2182                         p->numa_faults[mem_idx] += diff;
2183                         p->numa_faults[cpu_idx] += f_diff;
2184                         faults += p->numa_faults[mem_idx];
2185                         p->total_numa_faults += diff;
2186                         if (p->numa_group) {
2187                                 /*
2188                                  * safe because we can only change our own group
2189                                  *
2190                                  * mem_idx represents the offset for a given
2191                                  * nid and priv in a specific region because it
2192                                  * is at the beginning of the numa_faults array.
2193                                  */
2194                                 p->numa_group->faults[mem_idx] += diff;
2195                                 p->numa_group->faults_cpu[mem_idx] += f_diff;
2196                                 p->numa_group->total_faults += diff;
2197                                 group_faults += p->numa_group->faults[mem_idx];
2198                         }
2199                 }
2200 
2201                 if (faults > max_faults) {
2202                         max_faults = faults;
2203                         max_nid = nid;
2204                 }
2205 
2206                 if (group_faults > max_group_faults) {
2207                         max_group_faults = group_faults;
2208                         max_group_nid = nid;
2209                 }
2210         }
2211 
2212         update_task_scan_period(p, fault_types[0], fault_types[1]);
2213 
2214         if (p->numa_group) {
2215                 numa_group_count_active_nodes(p->numa_group);
2216                 spin_unlock_irq(group_lock);
2217                 max_nid = preferred_group_nid(p, max_group_nid);
2218         }
2219 
2220         if (max_faults) {
2221                 /* Set the new preferred node */
2222                 if (max_nid != p->numa_preferred_nid)
2223                         sched_setnuma(p, max_nid);
2224 
2225                 if (task_node(p) != p->numa_preferred_nid)
2226                         numa_migrate_preferred(p);
2227         }
2228 }
2229 
2230 static inline int get_numa_group(struct numa_group *grp)
2231 {
2232         return atomic_inc_not_zero(&grp->refcount);
2233 }
2234 
2235 static inline void put_numa_group(struct numa_group *grp)
2236 {
2237         if (atomic_dec_and_test(&grp->refcount))
2238                 kfree_rcu(grp, rcu);
2239 }
2240 
2241 static void task_numa_group(struct task_struct *p, int cpupid, int flags,
2242                         int *priv)
2243 {
2244         struct numa_group *grp, *my_grp;
2245         struct task_struct *tsk;
2246         bool join = false;
2247         int cpu = cpupid_to_cpu(cpupid);
2248         int i;
2249 
2250         if (unlikely(!p->numa_group)) {
2251                 unsigned int size = sizeof(struct numa_group) +
2252                                     4*nr_node_ids*sizeof(unsigned long);
2253 
2254                 grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
2255                 if (!grp)
2256                         return;
2257 
2258                 atomic_set(&grp->refcount, 1);
2259                 grp->active_nodes = 1;
2260                 grp->max_faults_cpu = 0;
2261                 spin_lock_init(&grp->lock);
2262                 grp->gid = p->pid;
2263                 /* Second half of the array tracks nids where faults happen */
2264                 grp->faults_cpu = grp->faults + NR_NUMA_HINT_FAULT_TYPES *
2265                                                 nr_node_ids;
2266 
2267                 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++)
2268                         grp->faults[i] = p->numa_faults[i];
2269 
2270                 grp->total_faults = p->total_numa_faults;
2271 
2272                 grp->nr_tasks++;
2273                 rcu_assign_pointer(p->numa_group, grp);
2274         }
2275 
2276         rcu_read_lock();
2277         tsk = READ_ONCE(cpu_rq(cpu)->curr);
2278 
2279         if (!cpupid_match_pid(tsk, cpupid))
2280                 goto no_join;
2281 
2282         grp = rcu_dereference(tsk->numa_group);
2283         if (!grp)
2284                 goto no_join;
2285 
2286         my_grp = p->numa_group;
2287         if (grp == my_grp)
2288                 goto no_join;
2289 
2290         /*
2291          * Only join the other group if its bigger; if we're the bigger group,
2292          * the other task will join us.
2293          */
2294         if (my_grp->nr_tasks > grp->nr_tasks)
2295                 goto no_join;
2296 
2297         /*
2298          * Tie-break on the grp address.
2299          */
2300         if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp)
2301                 goto no_join;
2302 
2303         /* Always join threads in the same process. */
2304         if (tsk->mm == current->mm)
2305                 join = true;
2306 
2307         /* Simple filter to avoid false positives due to PID collisions */
2308         if (flags & TNF_SHARED)
2309                 join = true;
2310 
2311         /* Update priv based on whether false sharing was detected */
2312         *priv = !join;
2313 
2314         if (join && !get_numa_group(grp))
2315                 goto no_join;
2316 
2317         rcu_read_unlock();
2318 
2319         if (!join)
2320                 return;
2321 
2322         BUG_ON(irqs_disabled());
2323         double_lock_irq(&my_grp->lock, &grp->lock);
2324 
2325         for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) {
2326                 my_grp->faults[i] -= p->numa_faults[i];
2327                 grp->faults[i] += p->numa_faults[i];
2328         }
2329         my_grp->total_faults -= p->total_numa_faults;
2330         grp->total_faults += p->total_numa_faults;
2331 
2332         my_grp->nr_tasks--;
2333         grp->nr_tasks++;
2334 
2335         spin_unlock(&my_grp->lock);
2336         spin_unlock_irq(&grp->lock);
2337 
2338         rcu_assign_pointer(p->numa_group, grp);
2339 
2340         put_numa_group(my_grp);
2341         return;
2342 
2343 no_join:
2344         rcu_read_unlock();
2345         return;
2346 }
2347 
2348 void task_numa_free(struct task_struct *p)
2349 {
2350         struct numa_group *grp = p->numa_group;
2351         void *numa_faults = p->numa_faults;
2352         unsigned long flags;
2353         int i;
2354 
2355         if (grp) {
2356                 spin_lock_irqsave(&grp->lock, flags);
2357                 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++)
2358                         grp->faults[i] -= p->numa_faults[i];
2359                 grp->total_faults -= p->total_numa_faults;
2360 
2361                 grp->nr_tasks--;
2362                 spin_unlock_irqrestore(&grp->lock, flags);
2363                 RCU_INIT_POINTER(p->numa_group, NULL);
2364                 put_numa_group(grp);
2365         }
2366 
2367         p->numa_faults = NULL;
2368         kfree(numa_faults);
2369 }
2370 
2371 /*
2372  * Got a PROT_NONE fault for a page on @node.
2373  */
2374 void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags)
2375 {
2376         struct task_struct *p = current;
2377         bool migrated = flags & TNF_MIGRATED;
2378         int cpu_node = task_node(current);
2379         int local = !!(flags & TNF_FAULT_LOCAL);
2380         struct numa_group *ng;
2381         int priv;
2382 
2383         if (!static_branch_likely(&sched_numa_balancing))
2384                 return;
2385 
2386         /* for example, ksmd faulting in a user's mm */
2387         if (!p->mm)
2388                 return;
2389 
2390         /* Allocate buffer to track faults on a per-node basis */
2391         if (unlikely(!p->numa_faults)) {
2392                 int size = sizeof(*p->numa_faults) *
2393                            NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids;
2394 
2395                 p->numa_faults = kzalloc(size, GFP_KERNEL|__GFP_NOWARN);
2396                 if (!p->numa_faults)
2397                         return;
2398 
2399                 p->total_numa_faults = 0;
2400                 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality));
2401         }
2402 
2403         /*
2404          * First accesses are treated as private, otherwise consider accesses
2405          * to be private if the accessing pid has not changed
2406          */
2407         if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) {
2408                 priv = 1;
2409         } else {
2410                 priv = cpupid_match_pid(p, last_cpupid);
2411                 if (!priv && !(flags & TNF_NO_GROUP))
2412                         task_numa_group(p, last_cpupid, flags, &priv);
2413         }
2414 
2415         /*
2416          * If a workload spans multiple NUMA nodes, a shared fault that
2417          * occurs wholly within the set of nodes that the workload is
2418          * actively using should be counted as local. This allows the
2419          * scan rate to slow down when a workload has settled down.
2420          */
2421         ng = p->numa_group;
2422         if (!priv && !local && ng && ng->active_nodes > 1 &&
2423                                 numa_is_active_node(cpu_node, ng) &&
2424                                 numa_is_active_node(mem_node, ng))
2425                 local = 1;
2426 
2427         task_numa_placement(p);
2428 
2429         /*
2430          * Retry task to preferred node migration periodically, in case it
2431          * case it previously failed, or the scheduler moved us.
2432          */
2433         if (time_after(jiffies, p->numa_migrate_retry))
2434                 numa_migrate_preferred(p);
2435 
2436         if (migrated)
2437                 p->numa_pages_migrated += pages;
2438         if (flags & TNF_MIGRATE_FAIL)
2439                 p->numa_faults_locality[2] += pages;
2440 
2441         p->numa_faults[task_faults_idx(NUMA_MEMBUF, mem_node, priv)] += pages;
2442         p->numa_faults[task_faults_idx(NUMA_CPUBUF, cpu_node, priv)] += pages;
2443         p->numa_faults_locality[local] += pages;
2444 }
2445 
2446 static void reset_ptenuma_scan(struct task_struct *p)
2447 {
2448         /*
2449          * We only did a read acquisition of the mmap sem, so
2450          * p->mm->numa_scan_seq is written to without exclusive access
2451          * and the update is not guaranteed to be atomic. That's not
2452          * much of an issue though, since this is just used for
2453          * statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not
2454          * expensive, to avoid any form of compiler optimizations:
2455          */
2456         WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1);
2457         p->mm->numa_scan_offset = 0;
2458 }
2459 
2460 /*
2461  * The expensive part of numa migration is done from task_work context.
2462  * Triggered from task_tick_numa().
2463  */
2464 void task_numa_work(struct callback_head *work)
2465 {
2466         unsigned long migrate, next_scan, now = jiffies;
2467         struct task_struct *p = current;
2468         struct mm_struct *mm = p->mm;
2469         u64 runtime = p->se.sum_exec_runtime;
2470         struct vm_area_struct *vma;
2471         unsigned long start, end;
2472         unsigned long nr_pte_updates = 0;
2473         long pages, virtpages;
2474 
2475         SCHED_WARN_ON(p != container_of(work, struct task_struct, numa_work));
2476 
2477         work->next = work; /* protect against double add */
2478         /*
2479          * Who cares about NUMA placement when they're dying.
2480          *
2481          * NOTE: make sure not to dereference p->mm before this check,
2482          * exit_task_work() happens _after_ exit_mm() so we could be called
2483          * without p->mm even though we still had it when we enqueued this
2484          * work.
2485          */
2486         if (p->flags & PF_EXITING)
2487                 return;
2488 
2489         if (!mm->numa_next_scan) {
2490                 mm->numa_next_scan = now +
2491                         msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
2492         }
2493 
2494         /*
2495          * Enforce maximal scan/migration frequency..
2496          */
2497         migrate = mm->numa_next_scan;
2498         if (time_before(now, migrate))
2499                 return;
2500 
2501         if (p->numa_scan_period == 0) {
2502                 p->numa_scan_period_max = task_scan_max(p);
2503                 p->numa_scan_period = task_scan_start(p);
2504         }
2505 
2506         next_scan = now + msecs_to_jiffies(p->numa_scan_period);
2507         if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate)
2508                 return;
2509 
2510         /*
2511          * Delay this task enough that another task of this mm will likely win
2512          * the next time around.
2513          */
2514         p->node_stamp += 2 * TICK_NSEC;
2515 
2516         start = mm->numa_scan_offset;
2517         pages = sysctl_numa_balancing_scan_size;
2518         pages <<= 20 - PAGE_SHIFT; /* MB in pages */
2519         virtpages = pages * 8;     /* Scan up to this much virtual space */
2520         if (!pages)
2521                 return;
2522 
2523 
2524         if (!down_read_trylock(&mm->mmap_sem))
2525                 return;
2526         vma = find_vma(mm, start);
2527         if (!vma) {
2528                 reset_ptenuma_scan(p);
2529                 start = 0;
2530                 vma = mm->mmap;
2531         }
2532         for (; vma; vma = vma->vm_next) {
2533                 if (!vma_migratable(vma) || !vma_policy_mof(vma) ||
2534                         is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) {
2535                         continue;
2536                 }
2537 
2538                 /*
2539                  * Shared library pages mapped by multiple processes are not
2540                  * migrated as it is expected they are cache replicated. Avoid
2541                  * hinting faults in read-only file-backed mappings or the vdso
2542                  * as migrating the pages will be of marginal benefit.
2543                  */
2544                 if (!vma->vm_mm ||
2545                     (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ)))
2546                         continue;
2547 
2548                 /*
2549                  * Skip inaccessible VMAs to avoid any confusion between
2550                  * PROT_NONE and NUMA hinting ptes
2551                  */
2552                 if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))
2553                         continue;
2554 
2555                 do {
2556                         start = max(start, vma->vm_start);
2557                         end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE);
2558                         end = min(end, vma->vm_end);
2559                         nr_pte_updates = change_prot_numa(vma, start, end);
2560 
2561                         /*
2562                          * Try to scan sysctl_numa_balancing_size worth of
2563                          * hpages that have at least one present PTE that
2564                          * is not already pte-numa. If the VMA contains
2565                          * areas that are unused or already full of prot_numa
2566                          * PTEs, scan up to virtpages, to skip through those
2567                          * areas faster.
2568                          */
2569                         if (nr_pte_updates)
2570                                 pages -= (end - start) >> PAGE_SHIFT;
2571                         virtpages -= (end - start) >> PAGE_SHIFT;
2572 
2573                         start = end;
2574                         if (pages <= 0 || virtpages <= 0)
2575                                 goto out;
2576 
2577                         cond_resched();
2578                 } while (end != vma->vm_end);
2579         }
2580 
2581 out:
2582         /*
2583          * It is possible to reach the end of the VMA list but the last few
2584          * VMAs are not guaranteed to the vma_migratable. If they are not, we
2585          * would find the !migratable VMA on the next scan but not reset the
2586          * scanner to the start so check it now.
2587          */
2588         if (vma)
2589                 mm->numa_scan_offset = start;
2590         else
2591                 reset_ptenuma_scan(p);
2592         up_read(&mm->mmap_sem);
2593 
2594         /*
2595          * Make sure tasks use at least 32x as much time to run other code
2596          * than they used here, to limit NUMA PTE scanning overhead to 3% max.
2597          * Usually update_task_scan_period slows down scanning enough; on an
2598          * overloaded system we need to limit overhead on a per task basis.
2599          */
2600         if (unlikely(p->se.sum_exec_runtime != runtime)) {
2601                 u64 diff = p->se.sum_exec_runtime - runtime;
2602                 p->node_stamp += 32 * diff;
2603         }
2604 }
2605 
2606 /*
2607  * Drive the periodic memory faults..
2608  */
2609 void task_tick_numa(struct rq *rq, struct task_struct *curr)
2610 {
2611         struct callback_head *work = &curr->numa_work;
2612         u64 period, now;
2613 
2614         /*
2615          * We don't care about NUMA placement if we don't have memory.
2616          */
2617         if (!curr->mm || (curr->flags & PF_EXITING) || work->next != work)
2618                 return;
2619 
2620         /*
2621          * Using runtime rather than walltime has the dual advantage that
2622          * we (mostly) drive the selection from busy threads and that the
2623          * task needs to have done some actual work before we bother with
2624          * NUMA placement.
2625          */
2626         now = curr->se.sum_exec_runtime;
2627         period = (u64)curr->numa_scan_period * NSEC_PER_MSEC;
2628 
2629         if (now > curr->node_stamp + period) {
2630                 if (!curr->node_stamp)
2631                         curr->numa_scan_period = task_scan_start(curr);
2632                 curr->node_stamp += period;
2633 
2634                 if (!time_before(jiffies, curr->mm->numa_next_scan)) {
2635                         init_task_work(work, task_numa_work); /* TODO: move this into sched_fork() */
2636                         task_work_add(curr, work, true);
2637                 }
2638         }
2639 }
2640 
2641 #else
2642 static void task_tick_numa(struct rq *rq, struct task_struct *curr)
2643 {
2644 }
2645 
2646 static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p)
2647 {
2648 }
2649 
2650 static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p)
2651 {
2652 }
2653 
2654 #endif /* CONFIG_NUMA_BALANCING */
2655 
2656 static void
2657 account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
2658 {
2659         update_load_add(&cfs_rq->load, se->load.weight);
2660         if (!parent_entity(se))
2661                 update_load_add(&rq_of(cfs_rq)->load, se->load.weight);
2662 #ifdef CONFIG_SMP
2663         if (entity_is_task(se)) {
2664                 struct rq *rq = rq_of(cfs_rq);
2665 
2666                 account_numa_enqueue(rq, task_of(se));
2667                 list_add(&se->group_node, &rq->cfs_tasks);
2668         }
2669 #endif
2670         cfs_rq->nr_running++;
2671 }
2672 
2673 static void
2674 account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
2675 {
2676         update_load_sub(&cfs_rq->load, se->load.weight);
2677         if (!parent_entity(se))
2678                 update_load_sub(&rq_of(cfs_rq)->load, se->load.weight);
2679 #ifdef CONFIG_SMP
2680         if (entity_is_task(se)) {
2681                 account_numa_dequeue(rq_of(cfs_rq), task_of(se));
2682                 list_del_init(&se->group_node);
2683         }
2684 #endif
2685         cfs_rq->nr_running--;
2686 }
2687 
2688 /*
2689  * Signed add and clamp on underflow.
2690  *
2691  * Explicitly do a load-store to ensure the intermediate value never hits
2692  * memory. This allows lockless observations without ever seeing the negative
2693  * values.
2694  */
2695 #define add_positive(_ptr, _val) do {                           \
2696         typeof(_ptr) ptr = (_ptr);                              \
2697         typeof(_val) val = (_val);                              \
2698         typeof(*ptr) res, var = READ_ONCE(*ptr);                \
2699                                                                 \
2700         res = var + val;                                        \
2701                                                                 \
2702         if (val < 0 && res > var)                               \
2703                 res = 0;                                        \
2704                                                                 \
2705         WRITE_ONCE(*ptr, res);                                  \
2706 } while (0)
2707 
2708 /*
2709  * Unsigned subtract and clamp on underflow.
2710  *
2711  * Explicitly do a load-store to ensure the intermediate value never hits
2712  * memory. This allows lockless observations without ever seeing the negative
2713  * values.
2714  */
2715 #define sub_positive(_ptr, _val) do {                           \
2716         typeof(_ptr) ptr = (_ptr);                              \
2717         typeof(*ptr) val = (_val);                              \
2718         typeof(*ptr) res, var = READ_ONCE(*ptr);                \
2719         res = var - val;                                        \
2720         if (res > var)                                          \
2721                 res = 0;                                        \
2722         WRITE_ONCE(*ptr, res);                                  \
2723 } while (0)
2724 
2725 #ifdef CONFIG_SMP
2726 /*
2727  * XXX we want to get rid of these helpers and use the full load resolution.
2728  */
2729 static inline long se_weight(struct sched_entity *se)
2730 {
2731         return scale_load_down(se->load.weight);
2732 }
2733 
2734 static inline long se_runnable(struct sched_entity *se)
2735 {
2736         return scale_load_down(se->runnable_weight);
2737 }
2738 
2739 static inline void
2740 enqueue_runnable_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
2741 {
2742         cfs_rq->runnable_weight += se->runnable_weight;
2743 
2744         cfs_rq->avg.runnable_load_avg += se->avg.runnable_load_avg;
2745         cfs_rq->avg.runnable_load_sum += se_runnable(se) * se->avg.runnable_load_sum;
2746 }
2747 
2748 static inline void
2749 dequeue_runnable_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
2750 {
2751         cfs_rq->runnable_weight -= se->runnable_weight;
2752 
2753         sub_positive(&cfs_rq->avg.runnable_load_avg, se->avg.runnable_load_avg);
2754         sub_positive(&cfs_rq->avg.runnable_load_sum,
2755                      se_runnable(se) * se->avg.runnable_load_sum);
2756 }
2757 
2758 static inline void
2759 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
2760 {
2761         cfs_rq->avg.load_avg += se->avg.load_avg;
2762         cfs_rq->avg.load_sum += se_weight(se) * se->avg.load_sum;
2763 }
2764 
2765 static inline void
2766 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
2767 {
2768         sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg);
2769         sub_positive(&cfs_rq->avg.load_sum, se_weight(se) * se->avg.load_sum);
2770 }
2771 #else
2772 static inline void
2773 enqueue_runnable_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { }
2774 static inline void
2775 dequeue_runnable_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { }
2776 static inline void
2777 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { }
2778 static inline void
2779 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { }
2780 #endif
2781 
2782 static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,
2783                             unsigned long weight, unsigned long runnable)
2784 {
2785         if (se->on_rq) {
2786                 /* commit outstanding execution time */
2787                 if (cfs_rq->curr == se)
2788                         update_curr(cfs_rq);
2789                 account_entity_dequeue(cfs_rq, se);
2790                 dequeue_runnable_load_avg(cfs_rq, se);
2791         }
2792         dequeue_load_avg(cfs_rq, se);
2793 
2794         se->runnable_weight = runnable;
2795         update_load_set(&se->load, weight);
2796 
2797 #ifdef CONFIG_SMP
2798         do {
2799                 u32 divider = LOAD_AVG_MAX - 1024 + se->avg.period_contrib;
2800 
2801                 se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider);
2802                 se->avg.runnable_load_avg =
2803                         div_u64(se_runnable(se) * se->avg.runnable_load_sum, divider);
2804         } while (0);
2805 #endif
2806 
2807         enqueue_load_avg(cfs_rq, se);
2808         if (se->on_rq) {
2809                 account_entity_enqueue(cfs_rq, se);
2810                 enqueue_runnable_load_avg(cfs_rq, se);
2811         }
2812 }
2813 
2814 void reweight_task(struct task_struct *p, int prio)
2815 {
2816         struct sched_entity *se = &p->se;
2817         struct cfs_rq *cfs_rq = cfs_rq_of(se);
2818         struct load_weight *load = &se->load;
2819         unsigned long weight = scale_load(sched_prio_to_weight[prio]);
2820 
2821         reweight_entity(cfs_rq, se, weight, weight);
2822         load->inv_weight = sched_prio_to_wmult[prio];
2823 }
2824 
2825 #ifdef CONFIG_FAIR_GROUP_SCHED
2826 # ifdef CONFIG_SMP
2827 /*
2828  * All this does is approximate the hierarchical proportion which includes that
2829  * global sum we all love to hate.
2830  *
2831  * That is, the weight of a group entity, is the proportional share of the
2832  * group weight based on the group runqueue weights. That is:
2833  *
2834  *                     tg->weight * grq->load.weight
2835  *   ge->load.weight = -----------------------------               (1)
2836  *                        \Sum grq->load.weight
2837  *
2838  * Now, because computing that sum is prohibitively expensive to compute (been
2839  * there, done that) we approximate it with this average stuff. The average
2840  * moves slower and therefore the approximation is cheaper and more stable.
2841  *
2842  * So instead of the above, we substitute:
2843  *
2844  *   grq->load.weight -> grq->avg.load_avg                         (2)
2845  *
2846  * which yields the following:
2847  *
2848  *                     tg->weight * grq->avg.load_avg
2849  *   ge->load.weight = ------------------------------              (3)
2850  *                              tg->load_avg
2851  *
2852  * Where: tg->load_avg ~= \Sum grq->avg.load_avg
2853  *
2854  * That is shares_avg, and it is right (given the approximation (2)).
2855  *
2856  * The problem with it is that because the average is slow -- it was designed
2857  * to be exactly that of course -- this leads to transients in boundary
2858  * conditions. In specific, the case where the group was idle and we start the
2859  * one task. It takes time for our CPU's grq->avg.load_avg to build up,
2860  * yielding bad latency etc..
2861  *
2862  * Now, in that special case (1) reduces to:
2863  *
2864  *                     tg->weight * grq->load.weight
2865  *   ge->load.weight = ----------------------------- = tg->weight   (4)
2866  *                          grp->load.weight
2867  *
2868  * That is, the sum collapses because all other CPUs are idle; the UP scenario.
2869  *
2870  * So what we do is modify our approximation (3) to approach (4) in the (near)
2871  * UP case, like:
2872  *
2873  *   ge->load.weight =
2874  *
2875  *              tg->weight * grq->load.weight
2876  *     ---------------------------------------------------         (5)
2877  *     tg->load_avg - grq->avg.load_avg + grq->load.weight
2878  *
2879  * But because grq->load.weight can drop to 0, resulting in a divide by zero,
2880  * we need to use grq->avg.load_avg as its lower bound, which then gives:
2881  *
2882  *
2883  *                     tg->weight * grq->load.weight
2884  *   ge->load.weight = -----------------------------               (6)
2885  *                              tg_load_avg'
2886  *
2887  * Where:
2888  *
2889  *   tg_load_avg' = tg->load_avg - grq->avg.load_avg +
2890  *                  max(grq->load.weight, grq->avg.load_avg)
2891  *
2892  * And that is shares_weight and is icky. In the (near) UP case it approaches
2893  * (4) while in the normal case it approaches (3). It consistently
2894  * overestimates the ge->load.weight and therefore:
2895  *
2896  *   \Sum ge->load.weight >= tg->weight
2897  *
2898  * hence icky!
2899  */
2900 static long calc_group_shares(struct cfs_rq *cfs_rq)
2901 {
2902         long tg_weight, tg_shares, load, shares;
2903         struct task_group *tg = cfs_rq->tg;
2904 
2905         tg_shares = READ_ONCE(tg->shares);
2906 
2907         load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg);
2908 
2909         tg_weight = atomic_long_read(&tg->load_avg);
2910 
2911         /* Ensure tg_weight >= load */
2912         tg_weight -= cfs_rq->tg_load_avg_contrib;
2913         tg_weight += load;
2914 
2915         shares = (tg_shares * load);
2916         if (tg_weight)
2917                 shares /= tg_weight;
2918 
2919         /*
2920          * MIN_SHARES has to be unscaled here to support per-CPU partitioning
2921          * of a group with small tg->shares value. It is a floor value which is
2922          * assigned as a minimum load.weight to the sched_entity representing
2923          * the group on a CPU.
2924          *
2925          * E.g. on 64-bit for a group with tg->shares of scale_load(15)=15*1024
2926          * on an 8-core system with 8 tasks each runnable on one CPU shares has
2927          * to be 15*1024*1/8=1920 instead of scale_load(MIN_SHARES)=2*1024. In
2928          * case no task is runnable on a CPU MIN_SHARES=2 should be returned
2929          * instead of 0.
2930          */
2931         return clamp_t(long, shares, MIN_SHARES, tg_shares);
2932 }
2933 
2934 /*
2935  * This calculates the effective runnable weight for a group entity based on
2936  * the group entity weight calculated above.
2937  *
2938  * Because of the above approximation (2), our group entity weight is
2939  * an load_avg based ratio (3). This means that it includes blocked load and
2940  * does not represent the runnable weight.
2941  *
2942  * Approximate the group entity's runnable weight per ratio from the group
2943  * runqueue:
2944  *
2945  *                                           grq->avg.runnable_load_avg
2946  *   ge->runnable_weight = ge->load.weight * -------------------------- (7)
2947  *                                               grq->avg.load_avg
2948  *
2949  * However, analogous to above, since the avg numbers are slow, this leads to
2950  * transients in the from-idle case. Instead we use:
2951  *
2952  *   ge->runnable_weight = ge->load.weight *
2953  *
2954  *              max(grq->avg.runnable_load_avg, grq->runnable_weight)
2955  *              -----------------------------------------------------   (8)
2956  *                    max(grq->avg.load_avg, grq->load.weight)
2957  *
2958  * Where these max() serve both to use the 'instant' values to fix the slow
2959  * from-idle and avoid the /0 on to-idle, similar to (6).
2960  */
2961 static long calc_group_runnable(struct cfs_rq *cfs_rq, long shares)
2962 {
2963         long runnable, load_avg;
2964 
2965         load_avg = max(cfs_rq->avg.load_avg,
2966                        scale_load_down(cfs_rq->load.weight));
2967 
2968         runnable = max(cfs_rq->avg.runnable_load_avg,
2969                        scale_load_down(cfs_rq->runnable_weight));
2970 
2971         runnable *= shares;
2972         if (load_avg)
2973                 runnable /= load_avg;
2974 
2975         return clamp_t(long, runnable, MIN_SHARES, shares);
2976 }
2977 # endif /* CONFIG_SMP */
2978 
2979 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq);
2980 
2981 /*
2982  * Recomputes the group entity based on the current state of its group
2983  * runqueue.
2984  */
2985 static void update_cfs_group(struct sched_entity *se)
2986 {
2987         struct cfs_rq *gcfs_rq = group_cfs_rq(se);
2988         long shares, runnable;
2989 
2990         if (!gcfs_rq)
2991                 return;
2992 
2993         if (throttled_hierarchy(gcfs_rq))
2994                 return;
2995 
2996 #ifndef CONFIG_SMP
2997         runnable = shares = READ_ONCE(gcfs_rq->tg->shares);
2998 
2999         if (likely(se->load.weight == shares))
3000                 return;
3001 #else
3002         shares   = calc_group_shares(gcfs_rq);
3003         runnable = calc_group_runnable(gcfs_rq, shares);
3004 #endif
3005 
3006         reweight_entity(cfs_rq_of(se), se, shares, runnable);
3007 }
3008 
3009 #else /* CONFIG_FAIR_GROUP_SCHED */
3010 static inline void update_cfs_group(struct sched_entity *se)
3011 {
3012 }
3013 #endif /* CONFIG_FAIR_GROUP_SCHED */
3014 
3015 static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq)
3016 {
3017         struct rq *rq = rq_of(cfs_rq);
3018 
3019         if (&rq->cfs == cfs_rq) {
3020                 /*
3021                  * There are a few boundary cases this might miss but it should
3022                  * get called often enough that that should (hopefully) not be
3023                  * a real problem -- added to that it only calls on the local
3024                  * CPU, so if we enqueue remotely we'll miss an update, but
3025                  * the next tick/schedule should update.
3026                  *
3027                  * It will not get called when we go idle, because the idle
3028                  * thread is a different class (!fair), nor will the utilization
3029                  * number include things like RT tasks.
3030                  *
3031                  * As is, the util number is not freq-invariant (we'd have to
3032                  * implement arch_scale_freq_capacity() for that).
3033                  *
3034                  * See cpu_util().
3035                  */
3036                 cpufreq_update_util(rq, 0);
3037         }
3038 }
3039 
3040 #ifdef CONFIG_SMP
3041 /*
3042  * Approximate:
3043  *   val * y^n,    where y^32 ~= 0.5 (~1 scheduling period)
3044  */
3045 static u64 decay_load(u64 val, u64 n)
3046 {
3047         unsigned int local_n;
3048 
3049         if (unlikely(n > LOAD_AVG_PERIOD * 63))
3050                 return 0;
3051 
3052         /* after bounds checking we can collapse to 32-bit */
3053         local_n = n;
3054 
3055         /*
3056          * As y^PERIOD = 1/2, we can combine
3057          *    y^n = 1/2^(n/PERIOD) * y^(n%PERIOD)
3058          * With a look-up table which covers y^n (n<PERIOD)
3059          *
3060          * To achieve constant time decay_load.
3061          */
3062         if (unlikely(local_n >= LOAD_AVG_PERIOD)) {
3063                 val >>= local_n / LOAD_AVG_PERIOD;
3064                 local_n %= LOAD_AVG_PERIOD;
3065         }
3066 
3067         val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32);
3068         return val;
3069 }
3070 
3071 static u32 __accumulate_pelt_segments(u64 periods, u32 d1, u32 d3)
3072 {
3073         u32 c1, c2, c3 = d3; /* y^0 == 1 */
3074 
3075         /*
3076          * c1 = d1 y^p
3077          */
3078         c1 = decay_load((u64)d1, periods);
3079 
3080         /*
3081          *            p-1
3082          * c2 = 1024 \Sum y^n
3083          *            n=1
3084          *
3085          *              inf        inf
3086          *    = 1024 ( \Sum y^n - \Sum y^n - y^0 )
3087          *              n=0        n=p
3088          */
3089         c2 = LOAD_AVG_MAX - decay_load(LOAD_AVG_MAX, periods) - 1024;
3090 
3091         return c1 + c2 + c3;
3092 }
3093 
3094 #define cap_scale(v, s) ((v)*(s) >> SCHED_CAPACITY_SHIFT)
3095 
3096 /*
3097  * Accumulate the three separate parts of the sum; d1 the remainder
3098  * of the last (incomplete) period, d2 the span of full periods and d3
3099  * the remainder of the (incomplete) current period.
3100  *
3101  *           d1          d2           d3
3102  *           ^           ^            ^
3103  *           |           |            |
3104  *         |<->|<----------------->|<--->|
3105  * ... |---x---|------| ... |------|-----x (now)
3106  *
3107  *                           p-1
3108  * u' = (u + d1) y^p + 1024 \Sum y^n + d3 y^0
3109  *                           n=1
3110  *
3111  *    = u y^p +                                 (Step 1)
3112  *
3113  *                     p-1
3114  *      d1 y^p + 1024 \Sum y^n + d3 y^0         (Step 2)
3115  *                     n=1
3116  */
3117 static __always_inline u32
3118 accumulate_sum(u64 delta, int cpu, struct sched_avg *sa,
3119                unsigned long load, unsigned long runnable, int running)
3120 {
3121         unsigned long scale_freq, scale_cpu;
3122         u32 contrib = (u32)delta; /* p == 0 -> delta < 1024 */
3123         u64 periods;
3124 
3125         scale_freq = arch_scale_freq_capacity(NULL, cpu);
3126         scale_cpu = arch_scale_cpu_capacity(NULL, cpu);
3127 
3128         delta += sa->period_contrib;
3129         periods = delta / 1024; /* A period is 1024us (~1ms) */
3130 
3131         /*
3132          * Step 1: decay old *_sum if we crossed period boundaries.
3133          */
3134         if (periods) {
3135                 sa->load_sum = decay_load(sa->load_sum, periods);
3136                 sa->runnable_load_sum =
3137                         decay_load(sa->runnable_load_sum, periods);
3138                 sa->util_sum = decay_load((u64)(sa->util_sum), periods);
3139 
3140                 /*
3141                  * Step 2
3142                  */
3143                 delta %= 1024;
3144                 contrib = __accumulate_pelt_segments(periods,
3145                                 1024 - sa->period_contrib, delta);
3146         }
3147         sa->period_contrib = delta;
3148 
3149         contrib = cap_scale(contrib, scale_freq);
3150         if (load)
3151                 sa->load_sum += load * contrib;
3152         if (runnable)
3153                 sa->runnable_load_sum += runnable * contrib;
3154         if (running)
3155                 sa->util_sum += contrib * scale_cpu;
3156 
3157         return periods;
3158 }
3159 
3160 /*
3161  * We can represent the historical contribution to runnable average as the
3162  * coefficients of a geometric series.  To do this we sub-divide our runnable
3163  * history into segments of approximately 1ms (1024us); label the segment that
3164  * occurred N-ms ago p_N, with p_0 corresponding to the current period, e.g.
3165  *
3166  * [<- 1024us ->|<- 1024us ->|<- 1024us ->| ...
3167  *      p0            p1           p2
3168  *     (now)       (~1ms ago)  (~2ms ago)
3169  *
3170  * Let u_i denote the fraction of p_i that the entity was runnable.
3171  *
3172  * We then designate the fractions u_i as our co-efficients, yielding the
3173  * following representation of historical load:
3174  *   u_0 + u_1*y + u_2*y^2 + u_3*y^3 + ...
3175  *
3176  * We choose y based on the with of a reasonably scheduling period, fixing:
3177  *   y^32 = 0.5
3178  *
3179  * This means that the contribution to load ~32ms ago (u_32) will be weighted
3180  * approximately half as much as the contribution to load within the last ms
3181  * (u_0).
3182  *
3183  * When a period "rolls over" and we have new u_0`, multiplying the previous
3184  * sum again by y is sufficient to update:
3185  *   load_avg = u_0` + y*(u_0 + u_1*y + u_2*y^2 + ... )
3186  *            = u_0 + u_1*y + u_2*y^2 + ... [re-labeling u_i --> u_{i+1}]
3187  */
3188 static __always_inline int
3189 ___update_load_sum(u64 now, int cpu, struct sched_avg *sa,
3190                   unsigned long load, unsigned long runnable, int running)
3191 {
3192         u64 delta;
3193 
3194         delta = now - sa->last_update_time;
3195         /*
3196          * This should only happen when time goes backwards, which it
3197          * unfortunately does during sched clock init when we swap over to TSC.
3198          */
3199         if ((s64)delta < 0) {
3200                 sa->last_update_time = now;
3201                 return 0;
3202         }
3203 
3204         /*
3205          * Use 1024ns as the unit of measurement since it's a reasonable
3206          * approximation of 1us and fast to compute.
3207          */
3208         delta >>= 10;
3209         if (!delta)
3210                 return 0;
3211 
3212         sa->last_update_time += delta << 10;
3213 
3214         /*
3215          * running is a subset of runnable (weight) so running can't be set if
3216          * runnable is clear. But there are some corner cases where the current
3217          * se has been already dequeued but cfs_rq->curr still points to it.
3218          * This means that weight will be 0 but not running for a sched_entity
3219          * but also for a cfs_rq if the latter becomes idle. As an example,
3220          * this happens during idle_balance() which calls
3221          * update_blocked_averages()
3222          */
3223         if (!load)
3224                 runnable = running = 0;
3225 
3226         /*
3227          * Now we know we crossed measurement unit boundaries. The *_avg
3228          * accrues by two steps:
3229          *
3230          * Step 1: accumulate *_sum since last_update_time. If we haven't
3231          * crossed period boundaries, finish.
3232          */
3233         if (!accumulate_sum(delta, cpu, sa, load, runnable, running))
3234                 return 0;
3235 
3236         return 1;
3237 }
3238 
3239 static __always_inline void
3240 ___update_load_avg(struct sched_avg *sa, unsigned long load, unsigned long runnable)
3241 {
3242         u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib;
3243 
3244         /*
3245          * Step 2: update *_avg.
3246          */
3247         sa->load_avg = div_u64(load * sa->load_sum, divider);
3248         sa->runnable_load_avg = div_u64(runnable * sa->runnable_load_sum, divider);
3249         sa->util_avg = sa->util_sum / divider;
3250 }
3251 
3252 /*
3253  * sched_entity:
3254  *
3255  *   task:
3256  *     se_runnable() == se_weight()
3257  *
3258  *   group: [ see update_cfs_group() ]
3259  *     se_weight()   = tg->weight * grq->load_avg / tg->load_avg
3260  *     se_runnable() = se_weight(se) * grq->runnable_load_avg / grq->load_avg
3261  *
3262  *   load_sum := runnable_sum
3263  *   load_avg = se_weight(se) * runnable_avg
3264  *
3265  *   runnable_load_sum := runnable_sum
3266  *   runnable_load_avg = se_runnable(se) * runnable_avg
3267  *
3268  * XXX collapse load_sum and runnable_load_sum
3269  *
3270  * cfq_rs:
3271  *
3272  *   load_sum = \Sum se_weight(se) * se->avg.load_sum
3273  *   load_avg = \Sum se->avg.load_avg
3274  *
3275  *   runnable_load_sum = \Sum se_runnable(se) * se->avg.runnable_load_sum
3276  *   runnable_load_avg = \Sum se->avg.runable_load_avg
3277  */
3278 
3279 static int
3280 __update_load_avg_blocked_se(u64 now, int cpu, struct sched_entity *se)
3281 {
3282         if (entity_is_task(se))
3283                 se->runnable_weight = se->load.weight;
3284 
3285         if (___update_load_sum(now, cpu, &se->avg, 0, 0, 0)) {
3286                 ___update_load_avg(&se->avg, se_weight(se), se_runnable(se));
3287                 return 1;
3288         }
3289 
3290         return 0;
3291 }
3292 
3293 static int
3294 __update_load_avg_se(u64 now, int cpu, struct cfs_rq *cfs_rq, struct sched_entity *se)
3295 {
3296         if (entity_is_task(se))
3297                 se->runnable_weight = se->load.weight;
3298 
3299         if (___update_load_sum(now, cpu, &se->avg, !!se->on_rq, !!se->on_rq,
3300                                 cfs_rq->curr == se)) {
3301 
3302                 ___update_load_avg(&se->avg, se_weight(se), se_runnable(se));
3303                 return 1;
3304         }
3305 
3306         return 0;
3307 }
3308 
3309 static int
3310 __update_load_avg_cfs_rq(u64 now, int cpu, struct cfs_rq *cfs_rq)
3311 {
3312         if (___update_load_sum(now, cpu, &cfs_rq->avg,
3313                                 scale_load_down(cfs_rq->load.weight),
3314                                 scale_load_down(cfs_rq->runnable_weight),
3315                                 cfs_rq->curr != NULL)) {
3316 
3317                 ___update_load_avg(&cfs_rq->avg, 1, 1);
3318                 return 1;
3319         }
3320 
3321         return 0;
3322 }
3323 
3324 #ifdef CONFIG_FAIR_GROUP_SCHED
3325 /**
3326  * update_tg_load_avg - update the tg's load avg
3327  * @cfs_rq: the cfs_rq whose avg changed
3328  * @force: update regardless of how small the difference
3329  *
3330  * This function 'ensures': tg->load_avg := \Sum tg->cfs_rq[]->avg.load.
3331  * However, because tg->load_avg is a global value there are performance
3332  * considerations.
3333  *
3334  * In order to avoid having to look at the other cfs_rq's, we use a
3335  * differential update where we store the last value we propagated. This in
3336  * turn allows skipping updates if the differential is 'small'.
3337  *
3338  * Updating tg's load_avg is necessary before update_cfs_share().
3339  */
3340 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force)
3341 {
3342         long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib;
3343 
3344         /*
3345          * No need to update load_avg for root_task_group as it is not used.
3346          */
3347         if (cfs_rq->tg == &root_task_group)
3348                 return;
3349 
3350         if (force || abs(delta) > cfs_rq->tg_load_avg_contrib / 64) {
3351                 atomic_long_add(delta, &cfs_rq->tg->load_avg);
3352                 cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg;
3353         }
3354 }
3355 
3356 /*
3357  * Called within set_task_rq() right before setting a task's cpu. The
3358  * caller only guarantees p->pi_lock is held; no other assumptions,
3359  * including the state of rq->lock, should be made.
3360  */
3361 void set_task_rq_fair(struct sched_entity *se,
3362                       struct cfs_rq *prev, struct cfs_rq *next)
3363 {
3364         u64 p_last_update_time;
3365         u64 n_last_update_time;
3366 
3367         if (!sched_feat(ATTACH_AGE_LOAD))
3368                 return;
3369 
3370         /*
3371          * We are supposed to update the task to "current" time, then its up to
3372          * date and ready to go to new CPU/cfs_rq. But we have difficulty in
3373          * getting what current time is, so simply throw away the out-of-date
3374          * time. This will result in the wakee task is less decayed, but giving
3375          * the wakee more load sounds not bad.
3376          */
3377         if (!(se->avg.last_update_time && prev))
3378                 return;
3379 
3380 #ifndef CONFIG_64BIT
3381         {
3382                 u64 p_last_update_time_copy;
3383                 u64 n_last_update_time_copy;
3384 
3385                 do {
3386                         p_last_update_time_copy = prev->load_last_update_time_copy;
3387                         n_last_update_time_copy = next->load_last_update_time_copy;
3388 
3389                         smp_rmb();
3390 
3391                         p_last_update_time = prev->avg.last_update_time;
3392                         n_last_update_time = next->avg.last_update_time;
3393 
3394                 } while (p_last_update_time != p_last_update_time_copy ||
3395                          n_last_update_time != n_last_update_time_copy);
3396         }
3397 #else
3398         p_last_update_time = prev->avg.last_update_time;
3399         n_last_update_time = next->avg.last_update_time;
3400 #endif
3401         __update_load_avg_blocked_se(p_last_update_time, cpu_of(rq_of(prev)), se);
3402         se->avg.last_update_time = n_last_update_time;
3403 }
3404 
3405 
3406 /*
3407  * When on migration a sched_entity joins/leaves the PELT hierarchy, we need to
3408  * propagate its contribution. The key to this propagation is the invariant
3409  * that for each group:
3410  *
3411  *   ge->avg == grq->avg                                                (1)
3412  *
3413  * _IFF_ we look at the pure running and runnable sums. Because they
3414  * represent the very same entity, just at different points in the hierarchy.
3415  *
3416  * Per the above update_tg_cfs_util() is trivial and simply copies the running
3417  * sum over (but still wrong, because the group entity and group rq do not have
3418  * their PELT windows aligned).
3419  *
3420  * However, update_tg_cfs_runnable() is more complex. So we have:
3421  *
3422  *   ge->avg.load_avg = ge->load.weight * ge->avg.runnable_avg          (2)
3423  *
3424  * And since, like util, the runnable part should be directly transferable,
3425  * the following would _appear_ to be the straight forward approach:
3426  *
3427  *   grq->avg.load_avg = grq->load.weight * grq->avg.runnable_avg       (3)
3428  *
3429  * And per (1) we have:
3430  *
3431  *   ge->avg.runnable_avg == grq->avg.runnable_avg
3432  *
3433  * Which gives:
3434  *
3435  *                      ge->load.weight * grq->avg.load_avg
3436  *   ge->avg.load_avg = -----------------------------------             (4)
3437  *                               grq->load.weight
3438  *
3439  * Except that is wrong!
3440  *
3441  * Because while for entities historical weight is not important and we
3442  * really only care about our future and therefore can consider a pure
3443  * runnable sum, runqueues can NOT do this.
3444  *
3445  * We specifically want runqueues to have a load_avg that includes
3446  * historical weights. Those represent the blocked load, the load we expect
3447  * to (shortly) return to us. This only works by keeping the weights as
3448  * integral part of the sum. We therefore cannot decompose as per (3).
3449  *
3450  * Another reason this doesn't work is that runnable isn't a 0-sum entity.
3451  * Imagine a rq with 2 tasks that each are runnable 2/3 of the time. Then the
3452  * rq itself is runnable anywhere between 2/3 and 1 depending on how the
3453  * runnable section of these tasks overlap (or not). If they were to perfectly
3454  * align the rq as a whole would be runnable 2/3 of the time. If however we
3455  * always have at least 1 runnable task, the rq as a whole is always runnable.
3456  *
3457  * So we'll have to approximate.. :/
3458  *
3459  * Given the constraint:
3460  *
3461  *   ge->avg.running_sum <= ge->avg.runnable_sum <= LOAD_AVG_MAX
3462  *
3463  * We can construct a rule that adds runnable to a rq by assuming minimal
3464  * overlap.
3465  *
3466  * On removal, we'll assume each task is equally runnable; which yields:
3467  *
3468  *   grq->avg.runnable_sum = grq->avg.load_sum / grq->load.weight
3469  *
3470  * XXX: only do this for the part of runnable > running ?
3471  *
3472  */
3473 
3474 static inline void
3475 update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
3476 {
3477         long delta = gcfs_rq->avg.util_avg - se->avg.util_avg;
3478 
3479         /* Nothing to update */
3480         if (!delta)
3481                 return;
3482 
3483         /*
3484          * The relation between sum and avg is:
3485          *
3486          *   LOAD_AVG_MAX - 1024 + sa->period_contrib
3487          *
3488          * however, the PELT windows are not aligned between grq and gse.
3489          */
3490 
3491         /* Set new sched_entity's utilization */
3492         se->avg.util_avg = gcfs_rq->avg.util_avg;
3493         se->avg.util_sum = se->avg.util_avg * LOAD_AVG_MAX;
3494 
3495         /* Update parent cfs_rq utilization */
3496         add_positive(&cfs_rq->avg.util_avg, delta);
3497         cfs_rq->avg.util_sum = cfs_rq->avg.util_avg * LOAD_AVG_MAX;
3498 }
3499 
3500 static inline void
3501 update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
3502 {
3503         long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum;
3504         unsigned long runnable_load_avg, load_avg;
3505         u64 runnable_load_sum, load_sum = 0;
3506         s64 delta_sum;
3507 
3508         if (!runnable_sum)
3509                 return;
3510 
3511         gcfs_rq->prop_runnable_sum = 0;
3512 
3513         if (runnable_sum >= 0) {
3514                 /*
3515                  * Add runnable; clip at LOAD_AVG_MAX. Reflects that until
3516                  * the CPU is saturated running == runnable.
3517                  */
3518                 runnable_sum += se->avg.load_sum;
3519                 runnable_sum = min(runnable_sum, (long)LOAD_AVG_MAX);
3520         } else {
3521                 /*
3522                  * Estimate the new unweighted runnable_sum of the gcfs_rq by
3523                  * assuming all tasks are equally runnable.
3524                  */
3525                 if (scale_load_down(gcfs_rq->load.weight)) {
3526                         load_sum = div_s64(gcfs_rq->avg.load_sum,
3527                                 scale_load_down(gcfs_rq->load.weight));
3528                 }
3529 
3530                 /* But make sure to not inflate se's runnable */
3531                 runnable_sum = min(se->avg.load_sum, load_sum);
3532         }
3533 
3534         /*
3535          * runnable_sum can't be lower than running_sum
3536          * As running sum is scale with cpu capacity wehreas the runnable sum
3537          * is not we rescale running_sum 1st
3538          */
3539         running_sum = se->avg.util_sum /
3540                 arch_scale_cpu_capacity(NULL, cpu_of(rq_of(cfs_rq)));
3541         runnable_sum = max(runnable_sum, running_sum);
3542 
3543         load_sum = (s64)se_weight(se) * runnable_sum;
3544         load_avg = div_s64(load_sum, LOAD_AVG_MAX);
3545 
3546         delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum;
3547         delta_avg = load_avg - se->avg.load_avg;
3548 
3549         se->avg.load_sum = runnable_sum;
3550         se->avg.load_avg = load_avg;
3551         add_positive(&cfs_rq->avg.load_avg, delta_avg);
3552         add_positive(&cfs_rq->avg.load_sum, delta_sum);
3553 
3554         runnable_load_sum = (s64)se_runnable(se) * runnable_sum;
3555         runnable_load_avg = div_s64(runnable_load_sum, LOAD_AVG_MAX);
3556         delta_sum = runnable_load_sum - se_weight(se) * se->avg.runnable_load_sum;
3557         delta_avg = runnable_load_avg - se->avg.runnable_load_avg;
3558 
3559         se->avg.runnable_load_sum = runnable_sum;
3560         se->avg.runnable_load_avg = runnable_load_avg;
3561 
3562         if (se->on_rq) {
3563                 add_positive(&cfs_rq->avg.runnable_load_avg, delta_avg);
3564                 add_positive(&cfs_rq->avg.runnable_load_sum, delta_sum);
3565         }
3566 }
3567 
3568 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum)
3569 {
3570         cfs_rq->propagate = 1;
3571         cfs_rq->prop_runnable_sum += runnable_sum;
3572 }
3573 
3574 /* Update task and its cfs_rq load average */
3575 static inline int propagate_entity_load_avg(struct sched_entity *se)
3576 {
3577         struct cfs_rq *cfs_rq, *gcfs_rq;
3578 
3579         if (entity_is_task(se))
3580                 return 0;
3581 
3582         gcfs_rq = group_cfs_rq(se);
3583         if (!gcfs_rq->propagate)
3584                 return 0;
3585 
3586         gcfs_rq->propagate = 0;
3587 
3588         cfs_rq = cfs_rq_of(se);
3589 
3590         add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum);
3591 
3592         update_tg_cfs_util(cfs_rq, se, gcfs_rq);
3593         update_tg_cfs_runnable(cfs_rq, se, gcfs_rq);
3594 
3595         return 1;
3596 }
3597 
3598 /*
3599  * Check if we need to update the load and the utilization of a blocked
3600  * group_entity:
3601  */
3602 static inline bool skip_blocked_update(struct sched_entity *se)
3603 {
3604         struct cfs_rq *gcfs_rq = group_cfs_rq(se);
3605 
3606         /*
3607          * If sched_entity still have not zero load or utilization, we have to
3608          * decay it:
3609          */
3610         if (se->avg.load_avg || se->avg.util_avg)
3611                 return false;
3612 
3613         /*
3614          * If there is a pending propagation, we have to update the load and
3615          * the utilization of the sched_entity:
3616          */
3617         if (gcfs_rq->propagate)
3618                 return false;
3619 
3620         /*
3621          * Otherwise, the load and the utilization of the sched_entity is
3622          * already zero and there is no pending propagation, so it will be a
3623          * waste of time to try to decay it:
3624          */
3625         return true;
3626 }
3627 
3628 #else /* CONFIG_FAIR_GROUP_SCHED */
3629 
3630 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) {}
3631 
3632 static inline int propagate_entity_load_avg(struct sched_entity *se)
3633 {
3634         return 0;
3635 }
3636 
3637 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) {}
3638 
3639 #endif /* CONFIG_FAIR_GROUP_SCHED */
3640 
3641 /**
3642  * update_cfs_rq_load_avg - update the cfs_rq's load/util averages
3643  * @now: current time, as per cfs_rq_clock_task()
3644  * @cfs_rq: cfs_rq to update
3645  *
3646  * The cfs_rq avg is the direct sum of all its entities (blocked and runnable)
3647  * avg. The immediate corollary is that all (fair) tasks must be attached, see
3648  * post_init_entity_util_avg().
3649  *
3650  * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example.
3651  *
3652  * Returns true if the load decayed or we removed load.
3653  *
3654  * Since both these conditions indicate a changed cfs_rq->avg.load we should
3655  * call update_tg_load_avg() when this function returns true.
3656  */
3657 static inline int
3658 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
3659 {
3660         unsigned long removed_load = 0, removed_util = 0, removed_runnable_sum = 0;
3661         struct sched_avg *sa = &cfs_rq->avg;
3662         int decayed = 0;
3663 
3664         if (cfs_rq->removed.nr) {
3665                 unsigned long r;
3666                 u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib;
3667 
3668                 raw_spin_lock(&cfs_rq->removed.lock);
3669                 swap(cfs_rq->removed.util_avg, removed_util);
3670                 swap(cfs_rq->removed.load_avg, removed_load);
3671                 swap(cfs_rq->removed.runnable_sum, removed_runnable_sum);
3672                 cfs_rq->removed.nr = 0;
3673                 raw_spin_unlock(&cfs_rq->removed.lock);
3674 
3675                 r = removed_load;
3676                 sub_positive(&sa->load_avg, r);
3677                 sub_positive(&sa->load_sum, r * divider);
3678 
3679                 r = removed_util;
3680                 sub_positive(&sa->util_avg, r);
3681                 sub_positive(&sa->util_sum, r * divider);
3682 
3683                 add_tg_cfs_propagate(cfs_rq, -(long)removed_runnable_sum);
3684 
3685                 decayed = 1;
3686         }
3687 
3688         decayed |= __update_load_avg_cfs_rq(now, cpu_of(rq_of(cfs_rq)), cfs_rq);
3689 
3690 #ifndef CONFIG_64BIT
3691         smp_wmb();
3692         cfs_rq->load_last_update_time_copy = sa->last_update_time;
3693 #endif
3694 
3695         if (decayed)
3696                 cfs_rq_util_change(cfs_rq);
3697 
3698         return decayed;
3699 }
3700 
3701 /**
3702  * attach_entity_load_avg - attach this entity to its cfs_rq load avg
3703  * @cfs_rq: cfs_rq to attach to
3704  * @se: sched_entity to attach
3705  *
3706  * Must call update_cfs_rq_load_avg() before this, since we rely on
3707  * cfs_rq->avg.last_update_time being current.
3708  */
3709 static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
3710 {
3711         u32 divider = LOAD_AVG_MAX - 1024 + cfs_rq->avg.period_contrib;
3712 
3713         /*
3714          * When we attach the @se to the @cfs_rq, we must align the decay
3715          * window because without that, really weird and wonderful things can
3716          * happen.
3717          *
3718          * XXX illustrate
3719          */
3720         se->avg.last_update_time = cfs_rq->avg.last_update_time;
3721         se->avg.period_contrib = cfs_rq->avg.period_contrib;
3722 
3723         /*
3724          * Hell(o) Nasty stuff.. we need to recompute _sum based on the new
3725          * period_contrib. This isn't strictly correct, but since we're
3726          * entirely outside of the PELT hierarchy, nobody cares if we truncate
3727          * _sum a little.
3728          */
3729         se->avg.util_sum = se->avg.util_avg * divider;
3730 
3731         se->avg.load_sum = divider;
3732         if (se_weight(se)) {
3733                 se->avg.load_sum =
3734                         div_u64(se->avg.load_avg * se->avg.load_sum, se_weight(se));
3735         }
3736 
3737         se->avg.runnable_load_sum = se->avg.load_sum;
3738 
3739         enqueue_load_avg(cfs_rq, se);
3740         cfs_rq->avg.util_avg += se->avg.util_avg;
3741         cfs_rq->avg.util_sum += se->avg.util_sum;
3742 
3743         add_tg_cfs_propagate(cfs_rq, se->avg.load_sum);
3744 
3745         cfs_rq_util_change(cfs_rq);
3746 }
3747 
3748 /**
3749  * detach_entity_load_avg - detach this entity from its cfs_rq load avg
3750  * @cfs_rq: cfs_rq to detach from
3751  * @se: sched_entity to detach
3752  *
3753  * Must call update_cfs_rq_load_avg() before this, since we rely on
3754  * cfs_rq->avg.last_update_time being current.
3755  */
3756 static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
3757 {
3758         dequeue_load_avg(cfs_rq, se);
3759         sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg);
3760         sub_positive(&cfs_rq->avg.util_sum, se->avg.util_sum);
3761 
3762         add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum);
3763 
3764         cfs_rq_util_change(cfs_rq);
3765 }
3766 
3767 /*
3768  * Optional action to be done while updating the load average
3769  */
3770 #define UPDATE_TG       0x1
3771 #define SKIP_AGE_LOAD   0x2
3772 #define DO_ATTACH       0x4
3773 
3774 /* Update task and its cfs_rq load average */
3775 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
3776 {
3777         u64 now = cfs_rq_clock_task(cfs_rq);
3778         struct rq *rq = rq_of(cfs_rq);
3779         int cpu = cpu_of(rq);
3780         int decayed;
3781 
3782         /*
3783          * Track task load average for carrying it to new CPU after migrated, and
3784          * track group sched_entity load average for task_h_load calc in migration
3785          */
3786         if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD))
3787                 __update_load_avg_se(now, cpu, cfs_rq, se);
3788 
3789         decayed  = update_cfs_rq_load_avg(now, cfs_rq);
3790         decayed |= propagate_entity_load_avg(se);
3791 
3792         if (!se->avg.last_update_time && (flags & DO_ATTACH)) {
3793 
3794                 attach_entity_load_avg(cfs_rq, se);
3795                 update_tg_load_avg(cfs_rq, 0);
3796 
3797         } else if (decayed && (flags & UPDATE_TG))
3798                 update_tg_load_avg(cfs_rq, 0);
3799 }
3800 
3801 #ifndef CONFIG_64BIT
3802 static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq)
3803 {
3804         u64 last_update_time_copy;
3805         u64 last_update_time;
3806 
3807         do {
3808                 last_update_time_copy = cfs_rq->load_last_update_time_copy;
3809                 smp_rmb();
3810                 last_update_time = cfs_rq->avg.last_update_time;
3811         } while (last_update_time != last_update_time_copy);
3812 
3813         return last_update_time;
3814 }
3815 #else
3816 static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq)
3817 {
3818         return cfs_rq->avg.last_update_time;
3819 }
3820 #endif
3821 
3822 /*
3823  * Synchronize entity load avg of dequeued entity without locking
3824  * the previous rq.
3825  */
3826 void sync_entity_load_avg(struct sched_entity *se)
3827 {
3828         struct cfs_rq *cfs_rq = cfs_rq_of(se);
3829         u64 last_update_time;
3830 
3831         last_update_time = cfs_rq_last_update_time(cfs_rq);
3832         __update_load_avg_blocked_se(last_update_time, cpu_of(rq_of(cfs_rq)), se);
3833 }
3834 
3835 /*
3836  * Task first catches up with cfs_rq, and then subtract
3837  * itself from the cfs_rq (task must be off the queue now).
3838  */
3839 void remove_entity_load_avg(struct sched_entity *se)
3840 {
3841         struct cfs_rq *cfs_rq = cfs_rq_of(se);
3842         unsigned long flags;
3843 
3844         /*
3845          * tasks cannot exit without having gone through wake_up_new_task() ->
3846          * post_init_entity_util_avg() which will have added things to the
3847          * cfs_rq, so we can remove unconditionally.
3848          *
3849          * Similarly for groups, they will have passed through
3850          * post_init_entity_util_avg() before unregister_sched_fair_group()
3851          * calls this.
3852          */
3853 
3854         sync_entity_load_avg(se);
3855 
3856         raw_spin_lock_irqsave(&cfs_rq->removed.lock, flags);
3857         ++cfs_rq->removed.nr;
3858         cfs_rq->removed.util_avg        += se->avg.util_avg;
3859         cfs_rq->removed.load_avg        += se->avg.load_avg;
3860         cfs_rq->removed.runnable_sum    += se->avg.load_sum; /* == runnable_sum */
3861         raw_spin_unlock_irqrestore(&cfs_rq->removed.lock, flags);
3862 }
3863 
3864 static inline unsigned long cfs_rq_runnable_load_avg(struct cfs_rq *cfs_rq)
3865 {
3866         return cfs_rq->avg.runnable_load_avg;
3867 }
3868 
3869 static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq)
3870 {
3871         return cfs_rq->avg.load_avg;
3872 }
3873 
3874 static int idle_balance(struct rq *this_rq, struct rq_flags *rf);
3875 
3876 #else /* CONFIG_SMP */
3877 
3878 static inline int
3879 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
3880 {
3881         return 0;
3882 }
3883 
3884 #define UPDATE_TG       0x0
3885 #define SKIP_AGE_LOAD   0x0
3886 #define DO_ATTACH       0x0
3887 
3888 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int not_used1)
3889 {
3890         cfs_rq_util_change(cfs_rq);
3891 }
3892 
3893 static inline void remove_entity_load_avg(struct sched_entity *se) {}
3894 
3895 static inline void
3896 attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {}
3897 static inline void
3898 detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {}
3899 
3900 static inline int idle_balance(struct rq *rq, struct rq_flags *rf)
3901 {
3902         return 0;
3903 }
3904 
3905 #endif /* CONFIG_SMP */
3906 
3907 static void check_spread(struct cfs_rq *cfs_rq, struct sched_entity *se)
3908 {
3909 #ifdef CONFIG_SCHED_DEBUG
3910         s64 d = se->vruntime - cfs_rq->min_vruntime;
3911 
3912         if (d < 0)
3913                 d = -d;
3914 
3915         if (d > 3*sysctl_sched_latency)
3916                 schedstat_inc(cfs_rq->nr_spread_over);
3917 #endif
3918 }
3919 
3920 static void
3921 place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
3922 {
3923         u64 vruntime = cfs_rq->min_vruntime;
3924 
3925         /*
3926          * The 'current' period is already promised to the current tasks,
3927          * however the extra weight of the new task will slow them down a
3928          * little, place the new task so that it fits in the slot that
3929          * stays open at the end.
3930          */
3931         if (initial && sched_feat(START_DEBIT))
3932                 vruntime += sched_vslice(cfs_rq, se);
3933 
3934         /* sleeps up to a single latency don't count. */
3935         if (!initial) {
3936                 unsigned long thresh = sysctl_sched_latency;
3937 
3938                 /*
3939                  * Halve their sleep time's effect, to allow
3940                  * for a gentler effect of sleepers:
3941                  */
3942                 if (sched_feat(GENTLE_FAIR_SLEEPERS))
3943                         thresh >>= 1;
3944 
3945                 vruntime -= thresh;
3946         }
3947 
3948         /* ensure we never gain time by being placed backwards. */
3949         se->vruntime = max_vruntime(se->vruntime, vruntime);
3950 }
3951 
3952 static void check_enqueue_throttle(struct cfs_rq *cfs_rq);
3953 
3954 static inline void check_schedstat_required(void)
3955 {
3956 #ifdef CONFIG_SCHEDSTATS
3957         if (schedstat_enabled())
3958                 return;
3959 
3960         /* Force schedstat enabled if a dependent tracepoint is active */
3961         if (trace_sched_stat_wait_enabled()    ||
3962                         trace_sched_stat_sleep_enabled()   ||
3963                         trace_sched_stat_iowait_enabled()  ||
3964                         trace_sched_stat_blocked_enabled() ||
3965                         trace_sched_stat_runtime_enabled())  {
3966                 printk_deferred_once("Scheduler tracepoints stat_sleep, stat_iowait, "
3967                              "stat_blocked and stat_runtime require the "
3968                              "kernel parameter schedstats=enable or "
3969                              "kernel.sched_schedstats=1\n");
3970         }
3971 #endif
3972 }
3973 
3974 
3975 /*
3976  * MIGRATION
3977  *
3978  *      dequeue
3979  *        update_curr()
3980  *          update_min_vruntime()
3981  *        vruntime -= min_vruntime
3982  *
3983  *      enqueue
3984  *        update_curr()
3985  *          update_min_vruntime()
3986  *        vruntime += min_vruntime
3987  *
3988  * this way the vruntime transition between RQs is done when both
3989  * min_vruntime are up-to-date.
3990  *
3991  * WAKEUP (remote)
3992  *
3993  *      ->migrate_task_rq_fair() (p->state == TASK_WAKING)
3994  *        vruntime -= min_vruntime
3995  *
3996  *      enqueue
3997  *        update_curr()
3998  *          update_min_vruntime()
3999  *        vruntime += min_vruntime
4000  *
4001  * this way we don't have the most up-to-date min_vruntime on the originating
4002  * CPU and an up-to-date min_vruntime on the destination CPU.
4003  */
4004 
4005 static void
4006 enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
4007 {
4008         bool renorm = !(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_MIGRATED);
4009         bool curr = cfs_rq->curr == se;
4010 
4011         /*
4012          * If we're the current task, we must renormalise before calling
4013          * update_curr().
4014          */
4015         if (renorm && curr)
4016                 se->vruntime += cfs_rq->min_vruntime;
4017 
4018         update_curr(cfs_rq);
4019 
4020         /*
4021          * Otherwise, renormalise after, such that we're placed at the current
4022          * moment in time, instead of some random moment in the past. Being
4023          * placed in the past could significantly boost this task to the
4024          * fairness detriment of existing tasks.
4025          */
4026         if (renorm && !curr)
4027                 se->vruntime += cfs_rq->min_vruntime;
4028 
4029         /*
4030          * When enqueuing a sched_entity, we must:
4031          *   - Update loads to have both entity and cfs_rq synced with now.
4032          *   - Add its load to cfs_rq->runnable_avg
4033          *   - For group_entity, update its weight to reflect the new share of
4034          *     its group cfs_rq
4035          *   - Add its new weight to cfs_rq->load.weight
4036          */
4037         update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH);
4038         update_cfs_group(se);
4039         enqueue_runnable_load_avg(cfs_rq, se);
4040         account_entity_enqueue(cfs_rq, se);
4041 
4042         if (flags & ENQUEUE_WAKEUP)
4043                 place_entity(cfs_rq, se, 0);
4044 
4045         check_schedstat_required();
4046         update_stats_enqueue(cfs_rq, se, flags);
4047         check_spread(cfs_rq, se);
4048         if (!curr)
4049                 __enqueue_entity(cfs_rq, se);
4050         se->on_rq = 1;
4051 
4052         if (cfs_rq->nr_running == 1) {
4053                 list_add_leaf_cfs_rq(cfs_rq);
4054                 check_enqueue_throttle(cfs_rq);
4055         }
4056 }
4057 
4058 static void __clear_buddies_last(struct sched_entity *se)
4059 {
4060         for_each_sched_entity(se) {
4061                 struct cfs_rq *cfs_rq = cfs_rq_of(se);
4062                 if (cfs_rq->last != se)
4063                         break;
4064 
4065                 cfs_rq->last = NULL;
4066         }
4067 }
4068 
4069 static void __clear_buddies_next(struct sched_entity *se)
4070 {
4071         for_each_sched_entity(se) {
4072                 struct cfs_rq *cfs_rq = cfs_rq_of(se);
4073                 if (cfs_rq->next != se)
4074                         break;
4075 
4076                 cfs_rq->next = NULL;
4077         }
4078 }
4079 
4080 static void __clear_buddies_skip(struct sched_entity *se)
4081 {
4082         for_each_sched_entity(se) {
4083                 struct cfs_rq *cfs_rq = cfs_rq_of(se);
4084                 if (cfs_rq->skip != se)
4085                         break;
4086 
4087                 cfs_rq->skip = NULL;
4088         }
4089 }
4090 
4091 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se)
4092 {
4093         if (cfs_rq->last == se)
4094                 __clear_buddies_last(se);
4095 
4096         if (cfs_rq->next == se)
4097                 __clear_buddies_next(se);
4098 
4099         if (cfs_rq->skip == se)
4100                 __clear_buddies_skip(se);
4101 }
4102 
4103 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
4104 
4105 static void
4106 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
4107 {
4108         /*
4109          * Update run-time statistics of the 'current'.
4110          */
4111         update_curr(cfs_rq);
4112 
4113         /*
4114          * When dequeuing a sched_entity, we must:
4115          *   - Update loads to have both entity and cfs_rq synced with now.
4116          *   - Substract its load from the cfs_rq->runnable_avg.
4117          *   - Substract its previous weight from cfs_rq->load.weight.
4118          *   - For group entity, update its weight to reflect the new share
4119          *     of its group cfs_rq.
4120          */
4121         update_load_avg(cfs_rq, se, UPDATE_TG);
4122         dequeue_runnable_load_avg(cfs_rq, se);
4123 
4124         update_stats_dequeue(cfs_rq, se, flags);
4125 
4126         clear_buddies(cfs_rq, se);
4127 
4128         if (se != cfs_rq->curr)
4129                 __dequeue_entity(cfs_rq, se);
4130         se->on_rq = 0;
4131         account_entity_dequeue(cfs_rq, se);
4132 
4133         /*
4134          * Normalize after update_curr(); which will also have moved
4135          * min_vruntime if @se is the one holding it back. But before doing
4136          * update_min_vruntime() again, which will discount @se's position and
4137          * can move min_vruntime forward still more.
4138          */
4139         if (!(flags & DEQUEUE_SLEEP))
4140                 se->vruntime -= cfs_rq->min_vruntime;
4141 
4142         /* return excess runtime on last dequeue */
4143         return_cfs_rq_runtime(cfs_rq);
4144 
4145         update_cfs_group(se);
4146 
4147         /*
4148          * Now advance min_vruntime if @se was the entity holding it back,
4149          * except when: DEQUEUE_SAVE && !DEQUEUE_MOVE, in this case we'll be
4150          * put back on, and if we advance min_vruntime, we'll be placed back
4151          * further than we started -- ie. we'll be penalized.
4152          */
4153         if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) == DEQUEUE_SAVE)
4154                 update_min_vruntime(cfs_rq);
4155 }
4156 
4157 /*
4158  * Preempt the current task with a newly woken task if needed:
4159  */
4160 static void
4161 check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr)
4162 {
4163         unsigned long ideal_runtime, delta_exec;
4164         struct sched_entity *se;
4165         s64 delta;
4166 
4167         ideal_runtime = sched_slice(cfs_rq, curr);
4168         delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime;
4169         if (delta_exec > ideal_runtime) {
4170                 resched_curr(rq_of(cfs_rq));
4171                 /*
4172                  * The current task ran long enough, ensure it doesn't get
4173                  * re-elected due to buddy favours.
4174                  */
4175                 clear_buddies(cfs_rq, curr);
4176                 return;
4177         }
4178 
4179         /*
4180          * Ensure that a task that missed wakeup preemption by a
4181          * narrow margin doesn't have to wait for a full slice.
4182          * This also mitigates buddy induced latencies under load.
4183          */
4184         if (delta_exec < sysctl_sched_min_granularity)
4185                 return;
4186 
4187         se = __pick_first_entity(cfs_rq);
4188         delta = curr->vruntime - se->vruntime;
4189 
4190         if (delta < 0)
4191                 return;
4192 
4193         if (delta > ideal_runtime)
4194                 resched_curr(rq_of(cfs_rq));
4195 }
4196 
4197 static void
4198 set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
4199 {
4200         /* 'current' is not kept within the tree. */
4201         if (se->on_rq) {
4202                 /*
4203                  * Any task has to be enqueued before it get to execute on
4204                  * a CPU. So account for the time it spent waiting on the
4205                  * runqueue.
4206                  */
4207                 update_stats_wait_end(cfs_rq, se);
4208                 __dequeue_entity(cfs_rq, se);
4209                 update_load_avg(cfs_rq, se, UPDATE_TG);
4210         }
4211 
4212         update_stats_curr_start(cfs_rq, se);
4213         cfs_rq->curr = se;
4214 
4215         /*
4216          * Track our maximum slice length, if the CPU's load is at
4217          * least twice that of our own weight (i.e. dont track it
4218          * when there are only lesser-weight tasks around):
4219          */
4220         if (schedstat_enabled() && rq_of(cfs_rq)->load.weight >= 2*se->load.weight) {
4221                 schedstat_set(se->statistics.slice_max,
4222                         max((u64)schedstat_val(se->statistics.slice_max),
4223                             se->sum_exec_runtime - se->prev_sum_exec_runtime));
4224         }
4225 
4226         se->prev_sum_exec_runtime = se->sum_exec_runtime;
4227 }
4228 
4229 static int
4230 wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se);
4231 
4232 /*
4233  * Pick the next process, keeping these things in mind, in this order:
4234  * 1) keep things fair between processes/task groups
4235  * 2) pick the "next" process, since someone really wants that to run
4236  * 3) pick the "last" process, for cache locality
4237  * 4) do not run the "skip" process, if something else is available
4238  */
4239 static struct sched_entity *
4240 pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr)
4241 {
4242         struct sched_entity *left = __pick_first_entity(cfs_rq);
4243         struct sched_entity *se;
4244 
4245         /*
4246          * If curr is set we have to see if its left of the leftmost entity
4247          * still in the tree, provided there was anything in the tree at all.
4248          */
4249         if (!left || (curr && entity_before(curr, left)))
4250                 left = curr;
4251 
4252         se = left; /* ideally we run the leftmost entity */
4253 
4254         /*
4255          * Avoid running the skip buddy, if running something else can
4256          * be done without getting too unfair.
4257          */
4258         if (cfs_rq->skip == se) {
4259                 struct sched_entity *second;
4260 
4261                 if (se == curr) {
4262                         second = __pick_first_entity(cfs_rq);
4263                 } else {
4264                         second = __pick_next_entity(se);
4265                         if (!second || (curr && entity_before(curr, second)))
4266                                 second = curr;
4267                 }
4268 
4269                 if (second && wakeup_preempt_entity(second, left) < 1)
4270                         se = second;
4271         }
4272 
4273         /*
4274          * Prefer last buddy, try to return the CPU to a preempted task.
4275          */
4276         if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1)
4277                 se = cfs_rq->last;
4278 
4279         /*
4280          * Someone really wants this to run. If it's not unfair, run it.
4281          */
4282         if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1)
4283                 se = cfs_rq->next;
4284 
4285         clear_buddies(cfs_rq, se);
4286 
4287         return se;
4288 }
4289 
4290 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq);
4291 
4292 static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev)
4293 {
4294         /*
4295          * If still on the runqueue then deactivate_task()
4296          * was not called and update_curr() has to be done:
4297          */
4298         if (prev->on_rq)
4299                 update_curr(cfs_rq);
4300 
4301         /* throttle cfs_rqs exceeding runtime */
4302         check_cfs_rq_runtime(cfs_rq);
4303 
4304         check_spread(cfs_rq, prev);
4305 
4306         if (prev->on_rq) {
4307                 update_stats_wait_start(cfs_rq, prev);
4308                 /* Put 'current' back into the tree. */
4309                 __enqueue_entity(cfs_rq, prev);
4310                 /* in !on_rq case, update occurred at dequeue */
4311                 update_load_avg(cfs_rq, prev, 0);
4312         }
4313         cfs_rq->curr = NULL;
4314 }
4315 
4316 static void
4317 entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
4318 {
4319         /*
4320          * Update run-time statistics of the 'current'.
4321          */
4322         update_curr(cfs_rq);
4323 
4324         /*
4325          * Ensure that runnable average is periodically updated.
4326          */
4327         update_load_avg(cfs_rq, curr, UPDATE_TG);
4328         update_cfs_group(curr);
4329 
4330 #ifdef CONFIG_SCHED_HRTICK
4331         /*
4332          * queued ticks are scheduled to match the slice, so don't bother
4333          * validating it and just reschedule.
4334          */
4335         if (queued) {
4336                 resched_curr(rq_of(cfs_rq));
4337                 return;
4338         }
4339         /*
4340          * don't let the period tick interfere with the hrtick preemption
4341          */
4342         if (!sched_feat(DOUBLE_TICK) &&
4343                         hrtimer_active(&rq_of(cfs_rq)->hrtick_timer))
4344                 return;
4345 #endif
4346 
4347         if (cfs_rq->nr_running > 1)
4348                 check_preempt_tick(cfs_rq, curr);
4349 }
4350 
4351 
4352 /**************************************************
4353  * CFS bandwidth control machinery
4354  */
4355 
4356 #ifdef CONFIG_CFS_BANDWIDTH
4357 
4358 #ifdef HAVE_JUMP_LABEL
4359 static struct static_key __cfs_bandwidth_used;
4360 
4361 static inline bool cfs_bandwidth_used(void)
4362 {
4363         return static_key_false(&__cfs_bandwidth_used);
4364 }
4365 
4366 void cfs_bandwidth_usage_inc(void)
4367 {
4368         static_key_slow_inc_cpuslocked(&__cfs_bandwidth_used);
4369 }
4370 
4371 void cfs_bandwidth_usage_dec(void)
4372 {
4373         static_key_slow_dec_cpuslocked(&__cfs_bandwidth_used);
4374 }
4375 #else /* HAVE_JUMP_LABEL */
4376 static bool cfs_bandwidth_used(void)
4377 {
4378         return true;
4379 }
4380 
4381 void cfs_bandwidth_usage_inc(void) {}
4382 void cfs_bandwidth_usage_dec(void) {}
4383 #endif /* HAVE_JUMP_LABEL */
4384 
4385 /*
4386  * default period for cfs group bandwidth.
4387  * default: 0.1s, units: nanoseconds
4388  */
4389 static inline u64 default_cfs_period(void)
4390 {
4391         return 100000000ULL;
4392 }
4393 
4394 static inline u64 sched_cfs_bandwidth_slice(void)
4395 {
4396         return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC;
4397 }
4398 
4399 /*
4400  * Replenish runtime according to assigned quota and update expiration time.
4401  * We use sched_clock_cpu directly instead of rq->clock to avoid adding
4402  * additional synchronization around rq->lock.
4403  *
4404  * requires cfs_b->lock
4405  */
4406 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)
4407 {
4408         u64 now;
4409 
4410         if (cfs_b->quota == RUNTIME_INF)
4411                 return;
4412 
4413         now = sched_clock_cpu(smp_processor_id());
4414         cfs_b->runtime = cfs_b->quota;
4415         cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period);
4416 }
4417 
4418 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
4419 {
4420         return &tg->cfs_bandwidth;
4421 }
4422 
4423 /* rq->task_clock normalized against any time this cfs_rq has spent throttled */
4424 static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq)
4425 {
4426         if (unlikely(cfs_rq->throttle_count))
4427                 return cfs_rq->throttled_clock_task - cfs_rq->throttled_clock_task_time;
4428 
4429         return rq_clock_task(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time;
4430 }
4431 
4432 /* returns 0 on failure to allocate runtime */
4433 static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
4434 {
4435         struct task_group *tg = cfs_rq->tg;
4436         struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);
4437         u64 amount = 0, min_amount, expires;
4438 
4439         /* note: this is a positive sum as runtime_remaining <= 0 */
4440         min_amount = sched_cfs_bandwidth_slice() - cfs_rq->runtime_remaining;
4441 
4442         raw_spin_lock(&cfs_b->lock);
4443         if (cfs_b->quota == RUNTIME_INF)
4444                 amount = min_amount;
4445         else {
4446                 start_cfs_bandwidth(cfs_b);
4447 
4448                 if (cfs_b->runtime > 0) {
4449                         amount = min(cfs_b->runtime, min_amount);
4450                         cfs_b->runtime -= amount;
4451                         cfs_b->idle = 0;
4452                 }
4453         }
4454         expires = cfs_b->runtime_expires;
4455         raw_spin_unlock(&cfs_b->lock);
4456 
4457         cfs_rq->runtime_remaining += amount;
4458         /*
4459          * we may have advanced our local expiration to account for allowed
4460          * spread between our sched_clock and the one on which runtime was
4461          * issued.
4462          */
4463         if ((s64)(expires - cfs_rq->runtime_expires) > 0)
4464                 cfs_rq->runtime_expires = expires;
4465 
4466         return cfs_rq->runtime_remaining > 0;
4467 }
4468 
4469 /*
4470  * Note: This depends on the synchronization provided by sched_clock and the
4471  * fact that rq->clock snapshots this value.
4472  */
4473 static void expire_cfs_rq_runtime(struct cfs_rq *cfs_rq)
4474 {
4475         struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
4476 
4477         /* if the deadline is ahead of our clock, nothing to do */
4478         if (likely((s64)(rq_clock(rq_of(cfs_rq)) - cfs_rq->runtime_expires) < 0))
4479                 return;
4480 
4481         if (cfs_rq->runtime_remaining < 0)
4482                 return;
4483 
4484         /*
4485          * If the local deadline has passed we have to consider the
4486          * possibility that our sched_clock is 'fast' and the global deadline
4487          * has not truly expired.
4488          *
4489          * Fortunately we can check determine whether this the case by checking
4490          * whether the global deadline has advanced. It is valid to compare
4491          * cfs_b->runtime_expires without any locks since we only care about
4492          * exact equality, so a partial write will still work.
4493          */
4494 
4495         if (cfs_rq->runtime_expires != cfs_b->runtime_expires) {
4496                 /* extend local deadline, drift is bounded above by 2 ticks */
4497                 cfs_rq->runtime_expires += TICK_NSEC;
4498         } else {
4499                 /* global deadline is ahead, expiration has passed */
4500                 cfs_rq->runtime_remaining = 0;
4501         }
4502 }
4503 
4504 static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
4505 {
4506         /* dock delta_exec before expiring quota (as it could span periods) */
4507         cfs_rq->runtime_remaining -= delta_exec;
4508         expire_cfs_rq_runtime(cfs_rq);
4509 
4510         if (likely(cfs_rq->runtime_remaining > 0))
4511                 return;
4512 
4513         /*
4514          * if we're unable to extend our runtime we resched so that the active
4515          * hierarchy can be throttled
4516          */
4517         if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr))
4518                 resched_curr(rq_of(cfs_rq));
4519 }
4520 
4521 static __always_inline
4522 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
4523 {
4524         if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled)
4525                 return;
4526 
4527         __account_cfs_rq_runtime(cfs_rq, delta_exec);
4528 }
4529 
4530 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
4531 {
4532         return cfs_bandwidth_used() && cfs_rq->throttled;
4533 }
4534 
4535 /* check whether cfs_rq, or any parent, is throttled */
4536 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
4537 {
4538         return cfs_bandwidth_used() && cfs_rq->throttle_count;
4539 }
4540 
4541 /*
4542  * Ensure that neither of the group entities corresponding to src_cpu or
4543  * dest_cpu are members of a throttled hierarchy when performing group
4544  * load-balance operations.
4545  */
4546 static inline int throttled_lb_pair(struct task_group *tg,
4547                                     int src_cpu, int dest_cpu)
4548 {
4549         struct cfs_rq *src_cfs_rq, *dest_cfs_rq;
4550 
4551         src_cfs_rq = tg->cfs_rq[src_cpu];
4552         dest_cfs_rq = tg->cfs_rq[dest_cpu];
4553 
4554         return throttled_hierarchy(src_cfs_rq) ||
4555                throttled_hierarchy(dest_cfs_rq);
4556 }
4557 
4558 /* updated child weight may affect parent so we have to do this bottom up */
4559 static int tg_unthrottle_up(struct task_group *tg, void *data)
4560 {
4561         struct rq *rq = data;
4562         struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
4563 
4564         cfs_rq->throttle_count--;
4565         if (!cfs_rq->throttle_count) {
4566                 /* adjust cfs_rq_clock_task() */
4567                 cfs_rq->throttled_clock_task_time += rq_clock_task(rq) -
4568                                              cfs_rq->throttled_clock_task;
4569         }
4570 
4571         return 0;
4572 }
4573 
4574 static int tg_throttle_down(struct task_group *tg, void *data)
4575 {
4576         struct rq *rq = data;
4577         struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
4578 
4579         /* group is entering throttled state, stop time */
4580         if (!cfs_rq->throttle_count)
4581                 cfs_rq->throttled_clock_task = rq_clock_task(rq);
4582         cfs_rq->throttle_count++;
4583 
4584         return 0;
4585 }
4586 
4587 static void throttle_cfs_rq(struct cfs_rq *cfs_rq)
4588 {
4589         struct rq *rq = rq_of(cfs_rq);
4590         struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
4591         struct sched_entity *se;
4592         long task_delta, dequeue = 1;
4593         bool empty;
4594 
4595         se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))];
4596 
4597         /* freeze hierarchy runnable averages while throttled */
4598         rcu_read_lock();
4599         walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq);
4600         rcu_read_unlock();
4601 
4602         task_delta = cfs_rq->h_nr_running;
4603         for_each_sched_entity(se) {
4604                 struct cfs_rq *qcfs_rq = cfs_rq_of(se);
4605                 /* throttled entity or throttle-on-deactivate */
4606                 if (!se->on_rq)
4607                         break;
4608 
4609                 if (dequeue)
4610                         dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP);
4611                 qcfs_rq->h_nr_running -= task_delta;
4612 
4613                 if (qcfs_rq->load.weight)
4614                         dequeue = 0;
4615         }
4616 
4617         if (!se)
4618                 sub_nr_running(rq, task_delta);
4619 
4620         cfs_rq->throttled = 1;
4621         cfs_rq->throttled_clock = rq_clock(rq);
4622         raw_spin_lock(&cfs_b->lock);
4623         empty = list_empty(&cfs_b->throttled_cfs_rq);
4624 
4625         /*
4626          * Add to the _head_ of the list, so that an already-started
4627          * distribute_cfs_runtime will not see us
4628          */
4629         list_add_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq);
4630 
4631         /*
4632          * If we're the first throttled task, make sure the bandwidth
4633          * timer is running.
4634          */
4635         if (empty)
4636                 start_cfs_bandwidth(cfs_b);
4637 
4638         raw_spin_unlock(&cfs_b->lock);
4639 }
4640 
4641 void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
4642 {
4643         struct rq *rq = rq_of(cfs_rq);
4644         struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
4645         struct sched_entity *se;
4646         int enqueue = 1;
4647         long task_delta;
4648 
4649         se = cfs_rq->tg->se[cpu_of(rq)];
4650 
4651         cfs_rq->throttled = 0;
4652 
4653         update_rq_clock(rq);
4654 
4655         raw_spin_lock(&cfs_b->lock);
4656         cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock;
4657         list_del_rcu(&cfs_rq->throttled_list);
4658         raw_spin_unlock(&cfs_b->lock);
4659 
4660         /* update hierarchical throttle state */
4661         walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq);
4662 
4663         if (!cfs_rq->load.weight)
4664                 return;
4665 
4666         task_delta = cfs_rq->h_nr_running;
4667         for_each_sched_entity(se) {
4668                 if (se->on_rq)
4669                         enqueue = 0;
4670 
4671                 cfs_rq = cfs_rq_of(se);
4672                 if (enqueue)
4673                         enqueue_entity(cfs_rq, se, ENQUEUE_WAKEUP);
4674                 cfs_rq->h_nr_running += task_delta;
4675 
4676                 if (cfs_rq_throttled(cfs_rq))
4677                         break;
4678         }
4679 
4680         if (!se)
4681                 add_nr_running(rq, task_delta);
4682 
4683         /* determine whether we need to wake up potentially idle cpu */
4684         if (rq->curr == rq->idle && rq->cfs.nr_running)
4685                 resched_curr(rq);
4686 }
4687 
4688 static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b,
4689                 u64 remaining, u64 expires)
4690 {
4691         struct cfs_rq *cfs_rq;
4692         u64 runtime;
4693         u64 starting_runtime = remaining;
4694 
4695         rcu_read_lock();
4696         list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,
4697                                 throttled_list) {
4698                 struct rq *rq = rq_of(cfs_rq);
4699                 struct rq_flags rf;
4700 
4701                 rq_lock(rq, &rf);
4702                 if (!cfs_rq_throttled(cfs_rq))
4703                         goto next;
4704 
4705                 runtime = -cfs_rq->runtime_remaining + 1;
4706                 if (runtime > remaining)
4707                         runtime = remaining;
4708                 remaining -= runtime;
4709 
4710                 cfs_rq->runtime_remaining += runtime;
4711                 cfs_rq->runtime_expires = expires;
4712 
4713                 /* we check whether we're throttled above */
4714                 if (cfs_rq->runtime_remaining > 0)
4715                         unthrottle_cfs_rq(cfs_rq);
4716 
4717 next:
4718                 rq_unlock(rq, &rf);
4719 
4720                 if (!remaining)
4721                         break;
4722         }
4723         rcu_read_unlock();
4724 
4725         return starting_runtime - remaining;
4726 }
4727 
4728 /*
4729  * Responsible for refilling a task_group's bandwidth and unthrottling its
4730  * cfs_rqs as appropriate. If there has been no activity within the last
4731  * period the timer is deactivated until scheduling resumes; cfs_b->idle is
4732  * used to track this state.
4733  */
4734 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)
4735 {
4736         u64 runtime, runtime_expires;
4737         int throttled;
4738 
4739         /* no need to continue the timer with no bandwidth constraint */
4740         if (cfs_b->quota == RUNTIME_INF)
4741                 goto out_deactivate;
4742 
4743         throttled = !list_empty(&cfs_b->throttled_cfs_rq);
4744         cfs_b->nr_periods += overrun;
4745 
4746         /*
4747          * idle depends on !throttled (for the case of a large deficit), and if
4748          * we're going inactive then everything else can be deferred
4749          */
4750         if (cfs_b->idle && !throttled)
4751                 goto out_deactivate;
4752 
4753         __refill_cfs_bandwidth_runtime(cfs_b);
4754 
4755         if (!throttled) {
4756                 /* mark as potentially idle for the upcoming period */
4757                 cfs_b->idle = 1;
4758                 return 0;
4759         }
4760 
4761         /* account preceding periods in which throttling occurred */
4762         cfs_b->nr_throttled += overrun;
4763 
4764         runtime_expires = cfs_b->runtime_expires;
4765 
4766         /*
4767          * This check is repeated as we are holding onto the new bandwidth while
4768          * we unthrottle. This can potentially race with an unthrottled group
4769          * trying to acquire new bandwidth from the global pool. This can result
4770          * in us over-using our runtime if it is all used during this loop, but
4771          * only by limited amounts in that extreme case.
4772          */
4773         while (throttled && cfs_b->runtime > 0) {
4774                 runtime = cfs_b->runtime;
4775                 raw_spin_unlock(&cfs_b->lock);
4776                 /* we can't nest cfs_b->lock while distributing bandwidth */
4777                 runtime = distribute_cfs_runtime(cfs_b, runtime,
4778                                                  runtime_expires);
4779                 raw_spin_lock(&cfs_b->lock);
4780 
4781                 throttled = !list_empty(&cfs_b->throttled_cfs_rq);
4782 
4783                 cfs_b->runtime -= min(runtime, cfs_b->runtime);
4784         }
4785 
4786         /*
4787          * While we are ensured activity in the period following an
4788          * unthrottle, this also covers the case in which the new bandwidth is
4789          * insufficient to cover the existing bandwidth deficit.  (Forcing the
4790          * timer to remain active while there are any throttled entities.)
4791          */
4792         cfs_b->idle = 0;
4793 
4794         return 0;
4795 
4796 out_deactivate:
4797         return 1;
4798 }
4799 
4800 /* a cfs_rq won't donate quota below this amount */
4801 static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC;
4802 /* minimum remaining period time to redistribute slack quota */
4803 static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC;
4804 /* how long we wait to gather additional slack before distributing */
4805 static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC;
4806 
4807 /*
4808  * Are we near the end of the current quota period?
4809  *
4810  * Requires cfs_b->lock for hrtimer_expires_remaining to be safe against the
4811  * hrtimer base being cleared by hrtimer_start. In the case of
4812  * migrate_hrtimers, base is never cleared, so we are fine.
4813  */
4814 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire)
4815 {
4816         struct hrtimer *refresh_timer = &cfs_b->period_timer;
4817         u64 remaining;
4818 
4819         /* if the call-back is running a quota refresh is already occurring */
4820         if (hrtimer_callback_running(refresh_timer))
4821                 return 1;
4822 
4823         /* is a quota refresh about to occur? */
4824         remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer));
4825         if (remaining < min_expire)
4826                 return 1;
4827 
4828         return 0;
4829 }
4830 
4831 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)
4832 {
4833         u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration;
4834 
4835         /* if there's a quota refresh soon don't bother with slack */
4836         if (runtime_refresh_within(cfs_b, min_left))
4837                 return;
4838 
4839         hrtimer_start(&cfs_b->slack_timer,
4840                         ns_to_ktime(cfs_bandwidth_slack_period),
4841                         HRTIMER_MODE_REL);
4842 }
4843 
4844 /* we know any runtime found here is valid as update_curr() precedes return */
4845 static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
4846 {
4847         struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
4848         s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime;
4849 
4850         if (slack_runtime <= 0)
4851                 return;
4852 
4853         raw_spin_lock(&cfs_b->lock);
4854         if (cfs_b->quota != RUNTIME_INF &&
4855             cfs_rq->runtime_expires == cfs_b->runtime_expires) {
4856                 cfs_b->runtime += slack_runtime;
4857 
4858                 /* we are under rq->lock, defer unthrottling using a timer */
4859                 if (cfs_b->runtime > sched_cfs_bandwidth_slice() &&
4860                     !list_empty(&cfs_b->throttled_cfs_rq))
4861                         start_cfs_slack_bandwidth(cfs_b);
4862         }
4863         raw_spin_unlock(&cfs_b->lock);
4864 
4865         /* even if it's not valid for return we don't want to try again */
4866         cfs_rq->runtime_remaining -= slack_runtime;
4867 }
4868 
4869 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
4870 {
4871         if (!cfs_bandwidth_used())
4872                 return;
4873 
4874         if (!cfs_rq->runtime_enabled || cfs_rq->nr_running)
4875                 return;
4876 
4877         __return_cfs_rq_runtime(cfs_rq);
4878 }
4879 
4880 /*
4881  * This is done with a timer (instead of inline with bandwidth return) since
4882  * it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs.
4883  */
4884 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
4885 {
4886         u64 runtime = 0, slice = sched_cfs_bandwidth_slice();
4887         u64 expires;
4888 
4889         /* confirm we're still not at a refresh boundary */
4890         raw_spin_lock(&cfs_b->lock);
4891         if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) {
4892                 raw_spin_unlock(&cfs_b->lock);
4893                 return;
4894         }
4895 
4896         if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice)
4897                 runtime = cfs_b->runtime;
4898 
4899         expires = cfs_b->runtime_expires;
4900         raw_spin_unlock(&cfs_b->lock);
4901 
4902         if (!runtime)
4903                 return;
4904 
4905         runtime = distribute_cfs_runtime(cfs_b, runtime, expires);
4906 
4907         raw_spin_lock(&cfs_b->lock);
4908         if (expires == cfs_b->runtime_expires)
4909                 cfs_b->runtime -= min(runtime, cfs_b->runtime);
4910         raw_spin_unlock(&cfs_b->lock);
4911 }
4912 
4913 /*
4914  * When a group wakes up we want to make sure that its quota is not already
4915  * expired/exceeded, otherwise it may be allowed to steal additional ticks of
4916  * runtime as update_curr() throttling can not not trigger until it's on-rq.
4917  */
4918 static void check_enqueue_throttle(struct cfs_rq *cfs_rq)
4919 {
4920         if (!cfs_bandwidth_used())
4921                 return;
4922 
4923         /* an active group must be handled by the update_curr()->put() path */
4924         if (!cfs_rq->runtime_enabled || cfs_rq->curr)
4925                 return;
4926 
4927         /* ensure the group is not already throttled */
4928         if (cfs_rq_throttled(cfs_rq))
4929                 return;
4930 
4931         /* update runtime allocation */
4932         account_cfs_rq_runtime(cfs_rq, 0);
4933         if (cfs_rq->runtime_remaining <= 0)
4934                 throttle_cfs_rq(cfs_rq);
4935 }
4936 
4937 static void sync_throttle(struct task_group *tg, int cpu)
4938 {
4939         struct cfs_rq *pcfs_rq, *cfs_rq;
4940 
4941         if (!cfs_bandwidth_used())
4942                 return;
4943 
4944         if (!tg->parent)
4945                 return;
4946 
4947         cfs_rq = tg->cfs_rq[cpu];
4948         pcfs_rq = tg->parent->cfs_rq[cpu];
4949 
4950         cfs_rq->throttle_count = pcfs_rq->throttle_count;
4951         cfs_rq->throttled_clock_task = rq_clock_task(cpu_rq(cpu));
4952 }
4953 
4954 /* conditionally throttle active cfs_rq's from put_prev_entity() */
4955 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq)
4956 {
4957         if (!cfs_bandwidth_used())
4958                 return false;
4959 
4960         if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0))
4961                 return false;
4962 
4963         /*
4964          * it's possible for a throttled entity to be forced into a running
4965          * state (e.g. set_curr_task), in this case we're finished.
4966          */
4967         if (cfs_rq_throttled(cfs_rq))
4968                 return true;
4969 
4970         throttle_cfs_rq(cfs_rq);
4971         return true;
4972 }
4973 
4974 static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer)
4975 {
4976         struct cfs_bandwidth *cfs_b =
4977                 container_of(timer, struct cfs_bandwidth, slack_timer);
4978 
4979         do_sched_cfs_slack_timer(cfs_b);
4980 
4981         return HRTIMER_NORESTART;
4982 }
4983 
4984 static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
4985 {
4986         struct cfs_bandwidth *cfs_b =
4987                 container_of(timer, struct cfs_bandwidth, period_timer);
4988         int overrun;
4989         int idle = 0;
4990 
4991         raw_spin_lock(&cfs_b->lock);
4992         for (;;) {
4993                 overrun = hrtimer_forward_now(timer, cfs_b->period);
4994                 if (!overrun)
4995                         break;
4996 
4997                 idle = do_sched_cfs_period_timer(cfs_b, overrun);
4998         }
4999         if (idle)
5000                 cfs_b->period_active = 0;
5001         raw_spin_unlock(&cfs_b->lock);
5002 
5003         return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
5004 }
5005 
5006 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
5007 {
5008         raw_spin_lock_init(&cfs_b->lock);
5009         cfs_b->runtime = 0;
5010         cfs_b->quota = RUNTIME_INF;
5011         cfs_b->period = ns_to_ktime(default_cfs_period());
5012 
5013         INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq);
5014         hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
5015         cfs_b->period_timer.function = sched_cfs_period_timer;
5016         hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
5017         cfs_b->slack_timer.function = sched_cfs_slack_timer;
5018 }
5019 
5020 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
5021 {
5022         cfs_rq->runtime_enabled = 0;
5023         INIT_LIST_HEAD(&cfs_rq->throttled_list);
5024 }
5025 
5026 void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
5027 {
5028         lockdep_assert_held(&cfs_b->lock);
5029 
5030         if (!cfs_b->period_active) {
5031                 cfs_b->period_active = 1;
5032                 hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period);
5033                 hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED);
5034         }
5035 }
5036 
5037 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
5038 {
5039         /* init_cfs_bandwidth() was not called */
5040         if (!cfs_b->throttled_cfs_rq.next)
5041                 return;
5042 
5043         hrtimer_cancel(&cfs_b->period_timer);
5044         hrtimer_cancel(&cfs_b->slack_timer);
5045 }
5046 
5047 /*
5048  * Both these cpu hotplug callbacks race against unregister_fair_sched_group()
5049  *
5050  * The race is harmless, since modifying bandwidth settings of unhooked group
5051  * bits doesn't do much.
5052  */
5053 
5054 /* cpu online calback */
5055 static void __maybe_unused update_runtime_enabled(struct rq *rq)
5056 {
5057         struct task_group *tg;
5058 
5059         lockdep_assert_held(&rq->lock);
5060 
5061         rcu_read_lock();
5062         list_for_each_entry_rcu(tg, &task_groups, list) {
5063                 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
5064                 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
5065 
5066                 raw_spin_lock(&cfs_b->lock);
5067                 cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;
5068                 raw_spin_unlock(&cfs_b->lock);
5069         }
5070         rcu_read_unlock();
5071 }
5072 
5073 /* cpu offline callback */
5074 static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq)
5075 {
5076         struct task_group *tg;
5077 
5078         lockdep_assert_held(&rq->lock);
5079 
5080         rcu_read_lock();
5081         list_for_each_entry_rcu(tg, &task_groups, list) {
5082                 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
5083 
5084                 if (!cfs_rq->runtime_enabled)
5085                         continue;
5086 
5087                 /*
5088                  * clock_task is not advancing so we just need to make sure
5089                  * there's some valid quota amount
5090                  */
5091                 cfs_rq->runtime_remaining = 1;
5092                 /*
5093                  * Offline rq is schedulable till cpu is completely disabled
5094                  * in take_cpu_down(), so we prevent new cfs throttling here.
5095                  */
5096                 cfs_rq->runtime_enabled = 0;
5097 
5098                 if (cfs_rq_throttled(cfs_rq))
5099                         unthrottle_cfs_rq(cfs_rq);
5100         }
5101         rcu_read_unlock();
5102 }
5103 
5104 #else /* CONFIG_CFS_BANDWIDTH */
5105 static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq)
5106 {
5107         return rq_clock_task(rq_of(cfs_rq));
5108 }
5109 
5110 static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {}
5111 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; }
5112 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {}
5113 static inline void sync_throttle(struct task_group *tg, int cpu) {}
5114 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
5115 
5116 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
5117 {
5118         return 0;
5119 }
5120 
5121 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
5122 {
5123         return 0;
5124 }
5125 
5126 static inline int throttled_lb_pair(struct task_group *tg,
5127                                     int src_cpu, int dest_cpu)
5128 {
5129         return 0;
5130 }
5131 
5132 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
5133 
5134 #ifdef CONFIG_FAIR_GROUP_SCHED
5135 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
5136 #endif
5137 
5138 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
5139 {
5140         return NULL;
5141 }
5142 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
5143 static inline void update_runtime_enabled(struct rq *rq) {}
5144 static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {}
5145 
5146 #endif /* CONFIG_CFS_BANDWIDTH */
5147 
5148 /**************************************************
5149  * CFS operations on tasks:
5150  */
5151 
5152 #ifdef CONFIG_SCHED_HRTICK
5153 static void hrtick_start_fair(struct rq *rq, struct task_struct *p)
5154 {
5155         struct sched_entity *se = &p->se;
5156         struct cfs_rq *cfs_rq = cfs_rq_of(se);
5157 
5158         SCHED_WARN_ON(task_rq(p) != rq);
5159 
5160         if (rq->cfs.h_nr_running > 1) {
5161                 u64 slice = sched_slice(cfs_rq, se);
5162                 u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime;
5163                 s64 delta = slice - ran;
5164 
5165                 if (delta < 0) {
5166                         if (rq->curr == p)
5167                                 resched_curr(rq);
5168                         return;
5169                 }
5170                 hrtick_start(rq, delta);
5171         }
5172 }
5173 
5174 /*
5175  * called from enqueue/dequeue and updates the hrtick when the
5176  * current task is from our class and nr_running is low enough
5177  * to matter.
5178  */
5179 static void hrtick_update(struct rq *rq)
5180 {
5181         struct task_struct *curr = rq->curr;
5182 
5183         if (!hrtick_enabled(rq) || curr->sched_class != &fair_sched_class)
5184                 return;
5185 
5186         if (cfs_rq_of(&curr->se)->nr_running < sched_nr_latency)
5187                 hrtick_start_fair(rq, curr);
5188 }
5189 #else /* !CONFIG_SCHED_HRTICK */
5190 static inline void
5191 hrtick_start_fair(struct rq *rq, struct task_struct *p)
5192 {
5193 }
5194 
5195 static inline void hrtick_update(struct rq *rq)
5196 {
5197 }
5198 #endif
5199 
5200 /*
5201  * The enqueue_task method is called before nr_running is
5202  * increased. Here we update the fair scheduling stats and
5203  * then put the task into the rbtree:
5204  */
5205 static void
5206 enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
5207 {
5208         struct cfs_rq *cfs_rq;
5209         struct sched_entity *se = &p->se;
5210 
5211         /*
5212          * If in_iowait is set, the code below may not trigger any cpufreq
5213          * utilization updates, so do it here explicitly with the IOWAIT flag
5214          * passed.
5215          */
5216         if (p->in_iowait)
5217                 cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT);
5218 
5219         for_each_sched_entity(se) {
5220                 if (se->on_rq)
5221                         break;
5222                 cfs_rq = cfs_rq_of(se);
5223                 enqueue_entity(cfs_rq, se, flags);
5224 
5225                 /*
5226                  * end evaluation on encountering a throttled cfs_rq
5227                  *
5228                  * note: in the case of encountering a throttled cfs_rq we will
5229                  * post the final h_nr_running increment below.
5230                  */
5231                 if (cfs_rq_throttled(cfs_rq))
5232                         break;
5233                 cfs_rq->h_nr_running++;
5234 
5235                 flags = ENQUEUE_WAKEUP;
5236         }
5237 
5238         for_each_sched_entity(se) {
5239                 cfs_rq = cfs_rq_of(se);
5240                 cfs_rq->h_nr_running++;
5241 
5242                 if (cfs_rq_throttled(cfs_rq))
5243                         break;
5244 
5245                 update_load_avg(cfs_rq, se, UPDATE_TG);
5246                 update_cfs_group(se);
5247         }
5248 
5249         if (!se)
5250                 add_nr_running(rq, 1);
5251 
5252         hrtick_update(rq);
5253 }
5254 
5255 static void set_next_buddy(struct sched_entity *se);
5256 
5257 /*
5258  * The dequeue_task method is called before nr_running is
5259  * decreased. We remove the task from the rbtree and
5260  * update the fair scheduling stats:
5261  */
5262 static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
5263 {
5264         struct cfs_rq *cfs_rq;
5265         struct sched_entity *se = &p->se;
5266         int task_sleep = flags & DEQUEUE_SLEEP;
5267 
5268         for_each_sched_entity(se) {
5269                 cfs_rq = cfs_rq_of(se);
5270                 dequeue_entity(cfs_rq, se, flags);
5271 
5272                 /*
5273                  * end evaluation on encountering a throttled cfs_rq
5274                  *
5275                  * note: in the case of encountering a throttled cfs_rq we will
5276                  * post the final h_nr_running decrement below.
5277                 */
5278                 if (cfs_rq_throttled(cfs_rq))
5279                         break;
5280                 cfs_rq->h_nr_running--;
5281 
5282                 /* Don't dequeue parent if it has other entities besides us */
5283                 if (cfs_rq->load.weight) {
5284                         /* Avoid re-evaluating load for this entity: */
5285                         se = parent_entity(se);
5286                         /*
5287                          * Bias pick_next to pick a task from this cfs_rq, as
5288                          * p is sleeping when it is within its sched_slice.
5289                          */
5290                         if (task_sleep && se && !throttled_hierarchy(cfs_rq))
5291                                 set_next_buddy(se);
5292                         break;
5293                 }
5294                 flags |= DEQUEUE_SLEEP;
5295         }
5296 
5297         for_each_sched_entity(se) {
5298                 cfs_rq = cfs_rq_of(se);
5299                 cfs_rq->h_nr_running--;
5300 
5301                 if (cfs_rq_throttled(cfs_rq))
5302                         break;
5303 
5304                 update_load_avg(cfs_rq, se, UPDATE_TG);
5305                 update_cfs_group(se);
5306         }
5307 
5308         if (!se)
5309                 sub_nr_running(rq, 1);
5310 
5311         hrtick_update(rq);
5312 }
5313 
5314 #ifdef CONFIG_SMP
5315 
5316 /* Working cpumask for: load_balance, load_balance_newidle. */
5317 DEFINE_PER_CPU(cpumask_var_t, load_balance_mask);
5318 DEFINE_PER_CPU(cpumask_var_t, select_idle_mask);
5319 
5320 #ifdef CONFIG_NO_HZ_COMMON
5321 /*
5322  * per rq 'load' arrray crap; XXX kill this.
5323  */
5324 
5325 /*
5326  * The exact cpuload calculated at every tick would be:
5327  *
5328  *   load' = (1 - 1/2^i) * load + (1/2^i) * cur_load
5329  *
5330  * If a cpu misses updates for n ticks (as it was idle) and update gets
5331  * called on the n+1-th tick when cpu may be busy, then we have:
5332  *
5333  *   load_n   = (1 - 1/2^i)^n * load_0
5334  *   load_n+1 = (1 - 1/2^i)   * load_n + (1/2^i) * cur_load
5335  *
5336  * decay_load_missed() below does efficient calculation of
5337  *
5338  *   load' = (1 - 1/2^i)^n * load
5339  *
5340  * Because x^(n+m) := x^n * x^m we can decompose any x^n in power-of-2 factors.
5341  * This allows us to precompute the above in said factors, thereby allowing the
5342  * reduction of an arbitrary n in O(log_2 n) steps. (See also
5343  * fixed_power_int())
5344  *
5345  * The calculation is approximated on a 128 point scale.
5346  */
5347 #define DEGRADE_SHIFT           7
5348 
5349 static const u8 degrade_zero_ticks[CPU_LOAD_IDX_MAX] = {0, 8, 32, 64, 128};
5350 static const u8 degrade_factor[CPU_LOAD_IDX_MAX][DEGRADE_SHIFT + 1] = {
5351         {   0,   0,  0,  0,  0,  0, 0, 0 },
5352         {  64,  32,  8,  0,  0,  0, 0, 0 },
5353         {  96,  72, 40, 12,  1,  0, 0, 0 },
5354         { 112,  98, 75, 43, 15,  1, 0, 0 },
5355         { 120, 112, 98, 76, 45, 16, 2, 0 }
5356 };
5357 
5358 /*
5359  * Update cpu_load for any missed ticks, due to tickless idle. The backlog
5360  * would be when CPU is idle and so we just decay the old load without
5361  * adding any new load.
5362  */
5363 static unsigned long
5364 decay_load_missed(unsigned long load, unsigned long missed_updates, int idx)
5365 {
5366         int j = 0;
5367 
5368         if (!missed_updates)
5369                 return load;
5370 
5371         if (missed_updates >= degrade_zero_ticks[idx])
5372                 return 0;
5373 
5374         if (idx == 1)
5375                 return load >> missed_updates;
5376 
5377         while (missed_updates) {
5378                 if (missed_updates % 2)
5379                         load = (load * degrade_factor[idx][j]) >> DEGRADE_SHIFT;
5380 
5381                 missed_updates >>= 1;
5382                 j++;
5383         }
5384         return load;
5385 }
5386 #endif /* CONFIG_NO_HZ_COMMON */
5387 
5388 /**
5389  * __cpu_load_update - update the rq->cpu_load[] statistics
5390  * @this_rq: The rq to update statistics for
5391  * @this_load: The current load
5392  * @pending_updates: The number of missed updates
5393  *
5394  * Update rq->cpu_load[] statistics. This function is usually called every
5395  * scheduler tick (TICK_NSEC).
5396  *
5397  * This function computes a decaying average:
5398  *
5399  *   load[i]' = (1 - 1/2^i) * load[i] + (1/2^i) * load
5400  *
5401  * Because of NOHZ it might not get called on every tick which gives need for
5402  * the @pending_updates argument.
5403  *
5404  *   load[i]_n = (1 - 1/2^i) * load[i]_n-1 + (1/2^i) * load_n-1
5405  *             = A * load[i]_n-1 + B ; A := (1 - 1/2^i), B := (1/2^i) * load
5406  *             = A * (A * load[i]_n-2 + B) + B
5407  *             = A * (A * (A * load[i]_n-3 + B) + B) + B
5408  *             = A^3 * load[i]_n-3 + (A^2 + A + 1) * B
5409  *             = A^n * load[i]_0 + (A^(n-1) + A^(n-2) + ... + 1) * B
5410  *             = A^n * load[i]_0 + ((1 - A^n) / (1 - A)) * B
5411  *             = (1 - 1/2^i)^n * (load[i]_0 - load) + load
5412  *
5413  * In the above we've assumed load_n := load, which is true for NOHZ_FULL as
5414  * any change in load would have resulted in the tick being turned back on.
5415  *
5416  * For regular NOHZ, this reduces to:
5417  *
5418  *   load[i]_n = (1 - 1/2^i)^n * load[i]_0
5419  *
5420  * see decay_load_misses(). For NOHZ_FULL we get to subtract and add the extra
5421  * term.
5422  */
5423 static void cpu_load_update(struct rq *this_rq, unsigned long this_load,
5424                             unsigned long pending_updates)
5425 {
5426         unsigned long __maybe_unused tickless_load = this_rq->cpu_load[0];
5427         int i, scale;
5428 
5429         this_rq->nr_load_updates++;
5430 
5431         /* Update our load: */
5432         this_rq->cpu_load[0] = this_load; /* Fasttrack for idx 0 */
5433         for (i = 1, scale = 2; i < CPU_LOAD_IDX_MAX; i++, scale += scale) {
5434                 unsigned long old_load, new_load;
5435 
5436                 /* scale is effectively 1 << i now, and >> i divides by scale */
5437 
5438                 old_load = this_rq->cpu_load[i];
5439 #ifdef CONFIG_NO_HZ_COMMON
5440                 old_load = decay_load_missed(old_load, pending_updates - 1, i);
5441                 if (tickless_load) {
5442                         old_load -= decay_load_missed(tickless_load, pending_updates - 1, i);
5443                         /*
5444                          * old_load can never be a negative value because a
5445                          * decayed tickless_load cannot be greater than the
5446                          * original tickless_load.
5447                          */
5448                         old_load += tickless_load;
5449                 }
5450 #endif
5451                 new_load = this_load;
5452                 /*
5453                  * Round up the averaging division if load is increasing. This
5454                  * prevents us from getting stuck on 9 if the load is 10, for
5455                  * example.
5456                  */
5457                 if (new_load > old_load)
5458                         new_load += scale - 1;
5459 
5460                 this_rq->cpu_load[i] = (old_load * (scale - 1) + new_load) >> i;
5461         }
5462 
5463         sched_avg_update(this_rq);
5464 }
5465 
5466 /* Used instead of source_load when we know the type == 0 */
5467 static unsigned long weighted_cpuload(struct rq *rq)
5468 {
5469         return cfs_rq_runnable_load_avg(&rq->cfs);
5470 }
5471 
5472 #ifdef CONFIG_NO_HZ_COMMON
5473 /*
5474  * There is no sane way to deal with nohz on smp when using jiffies because the
5475  * cpu doing the jiffies update might drift wrt the cpu doing the jiffy reading
5476  * causing off-by-one errors in observed deltas; {0,2} instead of {1,1}.
5477  *
5478  * Therefore we need to avoid the delta approach from the regular tick when
5479  * possible since that would seriously skew the load calculation. This is why we
5480  * use cpu_load_update_periodic() for CPUs out of nohz. However we'll rely on
5481  * jiffies deltas for updates happening while in nohz mode (idle ticks, idle
5482  * loop exit, nohz_idle_balance, nohz full exit...)
5483  *
5484  * This means we might still be one tick off for nohz periods.
5485  */
5486 
5487 static void cpu_load_update_nohz(struct rq *this_rq,
5488                                  unsigned long curr_jiffies,
5489                                  unsigned long load)
5490 {
5491         unsigned long pending_updates;
5492 
5493         pending_updates = curr_jiffies - this_rq->last_load_update_tick;
5494         if (pending_updates) {
5495                 this_rq->last_load_update_tick = curr_jiffies;
5496                 /*
5497                  * In the regular NOHZ case, we were idle, this means load 0.
5498                  * In the NOHZ_FULL case, we were non-idle, we should consider
5499                  * its weighted load.
5500                  */
5501                 cpu_load_update(this_rq, load, pending_updates);
5502         }
5503 }
5504 
5505 /*
5506  * Called from nohz_idle_balance() to update the load ratings before doing the
5507  * idle balance.
5508  */
5509 static void cpu_load_update_idle(struct rq *this_rq)
5510 {
5511         /*
5512          * bail if there's load or we're actually up-to-date.
5513          */
5514         if (weighted_cpuload(this_rq))
5515                 return;
5516 
5517         cpu_load_update_nohz(this_rq, READ_ONCE(jiffies), 0);
5518 }
5519 
5520 /*
5521  * Record CPU load on nohz entry so we know the tickless load to account
5522  * on nohz exit. cpu_load[0] happens then to be updated more frequently
5523  * than other cpu_load[idx] but it should be fine as cpu_load readers
5524  * shouldn't rely into synchronized cpu_load[*] updates.
5525  */
5526 void cpu_load_update_nohz_start(void)
5527 {
5528         struct rq *this_rq = this_rq();
5529 
5530         /*
5531          * This is all lockless but should be fine. If weighted_cpuload changes
5532          * concurrently we'll exit nohz. And cpu_load write can race with
5533          * cpu_load_update_idle() but both updater would be writing the same.
5534          */
5535         this_rq->cpu_load[0] = weighted_cpuload(this_rq);
5536 }
5537 
5538 /*
5539  * Account the tickless load in the end of a nohz frame.
5540  */
5541 void cpu_load_update_nohz_stop(void)
5542 {
5543         unsigned long curr_jiffies = READ_ONCE(jiffies);
5544         struct rq *this_rq = this_rq();
5545         unsigned long load;
5546         struct rq_flags rf;
5547 
5548         if (curr_jiffies == this_rq->last_load_update_tick)
5549                 return;
5550 
5551         load = weighted_cpuload(this_rq);
5552         rq_lock(this_rq, &rf);
5553         update_rq_clock(this_rq);
5554         cpu_load_update_nohz(this_rq, curr_jiffies, load);
5555         rq_unlock(this_rq, &rf);
5556 }
5557 #else /* !CONFIG_NO_HZ_COMMON */
5558 static inline void cpu_load_update_nohz(struct rq *this_rq,
5559                                         unsigned long curr_jiffies,
5560                                         unsigned long load) { }
5561 #endif /* CONFIG_NO_HZ_COMMON */
5562 
5563 static void cpu_load_update_periodic(struct rq *this_rq, unsigned long load)
5564 {
5565 #ifdef CONFIG_NO_HZ_COMMON
5566         /* See the mess around cpu_load_update_nohz(). */
5567         this_rq->last_load_update_tick = READ_ONCE(jiffies);
5568 #endif
5569         cpu_load_update(this_rq, load, 1);
5570 }
5571 
5572 /*
5573  * Called from scheduler_tick()
5574  */
5575 void cpu_load_update_active(struct rq *this_rq)
5576 {
5577         unsigned long load = weighted_cpuload(this_rq);
5578 
5579         if (tick_nohz_tick_stopped())
5580                 cpu_load_update_nohz(this_rq, READ_ONCE(jiffies), load);
5581         else
5582                 cpu_load_update_periodic(this_rq, load);
5583 }
5584 
5585 /*
5586  * Return a low guess at the load of a migration-source cpu weighted
5587  * according to the scheduling class and "nice" value.
5588  *
5589  * We want to under-estimate the load of migration sources, to
5590  * balance conservatively.
5591  */
5592 static unsigned long source_load(int cpu, int type)
5593 {
5594         struct rq *rq = cpu_rq(cpu);
5595         unsigned long total = weighted_cpuload(rq);
5596 
5597         if (type == 0 || !sched_feat(LB_BIAS))
5598                 return total;
5599 
5600         return min(rq->cpu_load[type-1], total);
5601 }
5602 
5603 /*
5604  * Return a high guess at the load of a migration-target cpu weighted
5605  * according to the scheduling class and "nice" value.
5606  */
5607 static unsigned long target_load(int cpu, int type)
5608 {
5609         struct rq *rq = cpu_rq(cpu);
5610         unsigned long total = weighted_cpuload(rq);
5611 
5612         if (type == 0 || !sched_feat(LB_BIAS))
5613                 return total;
5614 
5615         return max(rq->cpu_load[type-1], total);
5616 }
5617 
5618 static unsigned long capacity_of(int cpu)
5619 {
5620         return cpu_rq(cpu)->cpu_capacity;
5621 }
5622 
5623 static unsigned long capacity_orig_of(int cpu)
5624 {
5625         return cpu_rq(cpu)->cpu_capacity_orig;
5626 }
5627 
5628 static unsigned long cpu_avg_load_per_task(int cpu)
5629 {
5630         struct rq *rq = cpu_rq(cpu);
5631         unsigned long nr_running = READ_ONCE(rq->cfs.h_nr_running);
5632         unsigned long load_avg = weighted_cpuload(rq);
5633 
5634         if (nr_running)
5635                 return load_avg / nr_running;
5636 
5637         return 0;
5638 }
5639 
5640 static void record_wakee(struct task_struct *p)
5641 {
5642         /*
5643          * Only decay a single time; tasks that have less then 1 wakeup per
5644          * jiffy will not have built up many flips.
5645          */
5646         if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) {
5647                 current->wakee_flips >>= 1;
5648                 current->wakee_flip_decay_ts = jiffies;
5649         }
5650 
5651         if (current->last_wakee != p) {
5652                 current->last_wakee = p;
5653                 current->wakee_flips++;
5654         }
5655 }
5656 
5657 /*
5658  * Detect M:N waker/wakee relationships via a switching-frequency heuristic.
5659  *
5660  * A waker of many should wake a different task than the one last awakened
5661  * at a frequency roughly N times higher than one of its wakees.
5662  *
5663  * In order to determine whether we should let the load spread vs consolidating
5664  * to shared cache, we look for a minimum 'flip' frequency of llc_size in one
5665  * partner, and a factor of lls_size higher frequency in the other.
5666  *
5667  * With both conditions met, we can be relatively sure that the relationship is
5668  * non-monogamous, with partner count exceeding socket size.
5669  *
5670  * Waker/wakee being client/server, worker/dispatcher, interrupt source or
5671  * whatever is irrelevant, spread criteria is apparent partner count exceeds
5672  * socket size.
5673  */
5674 static int wake_wide(struct task_struct *p)
5675 {
5676         unsigned int master = current->wakee_flips;
5677         unsigned int slave = p->wakee_flips;
5678         int factor = this_cpu_read(sd_llc_size);
5679 
5680         if (master < slave)
5681                 swap(master, slave);
5682         if (slave < factor || master < slave * factor)
5683                 return 0;
5684         return 1;
5685 }
5686 
5687 /*
5688  * The purpose of wake_affine() is to quickly determine on which CPU we can run
5689  * soonest. For the purpose of speed we only consider the waking and previous
5690  * CPU.
5691  *
5692  * wake_affine_idle() - only considers 'now', it check if the waking CPU is (or
5693  *                      will be) idle.
5694  *
5695  * wake_affine_weight() - considers the weight to reflect the average
5696  *                        scheduling latency of the CPUs. This seems to work
5697  *                        for the overloaded case.
5698  */
5699 
5700 static bool
5701 wake_affine_idle(struct sched_domain *sd, struct task_struct *p,
5702                  int this_cpu, int prev_cpu, int sync)
5703 {
5704         if (idle_cpu(this_cpu))
5705                 return true;
5706 
5707         if (sync && cpu_rq(this_cpu)->nr_running == 1)
5708                 return true;
5709 
5710         return false;
5711 }
5712 
5713 static bool
5714 wake_affine_weight(struct sched_domain *sd, struct task_struct *p,
5715                    int this_cpu, int prev_cpu, int sync)
5716 {
5717         s64 this_eff_load, prev_eff_load;
5718         unsigned long task_load;
5719 
5720         this_eff_load = target_load(this_cpu, sd->wake_idx);
5721         prev_eff_load = source_load(prev_cpu, sd->wake_idx);
5722 
5723         if (sync) {
5724                 unsigned long current_load = task_h_load(current);
5725 
5726                 if (current_load > this_eff_load)
5727                         return true;
5728 
5729                 this_eff_load -= current_load;
5730         }
5731 
5732         task_load = task_h_load(p);
5733 
5734         this_eff_load += task_load;
5735         if (sched_feat(WA_BIAS))
5736                 this_eff_load *= 100;
5737         this_eff_load *= capacity_of(prev_cpu);
5738 
5739         prev_eff_load -= task_load;
5740         if (sched_feat(WA_BIAS))
5741                 prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2;
5742         prev_eff_load *= capacity_of(this_cpu);
5743 
5744         return this_eff_load <= prev_eff_load;
5745 }
5746 
5747 static int wake_affine(struct sched_domain *sd, struct task_struct *p,
5748                        int prev_cpu, int sync)
5749 {
5750         int this_cpu = smp_processor_id();
5751         bool affine = false;
5752 
5753         if (sched_feat(WA_IDLE) && !affine)
5754                 affine = wake_affine_idle(sd, p, this_cpu, prev_cpu, sync);
5755 
5756         if (sched_feat(WA_WEIGHT) && !affine)
5757                 affine = wake_affine_weight(sd, p, this_cpu, prev_cpu, sync);
5758 
5759         schedstat_inc(p->se.statistics.nr_wakeups_affine_attempts);
5760         if (affine) {
5761                 schedstat_inc(sd->ttwu_move_affine);
5762                 schedstat_inc(p->se.statistics.nr_wakeups_affine);
5763         }
5764 
5765         return affine;
5766 }
5767 
5768 static inline int task_util(struct task_struct *p);
5769 static int cpu_util_wake(int cpu, struct task_struct *p);
5770 
5771 static unsigned long capacity_spare_wake(int cpu, struct task_struct *p)
5772 {
5773         return capacity_orig_of(cpu) - cpu_util_wake(cpu, p);
5774 }
5775 
5776 /*
5777  * find_idlest_group finds and returns the least busy CPU group within the
5778  * domain.
5779  *
5780  * Assumes p is allowed on at least one CPU in sd.
5781  */
5782 static struct sched_group *
5783 find_idlest_group(struct sched_domain *sd, struct task_struct *p,
5784                   int this_cpu, int sd_flag)
5785 {
5786         struct sched_group *idlest = NULL, *group = sd->groups;
5787         struct sched_group *most_spare_sg = NULL;
5788         unsigned long min_runnable_load = ULONG_MAX;
5789         unsigned long this_runnable_load = ULONG_MAX;
5790         unsigned long min_avg_load = ULONG_MAX, this_avg_load = ULONG_MAX;
5791         unsigned long most_spare = 0, this_spare = 0;
5792         int load_idx = sd->forkexec_idx;
5793         int imbalance_scale = 100 + (sd->imbalance_pct-100)/2;
5794         unsigned long imbalance = scale_load_down(NICE_0_LOAD) *
5795                                 (sd->imbalance_pct-100) / 100;
5796 
5797         if (sd_flag & SD_BALANCE_WAKE)
5798                 load_idx = sd->wake_idx;
5799 
5800         do {
5801                 unsigned long load, avg_load, runnable_load;
5802                 unsigned long spare_cap, max_spare_cap;
5803                 int local_group;
5804                 int i;
5805 
5806                 /* Skip over this group if it has no CPUs allowed */
5807                 if (!cpumask_intersects(sched_group_span(group),
5808                                         &p->cpus_allowed))
5809                         continue;
5810 
5811                 local_group = cpumask_test_cpu(this_cpu,
5812                                                sched_group_span(group));
5813 
5814                 /*
5815                  * Tally up the load of all CPUs in the group and find
5816                  * the group containing the CPU with most spare capacity.
5817                  */
5818                 avg_load = 0;
5819                 runnable_load = 0;
5820                 max_spare_cap = 0;
5821 
5822                 for_each_cpu(i, sched_group_span(group)) {
5823                         /* Bias balancing toward cpus of our domain */
5824                         if (local_group)
5825                                 load = source_load(i, load_idx);
5826                         else
5827                                 load = target_load(i, load_idx);
5828 
5829                         runnable_load += load;
5830 
5831                         avg_load += cfs_rq_load_avg(&cpu_rq(i)->cfs);
5832 
5833                         spare_cap = capacity_spare_wake(i, p);
5834 
5835                         if (spare_cap > max_spare_cap)
5836                                 max_spare_cap = spare_cap;
5837                 }
5838 
5839                 /* Adjust by relative CPU capacity of the group */
5840                 avg_load = (avg_load * SCHED_CAPACITY_SCALE) /
5841                                         group->sgc->capacity;
5842                 runnable_load = (runnable_load * SCHED_CAPACITY_SCALE) /
5843                                         group->sgc->capacity;
5844 
5845                 if (local_group) {
5846                         this_runnable_load = runnable_load;
5847                         this_avg_load = avg_load;
5848                         this_spare = max_spare_cap;
5849                 } else {
5850                         if (min_runnable_load > (runnable_load + imbalance)) {
5851                                 /*
5852                                  * The runnable load is significantly smaller
5853                                  * so we can pick this new cpu
5854                                  */
5855                                 min_runnable_load = runnable_load;
5856                                 min_avg_load = avg_load;
5857                                 idlest = group;
5858                         } else if ((runnable_load < (min_runnable_load + imbalance)) &&
5859                                    (100*min_avg_load > imbalance_scale*avg_load)) {
5860                                 /*
5861                                  * The runnable loads are close so take the
5862                                  * blocked load into account through avg_load.
5863                                  */
5864                                 min_avg_load = avg_load;
5865                                 idlest = group;
5866                         }
5867 
5868                         if (most_spare < max_spare_cap) {
5869                                 most_spare = max_spare_cap;
5870                                 most_spare_sg = group;
5871                         }
5872                 }
5873         } while (group = group->next, group != sd->groups);
5874 
5875         /*
5876          * The cross-over point between using spare capacity or least load
5877          * is too conservative for high utilization tasks on partially
5878          * utilized systems if we require spare_capacity > task_util(p),
5879          * so we allow for some task stuffing by using
5880          * spare_capacity > task_util(p)/2.
5881          *
5882          * Spare capacity can't be used for fork because the utilization has
5883          * not been set yet, we must first select a rq to compute the initial
5884          * utilization.
5885          */
5886         if (sd_flag & SD_BALANCE_FORK)
5887                 goto skip_spare;
5888 
5889         if (this_spare > task_util(p) / 2 &&
5890             imbalance_scale*this_spare > 100*most_spare)
5891                 return NULL;
5892 
5893         if (most_spare > task_util(p) / 2)
5894                 return most_spare_sg;
5895 
5896 skip_spare:
5897         if (!idlest)
5898                 return NULL;
5899 
5900         if (min_runnable_load > (this_runnable_load + imbalance))
5901                 return NULL;
5902 
5903         if ((this_runnable_load < (min_runnable_load + imbalance)) &&
5904              (100*this_avg_load < imbalance_scale*min_avg_load))
5905                 return NULL;
5906 
5907         return idlest;
5908 }
5909 
5910 /*
5911  * find_idlest_group_cpu - find the idlest cpu among the cpus in group.
5912  */
5913 static int
5914 find_idlest_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
5915 {
5916         unsigned long load, min_load = ULONG_MAX;
5917         unsigned int min_exit_latency = UINT_MAX;
5918         u64 latest_idle_timestamp = 0;
5919         int least_loaded_cpu = this_cpu;
5920         int shallowest_idle_cpu = -1;
5921         int i;
5922 
5923         /* Check if we have any choice: */
5924         if (group->group_weight == 1)
5925                 return cpumask_first(sched_group_span(group));
5926 
5927         /* Traverse only the allowed CPUs */
5928         for_each_cpu_and(i, sched_group_span(group), &p->cpus_allowed) {
5929                 if (idle_cpu(i)) {
5930                         struct rq *rq = cpu_rq(i);
5931                         struct cpuidle_state *idle = idle_get_state(rq);
5932                         if (idle && idle->exit_latency < min_exit_latency) {
5933                                 /*
5934                                  * We give priority to a CPU whose idle state
5935                                  * has the smallest exit latency irrespective
5936                                  * of any idle timestamp.
5937                                  */
5938                                 min_exit_latency = idle->exit_latency;
5939                                 latest_idle_timestamp = rq->idle_stamp;
5940                                 shallowest_idle_cpu = i;
5941                         } else if ((!idle || idle->exit_latency == min_exit_latency) &&
5942                                    rq->idle_stamp > latest_idle_timestamp) {
5943                                 /*
5944                                  * If equal or no active idle state, then
5945                                  * the most recently idled CPU might have
5946                                  * a warmer cache.
5947                                  */
5948                                 latest_idle_timestamp = rq->idle_stamp;
5949                                 shallowest_idle_cpu = i;
5950                         }
5951                 } else if (shallowest_idle_cpu == -1) {
5952                         load = weighted_cpuload(cpu_rq(i));
5953                         if (load < min_load || (load == min_load && i == this_cpu)) {
5954                                 min_load = load;
5955                                 least_loaded_cpu = i;
5956                         }
5957                 }
5958         }
5959 
5960         return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu;
5961 }
5962 
5963 static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *p,
5964                                   int cpu, int prev_cpu, int sd_flag)
5965 {
5966         int new_cpu = cpu;
5967 
5968         if (!cpumask_intersects(sched_domain_span(sd), &p->cpus_allowed))
5969                 return prev_cpu;
5970 
5971         while (sd) {
5972                 struct sched_group *group;
5973                 struct sched_domain *tmp;
5974                 int weight;
5975 
5976                 if (!(sd->flags & sd_flag)) {
5977                         sd = sd->child;
5978                         continue;
5979                 }
5980 
5981                 group = find_idlest_group(sd, p, cpu, sd_flag);
5982                 if (!group) {
5983                         sd = sd->child;
5984                         continue;
5985                 }
5986 
5987                 new_cpu = find_idlest_group_cpu(group, p, cpu);
5988                 if (new_cpu == cpu) {
5989                         /* Now try balancing at a lower domain level of cpu */
5990                         sd = sd->child;
5991                         continue;
5992                 }
5993 
5994                 /* Now try balancing at a lower domain level of new_cpu */
5995                 cpu = new_cpu;
5996                 weight = sd->span_weight;
5997                 sd = NULL;
5998                 for_each_domain(cpu, tmp) {
5999                         if (weight <= tmp->span_weight)
6000                                 break;
6001                         if (tmp->flags & sd_flag)
6002                                 sd = tmp;
6003                 }
6004                 /* while loop will break here if sd == NULL */
6005         }
6006 
6007         return new_cpu;
6008 }
6009 
6010 #ifdef CONFIG_SCHED_SMT
6011 
6012 static inline void set_idle_cores(int cpu, int val)
6013 {
6014         struct sched_domain_shared *sds;
6015 
6016         sds = rcu_dereference(per_cpu(sd_llc_shared, cpu));
6017         if (sds)
6018                 WRITE_ONCE(sds->has_idle_cores, val);
6019 }
6020 
6021 static inline bool test_idle_cores(int cpu, bool def)
6022 {
6023         struct sched_domain_shared *sds;
6024 
6025         sds = rcu_dereference(per_cpu(sd_llc_shared, cpu));
6026         if (sds)
6027                 return READ_ONCE(sds->has_idle_cores);
6028 
6029         return def;
6030 }
6031 
6032 /*
6033  * Scans the local SMT mask to see if the entire core is idle, and records this
6034  * information in sd_llc_shared->has_idle_cores.
6035  *
6036  * Since SMT siblings share all cache levels, inspecting this limited remote
6037  * state should be fairly cheap.
6038  */
6039 void __update_idle_core(struct rq *rq)
6040 {
6041         int core = cpu_of(rq);
6042         int cpu;
6043 
6044         rcu_read_lock();
6045         if (test_idle_cores(core, true))
6046                 goto unlock;
6047 
6048         for_each_cpu(cpu, cpu_smt_mask(core)) {
6049                 if (cpu == core)
6050                         continue;
6051 
6052                 if (!idle_cpu(cpu))
6053                         goto unlock;
6054         }
6055 
6056         set_idle_cores(core, 1);
6057 unlock:
6058         rcu_read_unlock();
6059 }
6060 
6061 /*
6062  * Scan the entire LLC domain for idle cores; this dynamically switches off if
6063  * there are no idle cores left in the system; tracked through
6064  * sd_llc->shared->has_idle_cores and enabled through update_idle_core() above.
6065  */
6066 static int select_idle_core(struct task_struct *p, struct sched_domain *sd, int target)
6067 {
6068         struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_idle_mask);
6069         int core, cpu;
6070 
6071         if (!static_branch_likely(&sched_smt_present))
6072                 return -1;
6073 
6074         if (!test_idle_cores(target, false))
6075                 return -1;
6076 
6077         cpumask_and(cpus, sched_domain_span(sd), &p->cpus_allowed);
6078 
6079         for_each_cpu_wrap(core, cpus, target) {
6080                 bool idle = true;
6081 
6082                 for_each_cpu(cpu, cpu_smt_mask(core)) {
6083                         cpumask_clear_cpu(cpu, cpus);
6084                         if (!idle_cpu(cpu))
6085                                 idle = false;
6086                 }
6087 
6088                 if (idle)
6089                         return core;
6090         }
6091 
6092         /*
6093          * Failed to find an idle core; stop looking for one.
6094          */
6095         set_idle_cores(target, 0);
6096 
6097         return -1;
6098 }
6099 
6100 /*
6101  * Scan the local SMT mask for idle CPUs.
6102  */
6103 static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target)
6104 {
6105         int cpu;
6106 
6107         if (!static_branch_likely(&sched_smt_present))
6108                 return -1;
6109 
6110         for_each_cpu(cpu, cpu_smt_mask(target)) {
6111                 if (!cpumask_test_cpu(cpu, &p->cpus_allowed))
6112                         continue;
6113                 if (idle_cpu(cpu))
6114                         return cpu;
6115         }
6116 
6117         return -1;
6118 }
6119 
6120 #else /* CONFIG_SCHED_SMT */
6121 
6122 static inline int select_idle_core(struct task_struct *p, struct sched_domain *sd, int target)
6123 {
6124         return -1;
6125 }
6126 
6127 static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target)
6128 {
6129         return -1;
6130 }
6131 
6132 #endif /* CONFIG_SCHED_SMT */
6133 
6134 /*
6135  * Scan the LLC domain for idle CPUs; this is dynamically regulated by
6136  * comparing the average scan cost (tracked in sd->avg_scan_cost) against the
6137  * average idle time for this rq (as found in rq->avg_idle).
6138  */
6139 static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int target)
6140 {
6141         struct sched_domain *this_sd;
6142         u64 avg_cost, avg_idle;
6143         u64 time, cost;
6144         s64 delta;
6145         int cpu, nr = INT_MAX;
6146 
6147         this_sd = rcu_dereference(*this_cpu_ptr(&sd_llc));
6148         if (!this_sd)
6149                 return -1;
6150 
6151         /*
6152          * Due to large variance we need a large fuzz factor; hackbench in
6153          * particularly is sensitive here.
6154          */
6155         avg_idle = this_rq()->avg_idle / 512;
6156         avg_cost = this_sd->avg_scan_cost + 1;
6157 
6158         if (sched_feat(SIS_AVG_CPU) && avg_idle < avg_cost)
6159                 return -1;
6160 
6161         if (sched_feat(SIS_PROP)) {
6162                 u64 span_avg = sd->span_weight * avg_idle;
6163                 if (span_avg > 4*avg_cost)
6164                         nr = div_u64(span_avg, avg_cost);
6165                 else
6166                         nr = 4;
6167         }
6168 
6169         time = local_clock();
6170 
6171         for_each_cpu_wrap(cpu, sched_domain_span(sd), target) {
6172                 if (!--nr)
6173                         return -1;
6174                 if (!cpumask_test_cpu(cpu, &p->cpus_allowed))
6175                         continue;
6176                 if (idle_cpu(cpu))
6177                         break;
6178         }
6179 
6180         time = local_clock() - time;
6181         cost = this_sd->avg_scan_cost;
6182         delta = (s64)(time - cost) / 8;
6183         this_sd->avg_scan_cost += delta;
6184 
6185         return cpu;
6186 }
6187 
6188 /*
6189  * Try and locate an idle core/thread in the LLC cache domain.
6190  */
6191 static int select_idle_sibling(struct task_struct *p, int prev, int target)
6192 {
6193         struct sched_domain *sd;
6194         int i;
6195 
6196         if (idle_cpu(target))
6197                 return target;
6198 
6199         /*
6200          * If the previous cpu is cache affine and idle, don't be stupid.
6201          */
6202         if (prev != target && cpus_share_cache(prev, target) && idle_cpu(prev))
6203                 return prev;
6204 
6205         sd = rcu_dereference(per_cpu(sd_llc, target));
6206         if (!sd)
6207                 return target;
6208 
6209         i = select_idle_core(p, sd, target);
6210         if ((unsigned)i < nr_cpumask_bits)
6211                 return i;
6212 
6213         i = select_idle_cpu(p, sd, target);
6214         if ((unsigned)i < nr_cpumask_bits)
6215                 return i;
6216 
6217         i = select_idle_smt(p, sd, target);
6218         if ((unsigned)i < nr_cpumask_bits)
6219                 return i;
6220 
6221         return target;
6222 }
6223 
6224 /*
6225  * cpu_util returns the amount of capacity of a CPU that is used by CFS
6226  * tasks. The unit of the return value must be the one of capacity so we can
6227  * compare the utilization with the capacity of the CPU that is available for
6228  * CFS task (ie cpu_capacity).
6229  *
6230  * cfs_rq.avg.util_avg is the sum of running time of runnable tasks plus the
6231  * recent utilization of currently non-runnable tasks on a CPU. It represents
6232  * the amount of utilization of a CPU in the range [0..capacity_orig] where
6233  * capacity_orig is the cpu_capacity available at the highest frequency
6234  * (arch_scale_freq_capacity()).
6235  * The utilization of a CPU converges towards a sum equal to or less than the
6236  * current capacity (capacity_curr <= capacity_orig) of the CPU because it is
6237  * the running time on this CPU scaled by capacity_curr.
6238  *
6239  * Nevertheless, cfs_rq.avg.util_avg can be higher than capacity_curr or even
6240  * higher than capacity_orig because of unfortunate rounding in
6241  * cfs.avg.util_avg or just after migrating tasks and new task wakeups until
6242  * the average stabilizes with the new running time. We need to check that the
6243  * utilization stays within the range of [0..capacity_orig] and cap it if
6244  * necessary. Without utilization capping, a group could be seen as overloaded
6245  * (CPU0 utilization at 121% + CPU1 utilization at 80%) whereas CPU1 has 20% of
6246  * available capacity. We allow utilization to overshoot capacity_curr (but not
6247  * capacity_orig) as it useful for predicting the capacity required after task
6248  * migrations (scheduler-driven DVFS).
6249  */
6250 static int cpu_util(int cpu)
6251 {
6252         unsigned long util = cpu_rq(cpu)->cfs.avg.util_avg;
6253         unsigned long capacity = capacity_orig_of(cpu);
6254 
6255         return (util >= capacity) ? capacity : util;
6256 }
6257 
6258 static inline int task_util(struct task_struct *p)
6259 {
6260         return p->se.avg.util_avg;
6261 }
6262 
6263 /*
6264  * cpu_util_wake: Compute cpu utilization with any contributions from
6265  * the waking task p removed.
6266  */
6267 static int cpu_util_wake(int cpu, struct task_struct *p)
6268 {
6269         unsigned long util, capacity;
6270 
6271         /* Task has no contribution or is new */
6272         if (cpu != task_cpu(p) || !p->se.avg.last_update_time)
6273                 return cpu_util(cpu);
6274 
6275         capacity = capacity_orig_of(cpu);
6276         util = max_t(long, cpu_rq(cpu)->cfs.avg.util_avg - task_util(p), 0);
6277 
6278         return (util >= capacity) ? capacity : util;
6279 }
6280 
6281 /*
6282  * Disable WAKE_AFFINE in the case where task @p doesn't fit in the
6283  * capacity of either the waking CPU @cpu or the previous CPU @prev_cpu.
6284  *
6285  * In that case WAKE_AFFINE doesn't make sense and we'll let
6286  * BALANCE_WAKE sort things out.
6287  */
6288 static int wake_cap(struct task_struct *p, int cpu, int prev_cpu)
6289 {
6290         long min_cap, max_cap;
6291 
6292         min_cap = min(capacity_orig_of(prev_cpu), capacity_orig_of(cpu));
6293         max_cap = cpu_rq(cpu)->rd->max_cpu_capacity;
6294 
6295         /* Minimum capacity is close to max, no need to abort wake_affine */
6296         if (max_cap - min_cap < max_cap >> 3)
6297                 return 0;
6298 
6299         /* Bring task utilization in sync with prev_cpu */
6300         sync_entity_load_avg(&p->se);
6301 
6302         return min_cap * 1024 < task_util(p) * capacity_margin;
6303 }
6304 
6305 /*
6306  * select_task_rq_fair: Select target runqueue for the waking task in domains
6307  * that have the 'sd_flag' flag set. In practice, this is SD_BALANCE_WAKE,
6308  * SD_BALANCE_FORK, or SD_BALANCE_EXEC.
6309  *
6310  * Balances load by selecting the idlest cpu in the idlest group, or under
6311  * certain conditions an idle sibling cpu if the domain has SD_WAKE_AFFINE set.
6312  *
6313  * Returns the target cpu number.
6314  *
6315  * preempt must be disabled.
6316  */
6317 static int
6318 select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags)
6319 {
6320         struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL;
6321         int cpu = smp_processor_id();
6322         int new_cpu = prev_cpu;
6323         int want_affine = 0;
6324         int sync = wake_flags & WF_SYNC;
6325 
6326         if (sd_flag & SD_BALANCE_WAKE) {
6327                 record_wakee(p);
6328                 want_affine = !wake_wide(p) && !wake_cap(p, cpu, prev_cpu)
6329                               && cpumask_test_cpu(cpu, &p->cpus_allowed);
6330         }
6331 
6332         rcu_read_lock();
6333         for_each_domain(cpu, tmp) {
6334                 if (!(tmp->flags & SD_LOAD_BALANCE))
6335                         break;
6336 
6337                 /*
6338                  * If both cpu and prev_cpu are part of this domain,
6339                  * cpu is a valid SD_WAKE_AFFINE target.
6340                  */
6341                 if (want_affine && (tmp->flags & SD_WAKE_AFFINE) &&
6342                     cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) {
6343                         affine_sd = tmp;
6344                         break;
6345                 }
6346 
6347                 if (tmp->flags & sd_flag)
6348                         sd = tmp;
6349                 else if (!want_affine)
6350                         break;
6351         }
6352 
6353         if (affine_sd) {
6354                 sd = NULL; /* Prefer wake_affine over balance flags */
6355                 if (cpu == prev_cpu)
6356                         goto pick_cpu;
6357 
6358                 if (wake_affine(affine_sd, p, prev_cpu, sync))
6359                         new_cpu = cpu;
6360         }
6361 
6362         if (sd && !(sd_flag & SD_BALANCE_FORK)) {
6363                 /*
6364                  * We're going to need the task's util for capacity_spare_wake
6365                  * in find_idlest_group. Sync it up to prev_cpu's
6366                  * last_update_time.
6367                  */
6368                 sync_entity_load_avg(&p->se);
6369         }
6370 
6371         if (!sd) {
6372 pick_cpu:
6373                 if (sd_flag & SD_BALANCE_WAKE) /* XXX always ? */
6374                         new_cpu = select_idle_sibling(p, prev_cpu, new_cpu);
6375 
6376         } else {
6377                 new_cpu = find_idlest_cpu(sd, p, cpu, prev_cpu, sd_flag);
6378         }
6379         rcu_read_unlock();
6380 
6381         return new_cpu;
6382 }
6383 
6384 static void detach_entity_cfs_rq(struct sched_entity *se);
6385 
6386 /*
6387  * Called immediately before a task is migrated to a new cpu; task_cpu(p) and
6388  * cfs_rq_of(p) references at time of call are still valid and identify the
6389  * previous cpu. The caller guarantees p->pi_lock or task_rq(p)->lock is held.
6390  */
6391 static void migrate_task_rq_fair(struct task_struct *p)
6392 {
6393         /*
6394          * As blocked tasks retain absolute vruntime the migration needs to
6395          * deal with this by subtracting the old and adding the new
6396          * min_vruntime -- the latter is done by enqueue_entity() when placing
6397          * the task on the new runqueue.
6398          */
6399         if (p->state == TASK_WAKING) {
6400                 struct sched_entity *se = &p->se;
6401                 struct cfs_rq *cfs_rq = cfs_rq_of(se);
6402                 u64 min_vruntime;
6403 
6404 #ifndef CONFIG_64BIT
6405                 u64 min_vruntime_copy;
6406 
6407                 do {
6408                         min_vruntime_copy = cfs_rq->min_vruntime_copy;
6409                         smp_rmb();
6410                         min_vruntime = cfs_rq->min_vruntime;
6411                 } while (min_vruntime != min_vruntime_copy);
6412 #else
6413                 min_vruntime = cfs_rq->min_vruntime;
6414 #endif
6415 
6416                 se->vruntime -= min_vruntime;
6417         }
6418 
6419         if (p->on_rq == TASK_ON_RQ_MIGRATING) {
6420                 /*
6421                  * In case of TASK_ON_RQ_MIGRATING we in fact hold the 'old'
6422                  * rq->lock and can modify state directly.
6423                  */
6424                 lockdep_assert_held(&task_rq(p)->lock);
6425                 detach_entity_cfs_rq(&p->se);
6426 
6427         } else {
6428                 /*
6429                  * We are supposed to update the task to "current" time, then
6430                  * its up to date and ready to go to new CPU/cfs_rq. But we
6431                  * have difficulty in getting what current time is, so simply
6432                  * throw away the out-of-date time. This will result in the
6433                  * wakee task is less decayed, but giving the wakee more load
6434                  * sounds not bad.
6435                  */
6436                 remove_entity_load_avg(&p->se);
6437         }
6438 
6439         /* Tell new CPU we are migrated */
6440         p->se.avg.last_update_time = 0;
6441 
6442         /* We have migrated, no longer consider this task hot */
6443         p->se.exec_start = 0;
6444 }
6445 
6446 static void task_dead_fair(struct task_struct *p)
6447 {
6448         remove_entity_load_avg(&p->se);
6449 }
6450 #endif /* CONFIG_SMP */
6451 
6452 static unsigned long
6453 wakeup_gran(struct sched_entity *curr, struct sched_entity *se)
6454 {
6455         unsigned long gran = sysctl_sched_wakeup_granularity;
6456 
6457         /*
6458          * Since its curr running now, convert the gran from real-time
6459          * to virtual-time in his units.
6460          *
6461          * By using 'se' instead of 'curr' we penalize light tasks, so
6462          * they get preempted easier. That is, if 'se' < 'curr' then
6463          * the resulting gran will be larger, therefore penalizing the
6464          * lighter, if otoh 'se' > 'curr' then the resulting gran will
6465          * be smaller, again penalizing the lighter task.
6466          *
6467          * This is especially important for buddies when the leftmost
6468          * task is higher priority than the buddy.
6469          */
6470         return calc_delta_fair(gran, se);
6471 }
6472 
6473 /*