1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH) 4 * 5 * Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> 6 * 7 * Interactivity improvements by Mike Galbraith 8 * (C) 2007 Mike Galbraith <efault@gmx.de> 9 * 10 * Various enhancements by Dmitry Adamushko. 11 * (C) 2007 Dmitry Adamushko <dmitry.adamushko@gmail.com> 12 * 13 * Group scheduling enhancements by Srivatsa Vaddagiri 14 * Copyright IBM Corporation, 2007 15 * Author: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com> 16 * 17 * Scaled math optimizations by Thomas Gleixner 18 * Copyright (C) 2007, Thomas Gleixner <tglx@linutronix.de> 19 * 20 * Adaptive scheduling granularity, math enhancements by Peter Zijlstra 21 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra 22 */ 23 #include "sched.h" 24 25 #include <trace/events/sched.h> 26 27 /* 28 * Targeted preemption latency for CPU-bound tasks: 29 * 30 * NOTE: this latency value is not the same as the concept of 31 * 'timeslice length' - timeslices in CFS are of variable length 32 * and have no persistent notion like in traditional, time-slice 33 * based scheduling concepts. 34 * 35 * (to see the precise effective timeslice length of your workload, 36 * run vmstat and monitor the context-switches (cs) field) 37 * 38 * (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds) 39 */ 40 unsigned int sysctl_sched_latency = 6000000ULL; 41 unsigned int normalized_sysctl_sched_latency = 6000000ULL; 42 43 /* 44 * The initial- and re-scaling of tunables is configurable 45 * 46 * Options are: 47 * 48 * SCHED_TUNABLESCALING_NONE - unscaled, always *1 49 * SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus) 50 * SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus 51 * 52 * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus)) 53 */ 54 enum sched_tunable_scaling sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; 55 56 /* 57 * Minimal preemption granularity for CPU-bound tasks: 58 * 59 * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds) 60 */ 61 unsigned int sysctl_sched_min_granularity = 750000ULL; 62 unsigned int normalized_sysctl_sched_min_granularity = 750000ULL; 63 64 /* 65 * This value is kept at sysctl_sched_latency/sysctl_sched_min_granularity 66 */ 67 static unsigned int sched_nr_latency = 8; 68 69 /* 70 * After fork, child runs first. If set to 0 (default) then 71 * parent will (try to) run first. 72 */ 73 unsigned int sysctl_sched_child_runs_first __read_mostly; 74 75 /* 76 * SCHED_OTHER wake-up granularity. 77 * 78 * This option delays the preemption effects of decoupled workloads 79 * and reduces their over-scheduling. Synchronous workloads will still 80 * have immediate wakeup/sleep latencies. 81 * 82 * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds) 83 */ 84 unsigned int sysctl_sched_wakeup_granularity = 1000000UL; 85 unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL; 86 87 const_debug unsigned int sysctl_sched_migration_cost = 500000UL; 88 89 #ifdef CONFIG_SMP 90 /* 91 * For asym packing, by default the lower numbered CPU has higher priority. 92 */ 93 int __weak arch_asym_cpu_priority(int cpu) 94 { 95 return -cpu; 96 } 97 #endif 98 99 #ifdef CONFIG_CFS_BANDWIDTH 100 /* 101 * Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool 102 * each time a cfs_rq requests quota. 103 * 104 * Note: in the case that the slice exceeds the runtime remaining (either due 105 * to consumption or the quota being specified to be smaller than the slice) 106 * we will always only issue the remaining available time. 107 * 108 * (default: 5 msec, units: microseconds) 109 */ 110 unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; 111 #endif 112 113 /* 114 * The margin used when comparing utilization with CPU capacity: 115 * util * margin < capacity * 1024 116 * 117 * (default: ~20%) 118 */ 119 unsigned int capacity_margin = 1280; 120 121 static inline void update_load_add(struct load_weight *lw, unsigned long inc) 122 { 123 lw->weight += inc; 124 lw->inv_weight = 0; 125 } 126 127 static inline void update_load_sub(struct load_weight *lw, unsigned long dec) 128 { 129 lw->weight -= dec; 130 lw->inv_weight = 0; 131 } 132 133 static inline void update_load_set(struct load_weight *lw, unsigned long w) 134 { 135 lw->weight = w; 136 lw->inv_weight = 0; 137 } 138 139 /* 140 * Increase the granularity value when there are more CPUs, 141 * because with more CPUs the 'effective latency' as visible 142 * to users decreases. But the relationship is not linear, 143 * so pick a second-best guess by going with the log2 of the 144 * number of CPUs. 145 * 146 * This idea comes from the SD scheduler of Con Kolivas: 147 */ 148 static unsigned int get_update_sysctl_factor(void) 149 { 150 unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8); 151 unsigned int factor; 152 153 switch (sysctl_sched_tunable_scaling) { 154 case SCHED_TUNABLESCALING_NONE: 155 factor = 1; 156 break; 157 case SCHED_TUNABLESCALING_LINEAR: 158 factor = cpus; 159 break; 160 case SCHED_TUNABLESCALING_LOG: 161 default: 162 factor = 1 + ilog2(cpus); 163 break; 164 } 165 166 return factor; 167 } 168 169 static void update_sysctl(void) 170 { 171 unsigned int factor = get_update_sysctl_factor(); 172 173 #define SET_SYSCTL(name) \ 174 (sysctl_##name = (factor) * normalized_sysctl_##name) 175 SET_SYSCTL(sched_min_granularity); 176 SET_SYSCTL(sched_latency); 177 SET_SYSCTL(sched_wakeup_granularity); 178 #undef SET_SYSCTL 179 } 180 181 void sched_init_granularity(void) 182 { 183 update_sysctl(); 184 } 185 186 #define WMULT_CONST (~0U) 187 #define WMULT_SHIFT 32 188 189 static void __update_inv_weight(struct load_weight *lw) 190 { 191 unsigned long w; 192 193 if (likely(lw->inv_weight)) 194 return; 195 196 w = scale_load_down(lw->weight); 197 198 if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST)) 199 lw->inv_weight = 1; 200 else if (unlikely(!w)) 201 lw->inv_weight = WMULT_CONST; 202 else 203 lw->inv_weight = WMULT_CONST / w; 204 } 205 206 /* 207 * delta_exec * weight / lw.weight 208 * OR 209 * (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT 210 * 211 * Either weight := NICE_0_LOAD and lw \e sched_prio_to_wmult[], in which case 212 * we're guaranteed shift stays positive because inv_weight is guaranteed to 213 * fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22. 214 * 215 * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus 216 * weight/lw.weight <= 1, and therefore our shift will also be positive. 217 */ 218 static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw) 219 { 220 u64 fact = scale_load_down(weight); 221 int shift = WMULT_SHIFT; 222 223 __update_inv_weight(lw); 224 225 if (unlikely(fact >> 32)) { 226 while (fact >> 32) { 227 fact >>= 1; 228 shift--; 229 } 230 } 231 232 /* hint to use a 32x32->64 mul */ 233 fact = (u64)(u32)fact * lw->inv_weight; 234 235 while (fact >> 32) { 236 fact >>= 1; 237 shift--; 238 } 239 240 return mul_u64_u32_shr(delta_exec, fact, shift); 241 } 242 243 244 const struct sched_class fair_sched_class; 245 246 /************************************************************** 247 * CFS operations on generic schedulable entities: 248 */ 249 250 #ifdef CONFIG_FAIR_GROUP_SCHED 251 252 /* cpu runqueue to which this cfs_rq is attached */ 253 static inline struct rq *rq_of(struct cfs_rq *cfs_rq) 254 { 255 return cfs_rq->rq; 256 } 257 258 /* An entity is a task if it doesn't "own" a runqueue */ 259 #define entity_is_task(se) (!se->my_q) 260 261 static inline struct task_struct *task_of(struct sched_entity *se) 262 { 263 SCHED_WARN_ON(!entity_is_task(se)); 264 return container_of(se, struct task_struct, se); 265 } 266 267 /* Walk up scheduling entities hierarchy */ 268 #define for_each_sched_entity(se) \ 269 for (; se; se = se->parent) 270 271 static inline struct cfs_rq *task_cfs_rq(struct task_struct *p) 272 { 273 return p->se.cfs_rq; 274 } 275 276 /* runqueue on which this entity is (to be) queued */ 277 static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se) 278 { 279 return se->cfs_rq; 280 } 281 282 /* runqueue "owned" by this group */ 283 static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp) 284 { 285 return grp->my_q; 286 } 287 288 static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 289 { 290 if (!cfs_rq->on_list) { 291 struct rq *rq = rq_of(cfs_rq); 292 int cpu = cpu_of(rq); 293 /* 294 * Ensure we either appear before our parent (if already 295 * enqueued) or force our parent to appear after us when it is 296 * enqueued. The fact that we always enqueue bottom-up 297 * reduces this to two cases and a special case for the root 298 * cfs_rq. Furthermore, it also means that we will always reset 299 * tmp_alone_branch either when the branch is connected 300 * to a tree or when we reach the beg of the tree 301 */ 302 if (cfs_rq->tg->parent && 303 cfs_rq->tg->parent->cfs_rq[cpu]->on_list) { 304 /* 305 * If parent is already on the list, we add the child 306 * just before. Thanks to circular linked property of 307 * the list, this means to put the child at the tail 308 * of the list that starts by parent. 309 */ 310 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 311 &(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list)); 312 /* 313 * The branch is now connected to its tree so we can 314 * reset tmp_alone_branch to the beginning of the 315 * list. 316 */ 317 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 318 } else if (!cfs_rq->tg->parent) { 319 /* 320 * cfs rq without parent should be put 321 * at the tail of the list. 322 */ 323 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 324 &rq->leaf_cfs_rq_list); 325 /* 326 * We have reach the beg of a tree so we can reset 327 * tmp_alone_branch to the beginning of the list. 328 */ 329 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 330 } else { 331 /* 332 * The parent has not already been added so we want to 333 * make sure that it will be put after us. 334 * tmp_alone_branch points to the beg of the branch 335 * where we will add parent. 336 */ 337 list_add_rcu(&cfs_rq->leaf_cfs_rq_list, 338 rq->tmp_alone_branch); 339 /* 340 * update tmp_alone_branch to points to the new beg 341 * of the branch 342 */ 343 rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list; 344 } 345 346 cfs_rq->on_list = 1; 347 } 348 } 349 350 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 351 { 352 if (cfs_rq->on_list) { 353 list_del_rcu(&cfs_rq->leaf_cfs_rq_list); 354 cfs_rq->on_list = 0; 355 } 356 } 357 358 /* Iterate thr' all leaf cfs_rq's on a runqueue */ 359 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ 360 list_for_each_entry_safe(cfs_rq, pos, &rq->leaf_cfs_rq_list, \ 361 leaf_cfs_rq_list) 362 363 /* Do the two (enqueued) entities belong to the same group ? */ 364 static inline struct cfs_rq * 365 is_same_group(struct sched_entity *se, struct sched_entity *pse) 366 { 367 if (se->cfs_rq == pse->cfs_rq) 368 return se->cfs_rq; 369 370 return NULL; 371 } 372 373 static inline struct sched_entity *parent_entity(struct sched_entity *se) 374 { 375 return se->parent; 376 } 377 378 static void 379 find_matching_se(struct sched_entity **se, struct sched_entity **pse) 380 { 381 int se_depth, pse_depth; 382 383 /* 384 * preemption test can be made between sibling entities who are in the 385 * same cfs_rq i.e who have a common parent. Walk up the hierarchy of 386 * both tasks until we find their ancestors who are siblings of common 387 * parent. 388 */ 389 390 /* First walk up until both entities are at same depth */ 391 se_depth = (*se)->depth; 392 pse_depth = (*pse)->depth; 393 394 while (se_depth > pse_depth) { 395 se_depth--; 396 *se = parent_entity(*se); 397 } 398 399 while (pse_depth > se_depth) { 400 pse_depth--; 401 *pse = parent_entity(*pse); 402 } 403 404 while (!is_same_group(*se, *pse)) { 405 *se = parent_entity(*se); 406 *pse = parent_entity(*pse); 407 } 408 } 409 410 #else /* !CONFIG_FAIR_GROUP_SCHED */ 411 412 static inline struct task_struct *task_of(struct sched_entity *se) 413 { 414 return container_of(se, struct task_struct, se); 415 } 416 417 static inline struct rq *rq_of(struct cfs_rq *cfs_rq) 418 { 419 return container_of(cfs_rq, struct rq, cfs); 420 } 421 422 #define entity_is_task(se) 1 423 424 #define for_each_sched_entity(se) \ 425 for (; se; se = NULL) 426 427 static inline struct cfs_rq *task_cfs_rq(struct task_struct *p) 428 { 429 return &task_rq(p)->cfs; 430 } 431 432 static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se) 433 { 434 struct task_struct *p = task_of(se); 435 struct rq *rq = task_rq(p); 436 437 return &rq->cfs; 438 } 439 440 /* runqueue "owned" by this group */ 441 static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp) 442 { 443 return NULL; 444 } 445 446 static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 447 { 448 } 449 450 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 451 { 452 } 453 454 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ 455 for (cfs_rq = &rq->cfs, pos = NULL; cfs_rq; cfs_rq = pos) 456 457 static inline struct sched_entity *parent_entity(struct sched_entity *se) 458 { 459 return NULL; 460 } 461 462 static inline void 463 find_matching_se(struct sched_entity **se, struct sched_entity **pse) 464 { 465 } 466 467 #endif /* CONFIG_FAIR_GROUP_SCHED */ 468 469 static __always_inline 470 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); 471 472 /************************************************************** 473 * Scheduling class tree data structure manipulation methods: 474 */ 475 476 static inline u64 max_vruntime(u64 max_vruntime, u64 vruntime) 477 { 478 s64 delta = (s64)(vruntime - max_vruntime); 479 if (delta > 0) 480 max_vruntime = vruntime; 481 482 return max_vruntime; 483 } 484 485 static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime) 486 { 487 s64 delta = (s64)(vruntime - min_vruntime); 488 if (delta < 0) 489 min_vruntime = vruntime; 490 491 return min_vruntime; 492 } 493 494 static inline int entity_before(struct sched_entity *a, 495 struct sched_entity *b) 496 { 497 return (s64)(a->vruntime - b->vruntime) < 0; 498 } 499 500 static void update_min_vruntime(struct cfs_rq *cfs_rq) 501 { 502 struct sched_entity *curr = cfs_rq->curr; 503 struct rb_node *leftmost = rb_first_cached(&cfs_rq->tasks_timeline); 504 505 u64 vruntime = cfs_rq->min_vruntime; 506 507 if (curr) { 508 if (curr->on_rq) 509 vruntime = curr->vruntime; 510 else 511 curr = NULL; 512 } 513 514 if (leftmost) { /* non-empty tree */ 515 struct sched_entity *se; 516 se = rb_entry(leftmost, struct sched_entity, run_node); 517 518 if (!curr) 519 vruntime = se->vruntime; 520 else 521 vruntime = min_vruntime(vruntime, se->vruntime); 522 } 523 524 /* ensure we never gain time by being placed backwards. */ 525 cfs_rq->min_vruntime = max_vruntime(cfs_rq->min_vruntime, vruntime); 526 #ifndef CONFIG_64BIT 527 smp_wmb(); 528 cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime; 529 #endif 530 } 531 532 /* 533 * Enqueue an entity into the rb-tree: 534 */ 535 static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 536 { 537 struct rb_node **link = &cfs_rq->tasks_timeline.rb_root.rb_node; 538 struct rb_node *parent = NULL; 539 struct sched_entity *entry; 540 bool leftmost = true; 541 542 /* 543 * Find the right place in the rbtree: 544 */ 545 while (*link) { 546 parent = *link; 547 entry = rb_entry(parent, struct sched_entity, run_node); 548 /* 549 * We dont care about collisions. Nodes with 550 * the same key stay together. 551 */ 552 if (entity_before(se, entry)) { 553 link = &parent->rb_left; 554 } else { 555 link = &parent->rb_right; 556 leftmost = false; 557 } 558 } 559 560 rb_link_node(&se->run_node, parent, link); 561 rb_insert_color_cached(&se->run_node, 562 &cfs_rq->tasks_timeline, leftmost); 563 } 564 565 static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 566 { 567 rb_erase_cached(&se->run_node, &cfs_rq->tasks_timeline); 568 } 569 570 struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) 571 { 572 struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline); 573 574 if (!left) 575 return NULL; 576 577 return rb_entry(left, struct sched_entity, run_node); 578 } 579 580 static struct sched_entity *__pick_next_entity(struct sched_entity *se) 581 { 582 struct rb_node *next = rb_next(&se->run_node); 583 584 if (!next) 585 return NULL; 586 587 return rb_entry(next, struct sched_entity, run_node); 588 } 589 590 #ifdef CONFIG_SCHED_DEBUG 591 struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) 592 { 593 struct rb_node *last = rb_last(&cfs_rq->tasks_timeline.rb_root); 594 595 if (!last) 596 return NULL; 597 598 return rb_entry(last, struct sched_entity, run_node); 599 } 600 601 /************************************************************** 602 * Scheduling class statistics methods: 603 */ 604 605 int sched_proc_update_handler(struct ctl_table *table, int write, 606 void __user *buffer, size_t *lenp, 607 loff_t *ppos) 608 { 609 int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 610 unsigned int factor = get_update_sysctl_factor(); 611 612 if (ret || !write) 613 return ret; 614 615 sched_nr_latency = DIV_ROUND_UP(sysctl_sched_latency, 616 sysctl_sched_min_granularity); 617 618 #define WRT_SYSCTL(name) \ 619 (normalized_sysctl_##name = sysctl_##name / (factor)) 620 WRT_SYSCTL(sched_min_granularity); 621 WRT_SYSCTL(sched_latency); 622 WRT_SYSCTL(sched_wakeup_granularity); 623 #undef WRT_SYSCTL 624 625 return 0; 626 } 627 #endif 628 629 /* 630 * delta /= w 631 */ 632 static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) 633 { 634 if (unlikely(se->load.weight != NICE_0_LOAD)) 635 delta = __calc_delta(delta, NICE_0_LOAD, &se->load); 636 637 return delta; 638 } 639 640 /* 641 * The idea is to set a period in which each task runs once. 642 * 643 * When there are too many tasks (sched_nr_latency) we have to stretch 644 * this period because otherwise the slices get too small. 645 * 646 * p = (nr <= nl) ? l : l*nr/nl 647 */ 648 static u64 __sched_period(unsigned long nr_running) 649 { 650 if (unlikely(nr_running > sched_nr_latency)) 651 return nr_running * sysctl_sched_min_granularity; 652 else 653 return sysctl_sched_latency; 654 } 655 656 /* 657 * We calculate the wall-time slice from the period by taking a part 658 * proportional to the weight. 659 * 660 * s = p*P[w/rw] 661 */ 662 static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) 663 { 664 u64 slice = __sched_period(cfs_rq->nr_running + !se->on_rq); 665 666 for_each_sched_entity(se) { 667 struct load_weight *load; 668 struct load_weight lw; 669 670 cfs_rq = cfs_rq_of(se); 671 load = &cfs_rq->load; 672 673 if (unlikely(!se->on_rq)) { 674 lw = cfs_rq->load; 675 676 update_load_add(&lw, se->load.weight); 677 load = &lw; 678 } 679 slice = __calc_delta(slice, se->load.weight, load); 680 } 681 return slice; 682 } 683 684 /* 685 * We calculate the vruntime slice of a to-be-inserted task. 686 * 687 * vs = s/w 688 */ 689 static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se) 690 { 691 return calc_delta_fair(sched_slice(cfs_rq, se), se); 692 } 693 694 #ifdef CONFIG_SMP 695 696 #include "sched-pelt.h" 697 698 static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu); 699 static unsigned long task_h_load(struct task_struct *p); 700 701 /* Give new sched_entity start runnable values to heavy its load in infant time */ 702 void init_entity_runnable_average(struct sched_entity *se) 703 { 704 struct sched_avg *sa = &se->avg; 705 706 memset(sa, 0, sizeof(*sa)); 707 708 /* 709 * Tasks are intialized with full load to be seen as heavy tasks until 710 * they get a chance to stabilize to their real load level. 711 * Group entities are intialized with zero load to reflect the fact that 712 * nothing has been attached to the task group yet. 713 */ 714 if (entity_is_task(se)) 715 sa->runnable_load_avg = sa->load_avg = scale_load_down(se->load.weight); 716 717 se->runnable_weight = se->load.weight; 718 719 /* when this task enqueue'ed, it will contribute to its cfs_rq's load_avg */ 720 } 721 722 static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq); 723 static void attach_entity_cfs_rq(struct sched_entity *se); 724 725 /* 726 * With new tasks being created, their initial util_avgs are extrapolated 727 * based on the cfs_rq's current util_avg: 728 * 729 * util_avg = cfs_rq->util_avg / (cfs_rq->load_avg + 1) * se.load.weight 730 * 731 * However, in many cases, the above util_avg does not give a desired 732 * value. Moreover, the sum of the util_avgs may be divergent, such 733 * as when the series is a harmonic series. 734 * 735 * To solve this problem, we also cap the util_avg of successive tasks to 736 * only 1/2 of the left utilization budget: 737 * 738 * util_avg_cap = (1024 - cfs_rq->avg.util_avg) / 2^n 739 * 740 * where n denotes the nth task. 741 * 742 * For example, a simplest series from the beginning would be like: 743 * 744 * task util_avg: 512, 256, 128, 64, 32, 16, 8, ... 745 * cfs_rq util_avg: 512, 768, 896, 960, 992, 1008, 1016, ... 746 * 747 * Finally, that extrapolated util_avg is clamped to the cap (util_avg_cap) 748 * if util_avg > util_avg_cap. 749 */ 750 void post_init_entity_util_avg(struct sched_entity *se) 751 { 752 struct cfs_rq *cfs_rq = cfs_rq_of(se); 753 struct sched_avg *sa = &se->avg; 754 long cap = (long)(SCHED_CAPACITY_SCALE - cfs_rq->avg.util_avg) / 2; 755 756 if (cap > 0) { 757 if (cfs_rq->avg.util_avg != 0) { 758 sa->util_avg = cfs_rq->avg.util_avg * se->load.weight; 759 sa->util_avg /= (cfs_rq->avg.load_avg + 1); 760 761 if (sa->util_avg > cap) 762 sa->util_avg = cap; 763 } else { 764 sa->util_avg = cap; 765 } 766 } 767 768 if (entity_is_task(se)) { 769 struct task_struct *p = task_of(se); 770 if (p->sched_class != &fair_sched_class) { 771 /* 772 * For !fair tasks do: 773 * 774 update_cfs_rq_load_avg(now, cfs_rq); 775 attach_entity_load_avg(cfs_rq, se, 0); 776 switched_from_fair(rq, p); 777 * 778 * such that the next switched_to_fair() has the 779 * expected state. 780 */ 781 se->avg.last_update_time = cfs_rq_clock_task(cfs_rq); 782 return; 783 } 784 } 785 786 attach_entity_cfs_rq(se); 787 } 788 789 #else /* !CONFIG_SMP */ 790 void init_entity_runnable_average(struct sched_entity *se) 791 { 792 } 793 void post_init_entity_util_avg(struct sched_entity *se) 794 { 795 } 796 static void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) 797 { 798 } 799 #endif /* CONFIG_SMP */ 800 801 /* 802 * Update the current task's runtime statistics. 803 */ 804 static void update_curr(struct cfs_rq *cfs_rq) 805 { 806 struct sched_entity *curr = cfs_rq->curr; 807 u64 now = rq_clock_task(rq_of(cfs_rq)); 808 u64 delta_exec; 809 810 if (unlikely(!curr)) 811 return; 812 813 delta_exec = now - curr->exec_start; 814 if (unlikely((s64)delta_exec <= 0)) 815 return; 816 817 curr->exec_start = now; 818 819 schedstat_set(curr->statistics.exec_max, 820 max(delta_exec, curr->statistics.exec_max)); 821 822 curr->sum_exec_runtime += delta_exec; 823 schedstat_add(cfs_rq->exec_clock, delta_exec); 824 825 curr->vruntime += calc_delta_fair(delta_exec, curr); 826 update_min_vruntime(cfs_rq); 827 828 if (entity_is_task(curr)) { 829 struct task_struct *curtask = task_of(curr); 830 831 trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime); 832 cgroup_account_cputime(curtask, delta_exec); 833 account_group_exec_runtime(curtask, delta_exec); 834 } 835 836 account_cfs_rq_runtime(cfs_rq, delta_exec); 837 } 838 839 static void update_curr_fair(struct rq *rq) 840 { 841 update_curr(cfs_rq_of(&rq->curr->se)); 842 } 843 844 static inline void 845 update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se) 846 { 847 u64 wait_start, prev_wait_start; 848 849 if (!schedstat_enabled()) 850 return; 851 852 wait_start = rq_clock(rq_of(cfs_rq)); 853 prev_wait_start = schedstat_val(se->statistics.wait_start); 854 855 if (entity_is_task(se) && task_on_rq_migrating(task_of(se)) && 856 likely(wait_start > prev_wait_start)) 857 wait_start -= prev_wait_start; 858 859 __schedstat_set(se->statistics.wait_start, wait_start); 860 } 861 862 static inline void 863 update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se) 864 { 865 struct task_struct *p; 866 u64 delta; 867 868 if (!schedstat_enabled()) 869 return; 870 871 delta = rq_clock(rq_of(cfs_rq)) - schedstat_val(se->statistics.wait_start); 872 873 if (entity_is_task(se)) { 874 p = task_of(se); 875 if (task_on_rq_migrating(p)) { 876 /* 877 * Preserve migrating task's wait time so wait_start 878 * time stamp can be adjusted to accumulate wait time 879 * prior to migration. 880 */ 881 __schedstat_set(se->statistics.wait_start, delta); 882 return; 883 } 884 trace_sched_stat_wait(p, delta); 885 } 886 887 __schedstat_set(se->statistics.wait_max, 888 max(schedstat_val(se->statistics.wait_max), delta)); 889 __schedstat_inc(se->statistics.wait_count); 890 __schedstat_add(se->statistics.wait_sum, delta); 891 __schedstat_set(se->statistics.wait_start, 0); 892 } 893 894 static inline void 895 update_stats_enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) 896 { 897 struct task_struct *tsk = NULL; 898 u64 sleep_start, block_start; 899 900 if (!schedstat_enabled()) 901 return; 902 903 sleep_start = schedstat_val(se->statistics.sleep_start); 904 block_start = schedstat_val(se->statistics.block_start); 905 906 if (entity_is_task(se)) 907 tsk = task_of(se); 908 909 if (sleep_start) { 910 u64 delta = rq_clock(rq_of(cfs_rq)) - sleep_start; 911 912 if ((s64)delta < 0) 913 delta = 0; 914 915 if (unlikely(delta > schedstat_val(se->statistics.sleep_max))) 916 __schedstat_set(se->statistics.sleep_max, delta); 917 918 __schedstat_set(se->statistics.sleep_start, 0); 919 __schedstat_add(se->statistics.sum_sleep_runtime, delta); 920 921 if (tsk) { 922 account_scheduler_latency(tsk, delta >> 10, 1); 923 trace_sched_stat_sleep(tsk, delta); 924 } 925 } 926 if (block_start) { 927 u64 delta = rq_clock(rq_of(cfs_rq)) - block_start; 928 929 if ((s64)delta < 0) 930 delta = 0; 931 932 if (unlikely(delta > schedstat_val(se->statistics.block_max))) 933 __schedstat_set(se->statistics.block_max, delta); 934 935 __schedstat_set(se->statistics.block_start, 0); 936 __schedstat_add(se->statistics.sum_sleep_runtime, delta); 937 938 if (tsk) { 939 if (tsk->in_iowait) { 940 __schedstat_add(se->statistics.iowait_sum, delta); 941 __schedstat_inc(se->statistics.iowait_count); 942 trace_sched_stat_iowait(tsk, delta); 943 } 944 945 trace_sched_stat_blocked(tsk, delta); 946 947 /* 948 * Blocking time is in units of nanosecs, so shift by 949 * 20 to get a milliseconds-range estimation of the 950 * amount of time that the task spent sleeping: 951 */ 952 if (unlikely(prof_on == SLEEP_PROFILING)) { 953 profile_hits(SLEEP_PROFILING, 954 (void *)get_wchan(tsk), 955 delta >> 20); 956 } 957 account_scheduler_latency(tsk, delta >> 10, 0); 958 } 959 } 960 } 961 962 /* 963 * Task is being enqueued - update stats: 964 */ 965 static inline void 966 update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 967 { 968 if (!schedstat_enabled()) 969 return; 970 971 /* 972 * Are we enqueueing a waiting task? (for current tasks 973 * a dequeue/enqueue event is a NOP) 974 */ 975 if (se != cfs_rq->curr) 976 update_stats_wait_start(cfs_rq, se); 977 978 if (flags & ENQUEUE_WAKEUP) 979 update_stats_enqueue_sleeper(cfs_rq, se); 980 } 981 982 static inline void 983 update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 984 { 985 986 if (!schedstat_enabled()) 987 return; 988 989 /* 990 * Mark the end of the wait period if dequeueing a 991 * waiting task: 992 */ 993 if (se != cfs_rq->curr) 994 update_stats_wait_end(cfs_rq, se); 995 996 if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) { 997 struct task_struct *tsk = task_of(se); 998 999 if (tsk->state & TASK_INTERRUPTIBLE) 1000 __schedstat_set(se->statistics.sleep_start, 1001 rq_clock(rq_of(cfs_rq))); 1002 if (tsk->state & TASK_UNINTERRUPTIBLE) 1003 __schedstat_set(se->statistics.block_start, 1004 rq_clock(rq_of(cfs_rq))); 1005 } 1006 } 1007 1008 /* 1009 * We are picking a new current task - update its stats: 1010 */ 1011 static inline void 1012 update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) 1013 { 1014 /* 1015 * We are starting a new run period: 1016 */ 1017 se->exec_start = rq_clock_task(rq_of(cfs_rq)); 1018 } 1019 1020 /************************************************** 1021 * Scheduling class queueing methods: 1022 */ 1023 1024 #ifdef CONFIG_NUMA_BALANCING 1025 /* 1026 * Approximate time to scan a full NUMA task in ms. The task scan period is 1027 * calculated based on the tasks virtual memory size and 1028 * numa_balancing_scan_size. 1029 */ 1030 unsigned int sysctl_numa_balancing_scan_period_min = 1000; 1031 unsigned int sysctl_numa_balancing_scan_period_max = 60000; 1032 1033 /* Portion of address space to scan in MB */ 1034 unsigned int sysctl_numa_balancing_scan_size = 256; 1035 1036 /* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */ 1037 unsigned int sysctl_numa_balancing_scan_delay = 1000; 1038 1039 struct numa_group { 1040 atomic_t refcount; 1041 1042 spinlock_t lock; /* nr_tasks, tasks */ 1043 int nr_tasks; 1044 pid_t gid; 1045 int active_nodes; 1046 1047 struct rcu_head rcu; 1048 unsigned long total_faults; 1049 unsigned long max_faults_cpu; 1050 /* 1051 * Faults_cpu is used to decide whether memory should move 1052 * towards the CPU. As a consequence, these stats are weighted 1053 * more by CPU use than by memory faults. 1054 */ 1055 unsigned long *faults_cpu; 1056 unsigned long faults[0]; 1057 }; 1058 1059 static inline unsigned long group_faults_priv(struct numa_group *ng); 1060 static inline unsigned long group_faults_shared(struct numa_group *ng); 1061 1062 static unsigned int task_nr_scan_windows(struct task_struct *p) 1063 { 1064 unsigned long rss = 0; 1065 unsigned long nr_scan_pages; 1066 1067 /* 1068 * Calculations based on RSS as non-present and empty pages are skipped 1069 * by the PTE scanner and NUMA hinting faults should be trapped based 1070 * on resident pages 1071 */ 1072 nr_scan_pages = sysctl_numa_balancing_scan_size << (20 - PAGE_SHIFT); 1073 rss = get_mm_rss(p->mm); 1074 if (!rss) 1075 rss = nr_scan_pages; 1076 1077 rss = round_up(rss, nr_scan_pages); 1078 return rss / nr_scan_pages; 1079 } 1080 1081 /* For sanitys sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */ 1082 #define MAX_SCAN_WINDOW 2560 1083 1084 static unsigned int task_scan_min(struct task_struct *p) 1085 { 1086 unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size); 1087 unsigned int scan, floor; 1088 unsigned int windows = 1; 1089 1090 if (scan_size < MAX_SCAN_WINDOW) 1091 windows = MAX_SCAN_WINDOW / scan_size; 1092 floor = 1000 / windows; 1093 1094 scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p); 1095 return max_t(unsigned int, floor, scan); 1096 } 1097 1098 static unsigned int task_scan_start(struct task_struct *p) 1099 { 1100 unsigned long smin = task_scan_min(p); 1101 unsigned long period = smin; 1102 1103 /* Scale the maximum scan period with the amount of shared memory. */ 1104 if (p->numa_group) { 1105 struct numa_group *ng = p->numa_group; 1106 unsigned long shared = group_faults_shared(ng); 1107 unsigned long private = group_faults_priv(ng); 1108 1109 period *= atomic_read(&ng->refcount); 1110 period *= shared + 1; 1111 period /= private + shared + 1; 1112 } 1113 1114 return max(smin, period); 1115 } 1116 1117 static unsigned int task_scan_max(struct task_struct *p) 1118 { 1119 unsigned long smin = task_scan_min(p); 1120 unsigned long smax; 1121 1122 /* Watch for min being lower than max due to floor calculations */ 1123 smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p); 1124 1125 /* Scale the maximum scan period with the amount of shared memory. */ 1126 if (p->numa_group) { 1127 struct numa_group *ng = p->numa_group; 1128 unsigned long shared = group_faults_shared(ng); 1129 unsigned long private = group_faults_priv(ng); 1130 unsigned long period = smax; 1131 1132 period *= atomic_read(&ng->refcount); 1133 period *= shared + 1; 1134 period /= private + shared + 1; 1135 1136 smax = max(smax, period); 1137 } 1138 1139 return max(smin, smax); 1140 } 1141 1142 static void account_numa_enqueue(struct rq *rq, struct task_struct *p) 1143 { 1144 rq->nr_numa_running += (p->numa_preferred_nid != -1); 1145 rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p)); 1146 } 1147 1148 static void account_numa_dequeue(struct rq *rq, struct task_struct *p) 1149 { 1150 rq->nr_numa_running -= (p->numa_preferred_nid != -1); 1151 rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p)); 1152 } 1153 1154 /* Shared or private faults. */ 1155 #define NR_NUMA_HINT_FAULT_TYPES 2 1156 1157 /* Memory and CPU locality */ 1158 #define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2) 1159 1160 /* Averaged statistics, and temporary buffers. */ 1161 #define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2) 1162 1163 pid_t task_numa_group_id(struct task_struct *p) 1164 { 1165 return p->numa_group ? p->numa_group->gid : 0; 1166 } 1167 1168 /* 1169 * The averaged statistics, shared & private, memory & CPU, 1170 * occupy the first half of the array. The second half of the 1171 * array is for current counters, which are averaged into the 1172 * first set by task_numa_placement. 1173 */ 1174 static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv) 1175 { 1176 return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv; 1177 } 1178 1179 static inline unsigned long task_faults(struct task_struct *p, int nid) 1180 { 1181 if (!p->numa_faults) 1182 return 0; 1183 1184 return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] + 1185 p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)]; 1186 } 1187 1188 static inline unsigned long group_faults(struct task_struct *p, int nid) 1189 { 1190 if (!p->numa_group) 1191 return 0; 1192 1193 return p->numa_group->faults[task_faults_idx(NUMA_MEM, nid, 0)] + 1194 p->numa_group->faults[task_faults_idx(NUMA_MEM, nid, 1)]; 1195 } 1196 1197 static inline unsigned long group_faults_cpu(struct numa_group *group, int nid) 1198 { 1199 return group->faults_cpu[task_faults_idx(NUMA_MEM, nid, 0)] + 1200 group->faults_cpu[task_faults_idx(NUMA_MEM, nid, 1)]; 1201 } 1202 1203 static inline unsigned long group_faults_priv(struct numa_group *ng) 1204 { 1205 unsigned long faults = 0; 1206 int node; 1207 1208 for_each_online_node(node) { 1209 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 1)]; 1210 } 1211 1212 return faults; 1213 } 1214 1215 static inline unsigned long group_faults_shared(struct numa_group *ng) 1216 { 1217 unsigned long faults = 0; 1218 int node; 1219 1220 for_each_online_node(node) { 1221 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 0)]; 1222 } 1223 1224 return faults; 1225 } 1226 1227 /* 1228 * A node triggering more than 1/3 as many NUMA faults as the maximum is 1229 * considered part of a numa group's pseudo-interleaving set. Migrations 1230 * between these nodes are slowed down, to allow things to settle down. 1231 */ 1232 #define ACTIVE_NODE_FRACTION 3 1233 1234 static bool numa_is_active_node(int nid, struct numa_group *ng) 1235 { 1236 return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu; 1237 } 1238 1239 /* Handle placement on systems where not all nodes are directly connected. */ 1240 static unsigned long score_nearby_nodes(struct task_struct *p, int nid, 1241 int maxdist, bool task) 1242 { 1243 unsigned long score = 0; 1244 int node; 1245 1246 /* 1247 * All nodes are directly connected, and the same distance 1248 * from each other. No need for fancy placement algorithms. 1249 */ 1250 if (sched_numa_topology_type == NUMA_DIRECT) 1251 return 0; 1252 1253 /* 1254 * This code is called for each node, introducing N^2 complexity, 1255 * which should be ok given the number of nodes rarely exceeds 8. 1256 */ 1257 for_each_online_node(node) { 1258 unsigned long faults; 1259 int dist = node_distance(nid, node); 1260 1261 /* 1262 * The furthest away nodes in the system are not interesting 1263 * for placement; nid was already counted. 1264 */ 1265 if (dist == sched_max_numa_distance || node == nid) 1266 continue; 1267 1268 /* 1269 * On systems with a backplane NUMA topology, compare groups 1270 * of nodes, and move tasks towards the group with the most 1271 * memory accesses. When comparing two nodes at distance 1272 * "hoplimit", only nodes closer by than "hoplimit" are part 1273 * of each group. Skip other nodes. 1274 */ 1275 if (sched_numa_topology_type == NUMA_BACKPLANE && 1276 dist > maxdist) 1277 continue; 1278 1279 /* Add up the faults from nearby nodes. */ 1280 if (task) 1281 faults = task_faults(p, node); 1282 else 1283 faults = group_faults(p, node); 1284 1285 /* 1286 * On systems with a glueless mesh NUMA topology, there are 1287 * no fixed "groups of nodes". Instead, nodes that are not 1288 * directly connected bounce traffic through intermediate 1289 * nodes; a numa_group can occupy any set of nodes. 1290 * The further away a node is, the less the faults count. 1291 * This seems to result in good task placement. 1292 */ 1293 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 1294 faults *= (sched_max_numa_distance - dist); 1295 faults /= (sched_max_numa_distance - LOCAL_DISTANCE); 1296 } 1297 1298 score += faults; 1299 } 1300 1301 return score; 1302 } 1303 1304 /* 1305 * These return the fraction of accesses done by a particular task, or 1306 * task group, on a particular numa node. The group weight is given a 1307 * larger multiplier, in order to group tasks together that are almost 1308 * evenly spread out between numa nodes. 1309 */ 1310 static inline unsigned long task_weight(struct task_struct *p, int nid, 1311 int dist) 1312 { 1313 unsigned long faults, total_faults; 1314 1315 if (!p->numa_faults) 1316 return 0; 1317 1318 total_faults = p->total_numa_faults; 1319 1320 if (!total_faults) 1321 return 0; 1322 1323 faults = task_faults(p, nid); 1324 faults += score_nearby_nodes(p, nid, dist, true); 1325 1326 return 1000 * faults / total_faults; 1327 } 1328 1329 static inline unsigned long group_weight(struct task_struct *p, int nid, 1330 int dist) 1331 { 1332 unsigned long faults, total_faults; 1333 1334 if (!p->numa_group) 1335 return 0; 1336 1337 total_faults = p->numa_group->total_faults; 1338 1339 if (!total_faults) 1340 return 0; 1341 1342 faults = group_faults(p, nid); 1343 faults += score_nearby_nodes(p, nid, dist, false); 1344 1345 return 1000 * faults / total_faults; 1346 } 1347 1348 bool should_numa_migrate_memory(struct task_struct *p, struct page * page, 1349 int src_nid, int dst_cpu) 1350 { 1351 struct numa_group *ng = p->numa_group; 1352 int dst_nid = cpu_to_node(dst_cpu); 1353 int last_cpupid, this_cpupid; 1354 1355 this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid); 1356 1357 /* 1358 * Multi-stage node selection is used in conjunction with a periodic 1359 * migration fault to build a temporal task<->page relation. By using 1360 * a two-stage filter we remove short/unlikely relations. 1361 * 1362 * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate 1363 * a task's usage of a particular page (n_p) per total usage of this 1364 * page (n_t) (in a given time-span) to a probability. 1365 * 1366 * Our periodic faults will sample this probability and getting the 1367 * same result twice in a row, given these samples are fully 1368 * independent, is then given by P(n)^2, provided our sample period 1369 * is sufficiently short compared to the usage pattern. 1370 * 1371 * This quadric squishes small probabilities, making it less likely we 1372 * act on an unlikely task<->page relation. 1373 */ 1374 last_cpupid = page_cpupid_xchg_last(page, this_cpupid); 1375 if (!cpupid_pid_unset(last_cpupid) && 1376 cpupid_to_nid(last_cpupid) != dst_nid) 1377 return false; 1378 1379 /* Always allow migrate on private faults */ 1380 if (cpupid_match_pid(p, last_cpupid)) 1381 return true; 1382 1383 /* A shared fault, but p->numa_group has not been set up yet. */ 1384 if (!ng) 1385 return true; 1386 1387 /* 1388 * Destination node is much more heavily used than the source 1389 * node? Allow migration. 1390 */ 1391 if (group_faults_cpu(ng, dst_nid) > group_faults_cpu(ng, src_nid) * 1392 ACTIVE_NODE_FRACTION) 1393 return true; 1394 1395 /* 1396 * Distribute memory according to CPU & memory use on each node, 1397 * with 3/4 hysteresis to avoid unnecessary memory migrations: 1398 * 1399 * faults_cpu(dst) 3 faults_cpu(src) 1400 * --------------- * - > --------------- 1401 * faults_mem(dst) 4 faults_mem(src) 1402 */ 1403 return group_faults_cpu(ng, dst_nid) * group_faults(p, src_nid) * 3 > 1404 group_faults_cpu(ng, src_nid) * group_faults(p, dst_nid) * 4; 1405 } 1406 1407 static unsigned long weighted_cpuload(struct rq *rq); 1408 static unsigned long source_load(int cpu, int type); 1409 static unsigned long target_load(int cpu, int type); 1410 static unsigned long capacity_of(int cpu); 1411 1412 /* Cached statistics for all CPUs within a node */ 1413 struct numa_stats { 1414 unsigned long nr_running; 1415 unsigned long load; 1416 1417 /* Total compute capacity of CPUs on a node */ 1418 unsigned long compute_capacity; 1419 1420 /* Approximate capacity in terms of runnable tasks on a node */ 1421 unsigned long task_capacity; 1422 int has_free_capacity; 1423 }; 1424 1425 /* 1426 * XXX borrowed from update_sg_lb_stats 1427 */ 1428 static void update_numa_stats(struct numa_stats *ns, int nid) 1429 { 1430 int smt, cpu, cpus = 0; 1431 unsigned long capacity; 1432 1433 memset(ns, 0, sizeof(*ns)); 1434 for_each_cpu(cpu, cpumask_of_node(nid)) { 1435 struct rq *rq = cpu_rq(cpu); 1436 1437 ns->nr_running += rq->nr_running; 1438 ns->load += weighted_cpuload(rq); 1439 ns->compute_capacity += capacity_of(cpu); 1440 1441 cpus++; 1442 } 1443 1444 /* 1445 * If we raced with hotplug and there are no CPUs left in our mask 1446 * the @ns structure is NULL'ed and task_numa_compare() will 1447 * not find this node attractive. 1448 * 1449 * We'll either bail at !has_free_capacity, or we'll detect a huge 1450 * imbalance and bail there. 1451 */ 1452 if (!cpus) 1453 return; 1454 1455 /* smt := ceil(cpus / capacity), assumes: 1 < smt_power < 2 */ 1456 smt = DIV_ROUND_UP(SCHED_CAPACITY_SCALE * cpus, ns->compute_capacity); 1457 capacity = cpus / smt; /* cores */ 1458 1459 ns->task_capacity = min_t(unsigned, capacity, 1460 DIV_ROUND_CLOSEST(ns->compute_capacity, SCHED_CAPACITY_SCALE)); 1461 ns->has_free_capacity = (ns->nr_running < ns->task_capacity); 1462 } 1463 1464 struct task_numa_env { 1465 struct task_struct *p; 1466 1467 int src_cpu, src_nid; 1468 int dst_cpu, dst_nid; 1469 1470 struct numa_stats src_stats, dst_stats; 1471 1472 int imbalance_pct; 1473 int dist; 1474 1475 struct task_struct *best_task; 1476 long best_imp; 1477 int best_cpu; 1478 }; 1479 1480 static void task_numa_assign(struct task_numa_env *env, 1481 struct task_struct *p, long imp) 1482 { 1483 if (env->best_task) 1484 put_task_struct(env->best_task); 1485 if (p) 1486 get_task_struct(p); 1487 1488 env->best_task = p; 1489 env->best_imp = imp; 1490 env->best_cpu = env->dst_cpu; 1491 } 1492 1493 static bool load_too_imbalanced(long src_load, long dst_load, 1494 struct task_numa_env *env) 1495 { 1496 long imb, old_imb; 1497 long orig_src_load, orig_dst_load; 1498 long src_capacity, dst_capacity; 1499 1500 /* 1501 * The load is corrected for the CPU capacity available on each node. 1502 * 1503 * src_load dst_load 1504 * ------------ vs --------- 1505 * src_capacity dst_capacity 1506 */ 1507 src_capacity = env->src_stats.compute_capacity; 1508 dst_capacity = env->dst_stats.compute_capacity; 1509 1510 /* We care about the slope of the imbalance, not the direction. */ 1511 if (dst_load < src_load) 1512 swap(dst_load, src_load); 1513 1514 /* Is the difference below the threshold? */ 1515 imb = dst_load * src_capacity * 100 - 1516 src_load * dst_capacity * env->imbalance_pct; 1517 if (imb <= 0) 1518 return false; 1519 1520 /* 1521 * The imbalance is above the allowed threshold. 1522 * Compare it with the old imbalance. 1523 */ 1524 orig_src_load = env->src_stats.load; 1525 orig_dst_load = env->dst_stats.load; 1526 1527 if (orig_dst_load < orig_src_load) 1528 swap(orig_dst_load, orig_src_load); 1529 1530 old_imb = orig_dst_load * src_capacity * 100 - 1531 orig_src_load * dst_capacity * env->imbalance_pct; 1532 1533 /* Would this change make things worse? */ 1534 return (imb > old_imb); 1535 } 1536 1537 /* 1538 * This checks if the overall compute and NUMA accesses of the system would 1539 * be improved if the source tasks was migrated to the target dst_cpu taking 1540 * into account that it might be best if task running on the dst_cpu should 1541 * be exchanged with the source task 1542 */ 1543 static void task_numa_compare(struct task_numa_env *env, 1544 long taskimp, long groupimp) 1545 { 1546 struct rq *src_rq = cpu_rq(env->src_cpu); 1547 struct rq *dst_rq = cpu_rq(env->dst_cpu); 1548 struct task_struct *cur; 1549 long src_load, dst_load; 1550 long load; 1551 long imp = env->p->numa_group ? groupimp : taskimp; 1552 long moveimp = imp; 1553 int dist = env->dist; 1554 1555 rcu_read_lock(); 1556 cur = task_rcu_dereference(&dst_rq->curr); 1557 if (cur && ((cur->flags & PF_EXITING) || is_idle_task(cur))) 1558 cur = NULL; 1559 1560 /* 1561 * Because we have preemption enabled we can get migrated around and 1562 * end try selecting ourselves (current == env->p) as a swap candidate. 1563 */ 1564 if (cur == env->p) 1565 goto unlock; 1566 1567 /* 1568 * "imp" is the fault differential for the source task between the 1569 * source and destination node. Calculate the total differential for 1570 * the source task and potential destination task. The more negative 1571 * the value is, the more rmeote accesses that would be expected to 1572 * be incurred if the tasks were swapped. 1573 */ 1574 if (cur) { 1575 /* Skip this swap candidate if cannot move to the source CPU: */ 1576 if (!cpumask_test_cpu(env->src_cpu, &cur->cpus_allowed)) 1577 goto unlock; 1578 1579 /* 1580 * If dst and source tasks are in the same NUMA group, or not 1581 * in any group then look only at task weights. 1582 */ 1583 if (cur->numa_group == env->p->numa_group) { 1584 imp = taskimp + task_weight(cur, env->src_nid, dist) - 1585 task_weight(cur, env->dst_nid, dist); 1586 /* 1587 * Add some hysteresis to prevent swapping the 1588 * tasks within a group over tiny differences. 1589 */ 1590 if (cur->numa_group) 1591 imp -= imp/16; 1592 } else { 1593 /* 1594 * Compare the group weights. If a task is all by 1595 * itself (not part of a group), use the task weight 1596 * instead. 1597 */ 1598 if (cur->numa_group) 1599 imp += group_weight(cur, env->src_nid, dist) - 1600 group_weight(cur, env->dst_nid, dist); 1601 else 1602 imp += task_weight(cur, env->src_nid, dist) - 1603 task_weight(cur, env->dst_nid, dist); 1604 } 1605 } 1606 1607 if (imp <= env->best_imp && moveimp <= env->best_imp) 1608 goto unlock; 1609 1610 if (!cur) { 1611 /* Is there capacity at our destination? */ 1612 if (env->src_stats.nr_running <= env->src_stats.task_capacity && 1613 !env->dst_stats.has_free_capacity) 1614 goto unlock; 1615 1616 goto balance; 1617 } 1618 1619 /* Balance doesn't matter much if we're running a task per CPU: */ 1620 if (imp > env->best_imp && src_rq->nr_running == 1 && 1621 dst_rq->nr_running == 1) 1622 goto assign; 1623 1624 /* 1625 * In the overloaded case, try and keep the load balanced. 1626 */ 1627 balance: 1628 load = task_h_load(env->p); 1629 dst_load = env->dst_stats.load + load; 1630 src_load = env->src_stats.load - load; 1631 1632 if (moveimp > imp && moveimp > env->best_imp) { 1633 /* 1634 * If the improvement from just moving env->p direction is 1635 * better than swapping tasks around, check if a move is 1636 * possible. Store a slightly smaller score than moveimp, 1637 * so an actually idle CPU will win. 1638 */ 1639 if (!load_too_imbalanced(src_load, dst_load, env)) { 1640 imp = moveimp - 1; 1641 cur = NULL; 1642 goto assign; 1643 } 1644 } 1645 1646 if (imp <= env->best_imp) 1647 goto unlock; 1648 1649 if (cur) { 1650 load = task_h_load(cur); 1651 dst_load -= load; 1652 src_load += load; 1653 } 1654 1655 if (load_too_imbalanced(src_load, dst_load, env)) 1656 goto unlock; 1657 1658 /* 1659 * One idle CPU per node is evaluated for a task numa move. 1660 * Call select_idle_sibling to maybe find a better one. 1661 */ 1662 if (!cur) { 1663 /* 1664 * select_idle_siblings() uses an per-CPU cpumask that 1665 * can be used from IRQ context. 1666 */ 1667 local_irq_disable(); 1668 env->dst_cpu = select_idle_sibling(env->p, env->src_cpu, 1669 env->dst_cpu); 1670 local_irq_enable(); 1671 } 1672 1673 assign: 1674 task_numa_assign(env, cur, imp); 1675 unlock: 1676 rcu_read_unlock(); 1677 } 1678 1679 static void task_numa_find_cpu(struct task_numa_env *env, 1680 long taskimp, long groupimp) 1681 { 1682 int cpu; 1683 1684 for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) { 1685 /* Skip this CPU if the source task cannot migrate */ 1686 if (!cpumask_test_cpu(cpu, &env->p->cpus_allowed)) 1687 continue; 1688 1689 env->dst_cpu = cpu; 1690 task_numa_compare(env, taskimp, groupimp); 1691 } 1692 } 1693 1694 /* Only move tasks to a NUMA node less busy than the current node. */ 1695 static bool numa_has_capacity(struct task_numa_env *env) 1696 { 1697 struct numa_stats *src = &env->src_stats; 1698 struct numa_stats *dst = &env->dst_stats; 1699 1700 if (src->has_free_capacity && !dst->has_free_capacity) 1701 return false; 1702 1703 /* 1704 * Only consider a task move if the source has a higher load 1705 * than the destination, corrected for CPU capacity on each node. 1706 * 1707 * src->load dst->load 1708 * --------------------- vs --------------------- 1709 * src->compute_capacity dst->compute_capacity 1710 */ 1711 if (src->load * dst->compute_capacity * env->imbalance_pct > 1712 1713 dst->load * src->compute_capacity * 100) 1714 return true; 1715 1716 return false; 1717 } 1718 1719 static int task_numa_migrate(struct task_struct *p) 1720 { 1721 struct task_numa_env env = { 1722 .p = p, 1723 1724 .src_cpu = task_cpu(p), 1725 .src_nid = task_node(p), 1726 1727 .imbalance_pct = 112, 1728 1729 .best_task = NULL, 1730 .best_imp = 0, 1731 .best_cpu = -1, 1732 }; 1733 struct sched_domain *sd; 1734 unsigned long taskweight, groupweight; 1735 int nid, ret, dist; 1736 long taskimp, groupimp; 1737 1738 /* 1739 * Pick the lowest SD_NUMA domain, as that would have the smallest 1740 * imbalance and would be the first to start moving tasks about. 1741 * 1742 * And we want to avoid any moving of tasks about, as that would create 1743 * random movement of tasks -- counter the numa conditions we're trying 1744 * to satisfy here. 1745 */ 1746 rcu_read_lock(); 1747 sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu)); 1748 if (sd) 1749 env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2; 1750 rcu_read_unlock(); 1751 1752 /* 1753 * Cpusets can break the scheduler domain tree into smaller 1754 * balance domains, some of which do not cross NUMA boundaries. 1755 * Tasks that are "trapped" in such domains cannot be migrated 1756 * elsewhere, so there is no point in (re)trying. 1757 */ 1758 if (unlikely(!sd)) { 1759 p->numa_preferred_nid = task_node(p); 1760 return -EINVAL; 1761 } 1762 1763 env.dst_nid = p->numa_preferred_nid; 1764 dist = env.dist = node_distance(env.src_nid, env.dst_nid); 1765 taskweight = task_weight(p, env.src_nid, dist); 1766 groupweight = group_weight(p, env.src_nid, dist); 1767 update_numa_stats(&env.src_stats, env.src_nid); 1768 taskimp = task_weight(p, env.dst_nid, dist) - taskweight; 1769 groupimp = group_weight(p, env.dst_nid, dist) - groupweight; 1770 update_numa_stats(&env.dst_stats, env.dst_nid); 1771 1772 /* Try to find a spot on the preferred nid. */ 1773 if (numa_has_capacity(&env)) 1774 task_numa_find_cpu(&env, taskimp, groupimp); 1775 1776 /* 1777 * Look at other nodes in these cases: 1778 * - there is no space available on the preferred_nid 1779 * - the task is part of a numa_group that is interleaved across 1780 * multiple NUMA nodes; in order to better consolidate the group, 1781 * we need to check other locations. 1782 */ 1783 if (env.best_cpu == -1 || (p->numa_group && p->numa_group->active_nodes > 1)) { 1784 for_each_online_node(nid) { 1785 if (nid == env.src_nid || nid == p->numa_preferred_nid) 1786 continue; 1787 1788 dist = node_distance(env.src_nid, env.dst_nid); 1789 if (sched_numa_topology_type == NUMA_BACKPLANE && 1790 dist != env.dist) { 1791 taskweight = task_weight(p, env.src_nid, dist); 1792 groupweight = group_weight(p, env.src_nid, dist); 1793 } 1794 1795 /* Only consider nodes where both task and groups benefit */ 1796 taskimp = task_weight(p, nid, dist) - taskweight; 1797 groupimp = group_weight(p, nid, dist) - groupweight; 1798 if (taskimp < 0 && groupimp < 0) 1799 continue; 1800 1801 env.dist = dist; 1802 env.dst_nid = nid; 1803 update_numa_stats(&env.dst_stats, env.dst_nid); 1804 if (numa_has_capacity(&env)) 1805 task_numa_find_cpu(&env, taskimp, groupimp); 1806 } 1807 } 1808 1809 /* 1810 * If the task is part of a workload that spans multiple NUMA nodes, 1811 * and is migrating into one of the workload's active nodes, remember 1812 * this node as the task's preferred numa node, so the workload can 1813 * settle down. 1814 * A task that migrated to a second choice node will be better off 1815 * trying for a better one later. Do not set the preferred node here. 1816 */ 1817 if (p->numa_group) { 1818 struct numa_group *ng = p->numa_group; 1819 1820 if (env.best_cpu == -1) 1821 nid = env.src_nid; 1822 else 1823 nid = env.dst_nid; 1824 1825 if (ng->active_nodes > 1 && numa_is_active_node(env.dst_nid, ng)) 1826 sched_setnuma(p, env.dst_nid); 1827 } 1828 1829 /* No better CPU than the current one was found. */ 1830 if (env.best_cpu == -1) 1831 return -EAGAIN; 1832 1833 /* 1834 * Reset the scan period if the task is being rescheduled on an 1835 * alternative node to recheck if the tasks is now properly placed. 1836 */ 1837 p->numa_scan_period = task_scan_start(p); 1838 1839 if (env.best_task == NULL) { 1840 ret = migrate_task_to(p, env.best_cpu); 1841 if (ret != 0) 1842 trace_sched_stick_numa(p, env.src_cpu, env.best_cpu); 1843 return ret; 1844 } 1845 1846 ret = migrate_swap(p, env.best_task); 1847 if (ret != 0) 1848 trace_sched_stick_numa(p, env.src_cpu, task_cpu(env.best_task)); 1849 put_task_struct(env.best_task); 1850 return ret; 1851 } 1852 1853 /* Attempt to migrate a task to a CPU on the preferred node. */ 1854 static void numa_migrate_preferred(struct task_struct *p) 1855 { 1856 unsigned long interval = HZ; 1857 1858 /* This task has no NUMA fault statistics yet */ 1859 if (unlikely(p->numa_preferred_nid == -1 || !p->numa_faults)) 1860 return; 1861 1862 /* Periodically retry migrating the task to the preferred node */ 1863 interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16); 1864 p->numa_migrate_retry = jiffies + interval; 1865 1866 /* Success if task is already running on preferred CPU */ 1867 if (task_node(p) == p->numa_preferred_nid) 1868 return; 1869 1870 /* Otherwise, try migrate to a CPU on the preferred node */ 1871 task_numa_migrate(p); 1872 } 1873 1874 /* 1875 * Find out how many nodes on the workload is actively running on. Do this by 1876 * tracking the nodes from which NUMA hinting faults are triggered. This can 1877 * be different from the set of nodes where the workload's memory is currently 1878 * located. 1879 */ 1880 static void numa_group_count_active_nodes(struct numa_group *numa_group) 1881 { 1882 unsigned long faults, max_faults = 0; 1883 int nid, active_nodes = 0; 1884 1885 for_each_online_node(nid) { 1886 faults = group_faults_cpu(numa_group, nid); 1887 if (faults > max_faults) 1888 max_faults = faults; 1889 } 1890 1891 for_each_online_node(nid) { 1892 faults = group_faults_cpu(numa_group, nid); 1893 if (faults * ACTIVE_NODE_FRACTION > max_faults) 1894 active_nodes++; 1895 } 1896 1897 numa_group->max_faults_cpu = max_faults; 1898 numa_group->active_nodes = active_nodes; 1899 } 1900 1901 /* 1902 * When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS 1903 * increments. The more local the fault statistics are, the higher the scan 1904 * period will be for the next scan window. If local/(local+remote) ratio is 1905 * below NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS) 1906 * the scan period will decrease. Aim for 70% local accesses. 1907 */ 1908 #define NUMA_PERIOD_SLOTS 10 1909 #define NUMA_PERIOD_THRESHOLD 7 1910 1911 /* 1912 * Increase the scan period (slow down scanning) if the majority of 1913 * our memory is already on our local node, or if the majority of 1914 * the page accesses are shared with other processes. 1915 * Otherwise, decrease the scan period. 1916 */ 1917 static void update_task_scan_period(struct task_struct *p, 1918 unsigned long shared, unsigned long private) 1919 { 1920 unsigned int period_slot; 1921 int lr_ratio, ps_ratio; 1922 int diff; 1923 1924 unsigned long remote = p->numa_faults_locality[0]; 1925 unsigned long local = p->numa_faults_locality[1]; 1926 1927 /* 1928 * If there were no record hinting faults then either the task is 1929 * completely idle or all activity is areas that are not of interest 1930 * to automatic numa balancing. Related to that, if there were failed 1931 * migration then it implies we are migrating too quickly or the local 1932 * node is overloaded. In either case, scan slower 1933 */ 1934 if (local + shared == 0 || p->numa_faults_locality[2]) { 1935 p->numa_scan_period = min(p->numa_scan_period_max, 1936 p->numa_scan_period << 1); 1937 1938 p->mm->numa_next_scan = jiffies + 1939 msecs_to_jiffies(p->numa_scan_period); 1940 1941 return; 1942 } 1943 1944 /* 1945 * Prepare to scale scan period relative to the current period. 1946 * == NUMA_PERIOD_THRESHOLD scan period stays the same 1947 * < NUMA_PERIOD_THRESHOLD scan period decreases (scan faster) 1948 * >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower) 1949 */ 1950 period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS); 1951 lr_ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote); 1952 ps_ratio = (private * NUMA_PERIOD_SLOTS) / (private + shared); 1953 1954 if (ps_ratio >= NUMA_PERIOD_THRESHOLD) { 1955 /* 1956 * Most memory accesses are local. There is no need to 1957 * do fast NUMA scanning, since memory is already local. 1958 */ 1959 int slot = ps_ratio - NUMA_PERIOD_THRESHOLD; 1960 if (!slot) 1961 slot = 1; 1962 diff = slot * period_slot; 1963 } else if (lr_ratio >= NUMA_PERIOD_THRESHOLD) { 1964 /* 1965 * Most memory accesses are shared with other tasks. 1966 * There is no point in continuing fast NUMA scanning, 1967 * since other tasks may just move the memory elsewhere. 1968 */ 1969 int slot = lr_ratio - NUMA_PERIOD_THRESHOLD; 1970 if (!slot) 1971 slot = 1; 1972 diff = slot * period_slot; 1973 } else { 1974 /* 1975 * Private memory faults exceed (SLOTS-THRESHOLD)/SLOTS, 1976 * yet they are not on the local NUMA node. Speed up 1977 * NUMA scanning to get the memory moved over. 1978 */ 1979 int ratio = max(lr_ratio, ps_ratio); 1980 diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot; 1981 } 1982 1983 p->numa_scan_period = clamp(p->numa_scan_period + diff, 1984 task_scan_min(p), task_scan_max(p)); 1985 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); 1986 } 1987 1988 /* 1989 * Get the fraction of time the task has been running since the last 1990 * NUMA placement cycle. The scheduler keeps similar statistics, but 1991 * decays those on a 32ms period, which is orders of magnitude off 1992 * from the dozens-of-seconds NUMA balancing period. Use the scheduler 1993 * stats only if the task is so new there are no NUMA statistics yet. 1994 */ 1995 static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period) 1996 { 1997 u64 runtime, delta, now; 1998 /* Use the start of this time slice to avoid calculations. */ 1999 now = p->se.exec_start; 2000 runtime = p->se.sum_exec_runtime; 2001 2002 if (p->last_task_numa_placement) { 2003 delta = runtime - p->last_sum_exec_runtime; 2004 *period = now - p->last_task_numa_placement; 2005 } else { 2006 delta = p->se.avg.load_sum; 2007 *period = LOAD_AVG_MAX; 2008 } 2009 2010 p->last_sum_exec_runtime = runtime; 2011 p->last_task_numa_placement = now; 2012 2013 return delta; 2014 } 2015 2016 /* 2017 * Determine the preferred nid for a task in a numa_group. This needs to 2018 * be done in a way that produces consistent results with group_weight, 2019 * otherwise workloads might not converge. 2020 */ 2021 static int preferred_group_nid(struct task_struct *p, int nid) 2022 { 2023 nodemask_t nodes; 2024 int dist; 2025 2026 /* Direct connections between all NUMA nodes. */ 2027 if (sched_numa_topology_type == NUMA_DIRECT) 2028 return nid; 2029 2030 /* 2031 * On a system with glueless mesh NUMA topology, group_weight 2032 * scores nodes according to the number of NUMA hinting faults on 2033 * both the node itself, and on nearby nodes. 2034 */ 2035 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 2036 unsigned long score, max_score = 0; 2037 int node, max_node = nid; 2038 2039 dist = sched_max_numa_distance; 2040 2041 for_each_online_node(node) { 2042 score = group_weight(p, node, dist); 2043 if (score > max_score) { 2044 max_score = score; 2045 max_node = node; 2046 } 2047 } 2048 return max_node; 2049 } 2050 2051 /* 2052 * Finding the preferred nid in a system with NUMA backplane 2053 * interconnect topology is more involved. The goal is to locate 2054 * tasks from numa_groups near each other in the system, and 2055 * untangle workloads from different sides of the system. This requires 2056 * searching down the hierarchy of node groups, recursively searching 2057 * inside the highest scoring group of nodes. The nodemask tricks 2058 * keep the complexity of the search down. 2059 */ 2060 nodes = node_online_map; 2061 for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) { 2062 unsigned long max_faults = 0; 2063 nodemask_t max_group = NODE_MASK_NONE; 2064 int a, b; 2065 2066 /* Are there nodes at this distance from each other? */ 2067 if (!find_numa_distance(dist)) 2068 continue; 2069 2070 for_each_node_mask(a, nodes) { 2071 unsigned long faults = 0; 2072 nodemask_t this_group; 2073 nodes_clear(this_group); 2074 2075 /* Sum group's NUMA faults; includes a==b case. */ 2076 for_each_node_mask(b, nodes) { 2077 if (node_distance(a, b) < dist) { 2078 faults += group_faults(p, b); 2079 node_set(b, this_group); 2080 node_clear(b, nodes); 2081 } 2082 } 2083 2084 /* Remember the top group. */ 2085 if (faults > max_faults) { 2086 max_faults = faults; 2087 max_group = this_group; 2088 /* 2089 * subtle: at the smallest distance there is 2090 * just one node left in each "group", the 2091 * winner is the preferred nid. 2092 */ 2093 nid = a; 2094 } 2095 } 2096 /* Next round, evaluate the nodes within max_group. */ 2097 if (!max_faults) 2098 break; 2099 nodes = max_group; 2100 } 2101 return nid; 2102 } 2103 2104 static void task_numa_placement(struct task_struct *p) 2105 { 2106 int seq, nid, max_nid = -1, max_group_nid = -1; 2107 unsigned long max_faults = 0, max_group_faults = 0; 2108 unsigned long fault_types[2] = { 0, 0 }; 2109 unsigned long total_faults; 2110 u64 runtime, period; 2111 spinlock_t *group_lock = NULL; 2112 2113 /* 2114 * The p->mm->numa_scan_seq field gets updated without 2115 * exclusive access. Use READ_ONCE() here to ensure 2116 * that the field is read in a single access: 2117 */ 2118 seq = READ_ONCE(p->mm->numa_scan_seq); 2119 if (p->numa_scan_seq == seq) 2120 return; 2121 p->numa_scan_seq = seq; 2122 p->numa_scan_period_max = task_scan_max(p); 2123 2124 total_faults = p->numa_faults_locality[0] + 2125 p->numa_faults_locality[1]; 2126 runtime = numa_get_avg_runtime(p, &period); 2127 2128 /* If the task is part of a group prevent parallel updates to group stats */ 2129 if (p->numa_group) { 2130 group_lock = &p->numa_group->lock; 2131 spin_lock_irq(group_lock); 2132 } 2133 2134 /* Find the node with the highest number of faults */ 2135 for_each_online_node(nid) { 2136 /* Keep track of the offsets in numa_faults array */ 2137 int mem_idx, membuf_idx, cpu_idx, cpubuf_idx; 2138 unsigned long faults = 0, group_faults = 0; 2139 int priv; 2140 2141 for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) { 2142 long diff, f_diff, f_weight; 2143 2144 mem_idx = task_faults_idx(NUMA_MEM, nid, priv); 2145 membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv); 2146 cpu_idx = task_faults_idx(NUMA_CPU, nid, priv); 2147 cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv); 2148 2149 /* Decay existing window, copy faults since last scan */ 2150 diff = p->numa_faults[membuf_idx] - p->numa_faults[mem_idx] / 2; 2151 fault_types[priv] += p->numa_faults[membuf_idx]; 2152 p->numa_faults[membuf_idx] = 0; 2153 2154 /* 2155 * Normalize the faults_from, so all tasks in a group 2156 * count according to CPU use, instead of by the raw 2157 * number of faults. Tasks with little runtime have 2158 * little over-all impact on throughput, and thus their 2159 * faults are less important. 2160 */ 2161 f_weight = div64_u64(runtime << 16, period + 1); 2162 f_weight = (f_weight * p->numa_faults[cpubuf_idx]) / 2163 (total_faults + 1); 2164 f_diff = f_weight - p->numa_faults[cpu_idx] / 2; 2165 p->numa_faults[cpubuf_idx] = 0; 2166 2167 p->numa_faults[mem_idx] += diff; 2168 p->numa_faults[cpu_idx] += f_diff; 2169 faults += p->numa_faults[mem_idx]; 2170 p->total_numa_faults += diff; 2171 if (p->numa_group) { 2172 /* 2173 * safe because we can only change our own group 2174 * 2175 * mem_idx represents the offset for a given 2176 * nid and priv in a specific region because it 2177 * is at the beginning of the numa_faults array. 2178 */ 2179 p->numa_group->faults[mem_idx] += diff; 2180 p->numa_group->faults_cpu[mem_idx] += f_diff; 2181 p->numa_group->total_faults += diff; 2182 group_faults += p->numa_group->faults[mem_idx]; 2183 } 2184 } 2185 2186 if (faults > max_faults) { 2187 max_faults = faults; 2188 max_nid = nid; 2189 } 2190 2191 if (group_faults > max_group_faults) { 2192 max_group_faults = group_faults; 2193 max_group_nid = nid; 2194 } 2195 } 2196 2197 update_task_scan_period(p, fault_types[0], fault_types[1]); 2198 2199 if (p->numa_group) { 2200 numa_group_count_active_nodes(p->numa_group); 2201 spin_unlock_irq(group_lock); 2202 max_nid = preferred_group_nid(p, max_group_nid); 2203 } 2204 2205 if (max_faults) { 2206 /* Set the new preferred node */ 2207 if (max_nid != p->numa_preferred_nid) 2208 sched_setnuma(p, max_nid); 2209 2210 if (task_node(p) != p->numa_preferred_nid) 2211 numa_migrate_preferred(p); 2212 } 2213 } 2214 2215 static inline int get_numa_group(struct numa_group *grp) 2216 { 2217 return atomic_inc_not_zero(&grp->refcount); 2218 } 2219 2220 static inline void put_numa_group(struct numa_group *grp) 2221 { 2222 if (atomic_dec_and_test(&grp->refcount)) 2223 kfree_rcu(grp, rcu); 2224 } 2225 2226 static void task_numa_group(struct task_struct *p, int cpupid, int flags, 2227 int *priv) 2228 { 2229 struct numa_group *grp, *my_grp; 2230 struct task_struct *tsk; 2231 bool join = false; 2232 int cpu = cpupid_to_cpu(cpupid); 2233 int i; 2234 2235 if (unlikely(!p->numa_group)) { 2236 unsigned int size = sizeof(struct numa_group) + 2237 4*nr_node_ids*sizeof(unsigned long); 2238 2239 grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); 2240 if (!grp) 2241 return; 2242 2243 atomic_set(&grp->refcount, 1); 2244 grp->active_nodes = 1; 2245 grp->max_faults_cpu = 0; 2246 spin_lock_init(&grp->lock); 2247 grp->gid = p->pid; 2248 /* Second half of the array tracks nids where faults happen */ 2249 grp->faults_cpu = grp->faults + NR_NUMA_HINT_FAULT_TYPES * 2250 nr_node_ids; 2251 2252 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 2253 grp->faults[i] = p->numa_faults[i]; 2254 2255 grp->total_faults = p->total_numa_faults; 2256 2257 grp->nr_tasks++; 2258 rcu_assign_pointer(p->numa_group, grp); 2259 } 2260 2261 rcu_read_lock(); 2262 tsk = READ_ONCE(cpu_rq(cpu)->curr); 2263 2264 if (!cpupid_match_pid(tsk, cpupid)) 2265 goto no_join; 2266 2267 grp = rcu_dereference(tsk->numa_group); 2268 if (!grp) 2269 goto no_join; 2270 2271 my_grp = p->numa_group; 2272 if (grp == my_grp) 2273 goto no_join; 2274 2275 /* 2276 * Only join the other group if its bigger; if we're the bigger group, 2277 * the other task will join us. 2278 */ 2279 if (my_grp->nr_tasks > grp->nr_tasks) 2280 goto no_join; 2281 2282 /* 2283 * Tie-break on the grp address. 2284 */ 2285 if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp) 2286 goto no_join; 2287 2288 /* Always join threads in the same process. */ 2289 if (tsk->mm == current->mm) 2290 join = true; 2291 2292 /* Simple filter to avoid false positives due to PID collisions */ 2293 if (flags & TNF_SHARED) 2294 join = true; 2295 2296 /* Update priv based on whether false sharing was detected */ 2297 *priv = !join; 2298 2299 if (join && !get_numa_group(grp)) 2300 goto no_join; 2301 2302 rcu_read_unlock(); 2303 2304 if (!join) 2305 return; 2306 2307 BUG_ON(irqs_disabled()); 2308 double_lock_irq(&my_grp->lock, &grp->lock); 2309 2310 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) { 2311 my_grp->faults[i] -= p->numa_faults[i]; 2312 grp->faults[i] += p->numa_faults[i]; 2313 } 2314 my_grp->total_faults -= p->total_numa_faults; 2315 grp->total_faults += p->total_numa_faults; 2316 2317 my_grp->nr_tasks--; 2318 grp->nr_tasks++; 2319 2320 spin_unlock(&my_grp->lock); 2321 spin_unlock_irq(&grp->lock); 2322 2323 rcu_assign_pointer(p->numa_group, grp); 2324 2325 put_numa_group(my_grp); 2326 return; 2327 2328 no_join: 2329 rcu_read_unlock(); 2330 return; 2331 } 2332 2333 void task_numa_free(struct task_struct *p) 2334 { 2335 struct numa_group *grp = p->numa_group; 2336 void *numa_faults = p->numa_faults; 2337 unsigned long flags; 2338 int i; 2339 2340 if (grp) { 2341 spin_lock_irqsave(&grp->lock, flags); 2342 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 2343 grp->faults[i] -= p->numa_faults[i]; 2344 grp->total_faults -= p->total_numa_faults; 2345 2346 grp->nr_tasks--; 2347 spin_unlock_irqrestore(&grp->lock, flags); 2348 RCU_INIT_POINTER(p->numa_group, NULL); 2349 put_numa_group(grp); 2350 } 2351 2352 p->numa_faults = NULL; 2353 kfree(numa_faults); 2354 } 2355 2356 /* 2357 * Got a PROT_NONE fault for a page on @node. 2358 */ 2359 void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags) 2360 { 2361 struct task_struct *p = current; 2362 bool migrated = flags & TNF_MIGRATED; 2363 int cpu_node = task_node(current); 2364 int local = !!(flags & TNF_FAULT_LOCAL); 2365 struct numa_group *ng; 2366 int priv; 2367 2368 if (!static_branch_likely(&sched_numa_balancing)) 2369 return; 2370 2371 /* for example, ksmd faulting in a user's mm */ 2372 if (!p->mm) 2373 return; 2374 2375 /* Allocate buffer to track faults on a per-node basis */ 2376 if (unlikely(!p->numa_faults)) { 2377 int size = sizeof(*p->numa_faults) * 2378 NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids; 2379 2380 p->numa_faults = kzalloc(size, GFP_KERNEL|__GFP_NOWARN); 2381 if (!p->numa_faults) 2382 return; 2383 2384 p->total_numa_faults = 0; 2385 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); 2386 } 2387 2388 /* 2389 * First accesses are treated as private, otherwise consider accesses 2390 * to be private if the accessing pid has not changed 2391 */ 2392 if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) { 2393 priv = 1; 2394 } else { 2395 priv = cpupid_match_pid(p, last_cpupid); 2396 if (!priv && !(flags & TNF_NO_GROUP)) 2397 task_numa_group(p, last_cpupid, flags, &priv); 2398 } 2399 2400 /* 2401 * If a workload spans multiple NUMA nodes, a shared fault that 2402 * occurs wholly within the set of nodes that the workload is 2403 * actively using should be counted as local. This allows the 2404 * scan rate to slow down when a workload has settled down. 2405 */ 2406 ng = p->numa_group; 2407 if (!priv && !local && ng && ng->active_nodes > 1 && 2408 numa_is_active_node(cpu_node, ng) && 2409 numa_is_active_node(mem_node, ng)) 2410 local = 1; 2411 2412 task_numa_placement(p); 2413 2414 /* 2415 * Retry task to preferred node migration periodically, in case it 2416 * case it previously failed, or the scheduler moved us. 2417 */ 2418 if (time_after(jiffies, p->numa_migrate_retry)) 2419 numa_migrate_preferred(p); 2420 2421 if (migrated) 2422 p->numa_pages_migrated += pages; 2423 if (flags & TNF_MIGRATE_FAIL) 2424 p->numa_faults_locality[2] += pages; 2425 2426 p->numa_faults[task_faults_idx(NUMA_MEMBUF, mem_node, priv)] += pages; 2427 p->numa_faults[task_faults_idx(NUMA_CPUBUF, cpu_node, priv)] += pages; 2428 p->numa_faults_locality[local] += pages; 2429 } 2430 2431 static void reset_ptenuma_scan(struct task_struct *p) 2432 { 2433 /* 2434 * We only did a read acquisition of the mmap sem, so 2435 * p->mm->numa_scan_seq is written to without exclusive access 2436 * and the update is not guaranteed to be atomic. That's not 2437 * much of an issue though, since this is just used for 2438 * statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not 2439 * expensive, to avoid any form of compiler optimizations: 2440 */ 2441 WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1); 2442 p->mm->numa_scan_offset = 0; 2443 } 2444 2445 /* 2446 * The expensive part of numa migration is done from task_work context. 2447 * Triggered from task_tick_numa(). 2448 */ 2449 void task_numa_work(struct callback_head *work) 2450 { 2451 unsigned long migrate, next_scan, now = jiffies; 2452 struct task_struct *p = current; 2453 struct mm_struct *mm = p->mm; 2454 u64 runtime = p->se.sum_exec_runtime; 2455 struct vm_area_struct *vma; 2456 unsigned long start, end; 2457 unsigned long nr_pte_updates = 0; 2458 long pages, virtpages; 2459 2460 SCHED_WARN_ON(p != container_of(work, struct task_struct, numa_work)); 2461 2462 work->next = work; /* protect against double add */ 2463 /* 2464 * Who cares about NUMA placement when they're dying. 2465 * 2466 * NOTE: make sure not to dereference p->mm before this check, 2467 * exit_task_work() happens _after_ exit_mm() so we could be called 2468 * without p->mm even though we still had it when we enqueued this 2469 * work. 2470 */ 2471 if (p->flags & PF_EXITING) 2472 return; 2473 2474 if (!mm->numa_next_scan) { 2475 mm->numa_next_scan = now + 2476 msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 2477 } 2478 2479 /* 2480 * Enforce maximal scan/migration frequency.. 2481 */ 2482 migrate = mm->numa_next_scan; 2483 if (time_before(now, migrate)) 2484 return; 2485 2486 if (p->numa_scan_period == 0) { 2487 p->numa_scan_period_max = task_scan_max(p); 2488 p->numa_scan_period = task_scan_start(p); 2489 } 2490 2491 next_scan = now + msecs_to_jiffies(p->numa_scan_period); 2492 if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate) 2493 return; 2494 2495 /* 2496 * Delay this task enough that another task of this mm will likely win 2497 * the next time around. 2498 */ 2499 p->node_stamp += 2 * TICK_NSEC; 2500 2501 start = mm->numa_scan_offset; 2502 pages = sysctl_numa_balancing_scan_size; 2503 pages <<= 20 - PAGE_SHIFT; /* MB in pages */ 2504 virtpages = pages * 8; /* Scan up to this much virtual space */ 2505 if (!pages) 2506 return; 2507 2508 2509 if (!down_read_trylock(&mm->mmap_sem)) 2510 return; 2511 vma = find_vma(mm, start); 2512 if (!vma) { 2513 reset_ptenuma_scan(p); 2514 start = 0; 2515 vma = mm->mmap; 2516 } 2517 for (; vma; vma = vma->vm_next) { 2518 if (!vma_migratable(vma) || !vma_policy_mof(vma) || 2519 is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { 2520 continue; 2521 } 2522 2523 /* 2524 * Shared library pages mapped by multiple processes are not 2525 * migrated as it is expected they are cache replicated. Avoid 2526 * hinting faults in read-only file-backed mappings or the vdso 2527 * as migrating the pages will be of marginal benefit. 2528 */ 2529 if (!vma->vm_mm || 2530 (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ))) 2531 continue; 2532 2533 /* 2534 * Skip inaccessible VMAs to avoid any confusion between 2535 * PROT_NONE and NUMA hinting ptes 2536 */ 2537 if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))) 2538 continue; 2539 2540 do { 2541 start = max(start, vma->vm_start); 2542 end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); 2543 end = min(end, vma->vm_end); 2544 nr_pte_updates = change_prot_numa(vma, start, end); 2545 2546 /* 2547 * Try to scan sysctl_numa_balancing_size worth of 2548 * hpages that have at least one present PTE that 2549 * is not already pte-numa. If the VMA contains 2550 * areas that are unused or already full of prot_numa 2551 * PTEs, scan up to virtpages, to skip through those 2552 * areas faster. 2553 */ 2554 if (nr_pte_updates) 2555 pages -= (end - start) >> PAGE_SHIFT; 2556 virtpages -= (end - start) >> PAGE_SHIFT; 2557 2558 start = end; 2559 if (pages <= 0 || virtpages <= 0) 2560 goto out; 2561 2562 cond_resched(); 2563 } while (end != vma->vm_end); 2564 } 2565 2566 out: 2567 /* 2568 * It is possible to reach the end of the VMA list but the last few 2569 * VMAs are not guaranteed to the vma_migratable. If they are not, we 2570 * would find the !migratable VMA on the next scan but not reset the 2571 * scanner to the start so check it now. 2572 */ 2573 if (vma) 2574 mm->numa_scan_offset = start; 2575 else 2576 reset_ptenuma_scan(p); 2577 up_read(&mm->mmap_sem); 2578 2579 /* 2580 * Make sure tasks use at least 32x as much time to run other code 2581 * than they used here, to limit NUMA PTE scanning overhead to 3% max. 2582 * Usually update_task_scan_period slows down scanning enough; on an 2583 * overloaded system we need to limit overhead on a per task basis. 2584 */ 2585 if (unlikely(p->se.sum_exec_runtime != runtime)) { 2586 u64 diff = p->se.sum_exec_runtime - runtime; 2587 p->node_stamp += 32 * diff; 2588 } 2589 } 2590 2591 /* 2592 * Drive the periodic memory faults.. 2593 */ 2594 void task_tick_numa(struct rq *rq, struct task_struct *curr) 2595 { 2596 struct callback_head *work = &curr->numa_work; 2597 u64 period, now; 2598 2599 /* 2600 * We don't care about NUMA placement if we don't have memory. 2601 */ 2602 if (!curr->mm || (curr->flags & PF_EXITING) || work->next != work) 2603 return; 2604 2605 /* 2606 * Using runtime rather than walltime has the dual advantage that 2607 * we (mostly) drive the selection from busy threads and that the 2608 * task needs to have done some actual work before we bother with 2609 * NUMA placement. 2610 */ 2611 now = curr->se.sum_exec_runtime; 2612 period = (u64)curr->numa_scan_period * NSEC_PER_MSEC; 2613 2614 if (now > curr->node_stamp + period) { 2615 if (!curr->node_stamp) 2616 curr->numa_scan_period = task_scan_start(curr); 2617 curr->node_stamp += period; 2618 2619 if (!time_before(jiffies, curr->mm->numa_next_scan)) { 2620 init_task_work(work, task_numa_work); /* TODO: move this into sched_fork() */ 2621 task_work_add(curr, work, true); 2622 } 2623 } 2624 } 2625 2626 #else 2627 static void task_tick_numa(struct rq *rq, struct task_struct *curr) 2628 { 2629 } 2630 2631 static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p) 2632 { 2633 } 2634 2635 static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p) 2636 { 2637 } 2638 2639 #endif /* CONFIG_NUMA_BALANCING */ 2640 2641 static void 2642 account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) 2643 { 2644 update_load_add(&cfs_rq->load, se->load.weight); 2645 if (!parent_entity(se)) 2646 update_load_add(&rq_of(cfs_rq)->load, se->load.weight); 2647 #ifdef CONFIG_SMP 2648 if (entity_is_task(se)) { 2649 struct rq *rq = rq_of(cfs_rq); 2650 2651 account_numa_enqueue(rq, task_of(se)); 2652 list_add(&se->group_node, &rq->cfs_tasks); 2653 } 2654 #endif 2655 cfs_rq->nr_running++; 2656 } 2657 2658 static void 2659 account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) 2660 { 2661 update_load_sub(&cfs_rq->load, se->load.weight); 2662 if (!parent_entity(se)) 2663 update_load_sub(&rq_of(cfs_rq)->load, se->load.weight); 2664 #ifdef CONFIG_SMP 2665 if (entity_is_task(se)) { 2666 account_numa_dequeue(rq_of(cfs_rq), task_of(se)); 2667 list_del_init(&se->group_node); 2668 } 2669 #endif 2670 cfs_rq->nr_running--; 2671 } 2672 2673 /* 2674 * Signed add and clamp on underflow. 2675 * 2676 * Explicitly do a load-store to ensure the intermediate value never hits 2677 * memory. This allows lockless observations without ever seeing the negative 2678 * values. 2679 */ 2680 #define add_positive(_ptr, _val) do { \ 2681 typeof(_ptr) ptr = (_ptr); \ 2682 typeof(_val) val = (_val); \ 2683 typeof(*ptr) res, var = READ_ONCE(*ptr); \ 2684 \ 2685 res = var + val; \ 2686 \ 2687 if (val < 0 && res > var) \ 2688 res = 0; \ 2689 \ 2690 WRITE_ONCE(*ptr, res); \ 2691 } while (0) 2692 2693 /* 2694 * Unsigned subtract and clamp on underflow. 2695 * 2696 * Explicitly do a load-store to ensure the intermediate value never hits 2697 * memory. This allows lockless observations without ever seeing the negative 2698 * values. 2699 */ 2700 #define sub_positive(_ptr, _val) do { \ 2701 typeof(_ptr) ptr = (_ptr); \ 2702 typeof(*ptr) val = (_val); \ 2703 typeof(*ptr) res, var = READ_ONCE(*ptr); \ 2704 res = var - val; \ 2705 if (res > var) \ 2706 res = 0; \ 2707 WRITE_ONCE(*ptr, res); \ 2708 } while (0) 2709 2710 #ifdef CONFIG_SMP 2711 /* 2712 * XXX we want to get rid of these helpers and use the full load resolution. 2713 */ 2714 static inline long se_weight(struct sched_entity *se) 2715 { 2716 return scale_load_down(se->load.weight); 2717 } 2718 2719 static inline long se_runnable(struct sched_entity *se) 2720 { 2721 return scale_load_down(se->runnable_weight); 2722 } 2723 2724 static inline void 2725 enqueue_runnable_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 2726 { 2727 cfs_rq->runnable_weight += se->runnable_weight; 2728 2729 cfs_rq->avg.runnable_load_avg += se->avg.runnable_load_avg; 2730 cfs_rq->avg.runnable_load_sum += se_runnable(se) * se->avg.runnable_load_sum; 2731 } 2732 2733 static inline void 2734 dequeue_runnable_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 2735 { 2736 cfs_rq->runnable_weight -= se->runnable_weight; 2737 2738 sub_positive(&cfs_rq->avg.runnable_load_avg, se->avg.runnable_load_avg); 2739 sub_positive(&cfs_rq->avg.runnable_load_sum, 2740 se_runnable(se) * se->avg.runnable_load_sum); 2741 } 2742 2743 static inline void 2744 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 2745 { 2746 cfs_rq->avg.load_avg += se->avg.load_avg; 2747 cfs_rq->avg.load_sum += se_weight(se) * se->avg.load_sum; 2748 } 2749 2750 static inline void 2751 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 2752 { 2753 sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg); 2754 sub_positive(&cfs_rq->avg.load_sum, se_weight(se) * se->avg.load_sum); 2755 } 2756 #else 2757 static inline void 2758 enqueue_runnable_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { } 2759 static inline void 2760 dequeue_runnable_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { } 2761 static inline void 2762 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { } 2763 static inline void 2764 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { } 2765 #endif 2766 2767 static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, 2768 unsigned long weight, unsigned long runnable) 2769 { 2770 if (se->on_rq) { 2771 /* commit outstanding execution time */ 2772 if (cfs_rq->curr == se) 2773 update_curr(cfs_rq); 2774 account_entity_dequeue(cfs_rq, se); 2775 dequeue_runnable_load_avg(cfs_rq, se); 2776 } 2777 dequeue_load_avg(cfs_rq, se); 2778 2779 se->runnable_weight = runnable; 2780 update_load_set(&se->load, weight); 2781 2782 #ifdef CONFIG_SMP 2783 do { 2784 u32 divider = LOAD_AVG_MAX - 1024 + se->avg.period_contrib; 2785 2786 se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); 2787 se->avg.runnable_load_avg = 2788 div_u64(se_runnable(se) * se->avg.runnable_load_sum, divider); 2789 } while (0); 2790 #endif 2791 2792 enqueue_load_avg(cfs_rq, se); 2793 if (se->on_rq) { 2794 account_entity_enqueue(cfs_rq, se); 2795 enqueue_runnable_load_avg(cfs_rq, se); 2796 } 2797 } 2798 2799 void reweight_task(struct task_struct *p, int prio) 2800 { 2801 struct sched_entity *se = &p->se; 2802 struct cfs_rq *cfs_rq = cfs_rq_of(se); 2803 struct load_weight *load = &se->load; 2804 unsigned long weight = scale_load(sched_prio_to_weight[prio]); 2805 2806 reweight_entity(cfs_rq, se, weight, weight); 2807 load->inv_weight = sched_prio_to_wmult[prio]; 2808 } 2809 2810 #ifdef CONFIG_FAIR_GROUP_SCHED 2811 #ifdef CONFIG_SMP 2812 /* 2813 * All this does is approximate the hierarchical proportion which includes that 2814 * global sum we all love to hate. 2815 * 2816 * That is, the weight of a group entity, is the proportional share of the 2817 * group weight based on the group runqueue weights. That is: 2818 * 2819 * tg->weight * grq->load.weight 2820 * ge->load.weight = ----------------------------- (1) 2821 * \Sum grq->load.weight 2822 * 2823 * Now, because computing that sum is prohibitively expensive to compute (been 2824 * there, done that) we approximate it with this average stuff. The average 2825 * moves slower and therefore the approximation is cheaper and more stable. 2826 * 2827 * So instead of the above, we substitute: 2828 * 2829 * grq->load.weight -> grq->avg.load_avg (2) 2830 * 2831 * which yields the following: 2832 * 2833 * tg->weight * grq->avg.load_avg 2834 * ge->load.weight = ------------------------------ (3) 2835 * tg->load_avg 2836 * 2837 * Where: tg->load_avg ~= \Sum grq->avg.load_avg 2838 * 2839 * That is shares_avg, and it is right (given the approximation (2)). 2840 * 2841 * The problem with it is that because the average is slow -- it was designed 2842 * to be exactly that of course -- this leads to transients in boundary 2843 * conditions. In specific, the case where the group was idle and we start the 2844 * one task. It takes time for our CPU's grq->avg.load_avg to build up, 2845 * yielding bad latency etc.. 2846 * 2847 * Now, in that special case (1) reduces to: 2848 * 2849 * tg->weight * grq->load.weight 2850 * ge->load.weight = ----------------------------- = tg->weight (4) 2851 * grp->load.weight 2852 * 2853 * That is, the sum collapses because all other CPUs are idle; the UP scenario. 2854 * 2855 * So what we do is modify our approximation (3) to approach (4) in the (near) 2856 * UP case, like: 2857 * 2858 * ge->load.weight = 2859 * 2860 * tg->weight * grq->load.weight 2861 * --------------------------------------------------- (5) 2862 * tg->load_avg - grq->avg.load_avg + grq->load.weight 2863 * 2864 * But because grq->load.weight can drop to 0, resulting in a divide by zero, 2865 * we need to use grq->avg.load_avg as its lower bound, which then gives: 2866 * 2867 * 2868 * tg->weight * grq->load.weight 2869 * ge->load.weight = ----------------------------- (6) 2870 * tg_load_avg' 2871 * 2872 * Where: 2873 * 2874 * tg_load_avg' = tg->load_avg - grq->avg.load_avg + 2875 * max(grq->load.weight, grq->avg.load_avg) 2876 * 2877 * And that is shares_weight and is icky. In the (near) UP case it approaches 2878 * (4) while in the normal case it approaches (3). It consistently 2879 * overestimates the ge->load.weight and therefore: 2880 * 2881 * \Sum ge->load.weight >= tg->weight 2882 * 2883 * hence icky! 2884 */ 2885 static long calc_group_shares(struct cfs_rq *cfs_rq) 2886 { 2887 long tg_weight, tg_shares, load, shares; 2888 struct task_group *tg = cfs_rq->tg; 2889 2890 tg_shares = READ_ONCE(tg->shares); 2891 2892 load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); 2893 2894 tg_weight = atomic_long_read(&tg->load_avg); 2895 2896 /* Ensure tg_weight >= load */ 2897 tg_weight -= cfs_rq->tg_load_avg_contrib; 2898 tg_weight += load; 2899 2900 shares = (tg_shares * load); 2901 if (tg_weight) 2902 shares /= tg_weight; 2903 2904 /* 2905 * MIN_SHARES has to be unscaled here to support per-CPU partitioning 2906 * of a group with small tg->shares value. It is a floor value which is 2907 * assigned as a minimum load.weight to the sched_entity representing 2908 * the group on a CPU. 2909 * 2910 * E.g. on 64-bit for a group with tg->shares of scale_load(15)=15*1024 2911 * on an 8-core system with 8 tasks each runnable on one CPU shares has 2912 * to be 15*1024*1/8=1920 instead of scale_load(MIN_SHARES)=2*1024. In 2913 * case no task is runnable on a CPU MIN_SHARES=2 should be returned 2914 * instead of 0. 2915 */ 2916 return clamp_t(long, shares, MIN_SHARES, tg_shares); 2917 } 2918 2919 /* 2920 * This calculates the effective runnable weight for a group entity based on 2921 * the group entity weight calculated above. 2922 * 2923 * Because of the above approximation (2), our group entity weight is 2924 * an load_avg based ratio (3). This means that it includes blocked load and 2925 * does not represent the runnable weight. 2926 * 2927 * Approximate the group entity's runnable weight per ratio from the group 2928 * runqueue: 2929 * 2930 * grq->avg.runnable_load_avg 2931 * ge->runnable_weight = ge->load.weight * -------------------------- (7) 2932 * grq->avg.load_avg 2933 * 2934 * However, analogous to above, since the avg numbers are slow, this leads to 2935 * transients in the from-idle case. Instead we use: 2936 * 2937 * ge->runnable_weight = ge->load.weight * 2938 * 2939 * max(grq->avg.runnable_load_avg, grq->runnable_weight) 2940 * ----------------------------------------------------- (8) 2941 * max(grq->avg.load_avg, grq->load.weight) 2942 * 2943 * Where these max() serve both to use the 'instant' values to fix the slow 2944 * from-idle and avoid the /0 on to-idle, similar to (6). 2945 */ 2946 static long calc_group_runnable(struct cfs_rq *cfs_rq, long shares) 2947 { 2948 long runnable, load_avg; 2949 2950 load_avg = max(cfs_rq->avg.load_avg, 2951 scale_load_down(cfs_rq->load.weight)); 2952 2953 runnable = max(cfs_rq->avg.runnable_load_avg, 2954 scale_load_down(cfs_rq->runnable_weight)); 2955 2956 runnable *= shares; 2957 if (load_avg) 2958 runnable /= load_avg; 2959 2960 return clamp_t(long, runnable, MIN_SHARES, shares); 2961 } 2962 #endif /* CONFIG_SMP */ 2963 2964 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); 2965 2966 /* 2967 * Recomputes the group entity based on the current state of its group 2968 * runqueue. 2969 */ 2970 static void update_cfs_group(struct sched_entity *se) 2971 { 2972 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 2973 long shares, runnable; 2974 2975 if (!gcfs_rq) 2976 return; 2977 2978 if (throttled_hierarchy(gcfs_rq)) 2979 return; 2980 2981 #ifndef CONFIG_SMP 2982 runnable = shares = READ_ONCE(gcfs_rq->tg->shares); 2983 2984 if (likely(se->load.weight == shares)) 2985 return; 2986 #else 2987 shares = calc_group_shares(gcfs_rq); 2988 runnable = calc_group_runnable(gcfs_rq, shares); 2989 #endif 2990 2991 reweight_entity(cfs_rq_of(se), se, shares, runnable); 2992 } 2993 2994 #else /* CONFIG_FAIR_GROUP_SCHED */ 2995 static inline void update_cfs_group(struct sched_entity *se) 2996 { 2997 } 2998 #endif /* CONFIG_FAIR_GROUP_SCHED */ 2999 3000 static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq, int flags) 3001 { 3002 struct rq *rq = rq_of(cfs_rq); 3003 3004 if (&rq->cfs == cfs_rq || (flags & SCHED_CPUFREQ_MIGRATION)) { 3005 /* 3006 * There are a few boundary cases this might miss but it should 3007 * get called often enough that that should (hopefully) not be 3008 * a real problem. 3009 * 3010 * It will not get called when we go idle, because the idle 3011 * thread is a different class (!fair), nor will the utilization 3012 * number include things like RT tasks. 3013 * 3014 * As is, the util number is not freq-invariant (we'd have to 3015 * implement arch_scale_freq_capacity() for that). 3016 * 3017 * See cpu_util(). 3018 */ 3019 cpufreq_update_util(rq, flags); 3020 } 3021 } 3022 3023 #ifdef CONFIG_SMP 3024 /* 3025 * Approximate: 3026 * val * y^n, where y^32 ~= 0.5 (~1 scheduling period) 3027 */ 3028 static u64 decay_load(u64 val, u64 n) 3029 { 3030 unsigned int local_n; 3031 3032 if (unlikely(n > LOAD_AVG_PERIOD * 63)) 3033 return 0; 3034 3035 /* after bounds checking we can collapse to 32-bit */ 3036 local_n = n; 3037 3038 /* 3039 * As y^PERIOD = 1/2, we can combine 3040 * y^n = 1/2^(n/PERIOD) * y^(n%PERIOD) 3041 * With a look-up table which covers y^n (n<PERIOD) 3042 * 3043 * To achieve constant time decay_load. 3044 */ 3045 if (unlikely(local_n >= LOAD_AVG_PERIOD)) { 3046 val >>= local_n / LOAD_AVG_PERIOD; 3047 local_n %= LOAD_AVG_PERIOD; 3048 } 3049 3050 val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32); 3051 return val; 3052 } 3053 3054 static u32 __accumulate_pelt_segments(u64 periods, u32 d1, u32 d3) 3055 { 3056 u32 c1, c2, c3 = d3; /* y^0 == 1 */ 3057 3058 /* 3059 * c1 = d1 y^p 3060 */ 3061 c1 = decay_load((u64)d1, periods); 3062 3063 /* 3064 * p-1 3065 * c2 = 1024 \Sum y^n 3066 * n=1 3067 * 3068 * inf inf 3069 * = 1024 ( \Sum y^n - \Sum y^n - y^0 ) 3070 * n=0 n=p 3071 */ 3072 c2 = LOAD_AVG_MAX - decay_load(LOAD_AVG_MAX, periods) - 1024; 3073 3074 return c1 + c2 + c3; 3075 } 3076 3077 /* 3078 * Accumulate the three separate parts of the sum; d1 the remainder 3079 * of the last (incomplete) period, d2 the span of full periods and d3 3080 * the remainder of the (incomplete) current period. 3081 * 3082 * d1 d2 d3 3083 * ^ ^ ^ 3084 * | | | 3085 * |<->|<----------------->|<--->| 3086 * ... |---x---|------| ... |------|-----x (now) 3087 * 3088 * p-1 3089 * u' = (u + d1) y^p + 1024 \Sum y^n + d3 y^0 3090 * n=1 3091 * 3092 * = u y^p + (Step 1) 3093 * 3094 * p-1 3095 * d1 y^p + 1024 \Sum y^n + d3 y^0 (Step 2) 3096 * n=1 3097 */ 3098 static __always_inline u32 3099 accumulate_sum(u64 delta, int cpu, struct sched_avg *sa, 3100 unsigned long load, unsigned long runnable, int running) 3101 { 3102 unsigned long scale_freq, scale_cpu; 3103 u32 contrib = (u32)delta; /* p == 0 -> delta < 1024 */ 3104 u64 periods; 3105 3106 scale_freq = arch_scale_freq_capacity(cpu); 3107 scale_cpu = arch_scale_cpu_capacity(NULL, cpu); 3108 3109 delta += sa->period_contrib; 3110 periods = delta / 1024; /* A period is 1024us (~1ms) */ 3111 3112 /* 3113 * Step 1: decay old *_sum if we crossed period boundaries. 3114 */ 3115 if (periods) { 3116 sa->load_sum = decay_load(sa->load_sum, periods); 3117 sa->runnable_load_sum = 3118 decay_load(sa->runnable_load_sum, periods); 3119 sa->util_sum = decay_load((u64)(sa->util_sum), periods); 3120 3121 /* 3122 * Step 2 3123 */ 3124 delta %= 1024; 3125 contrib = __accumulate_pelt_segments(periods, 3126 1024 - sa->period_contrib, delta); 3127 } 3128 sa->period_contrib = delta; 3129 3130 contrib = cap_scale(contrib, scale_freq); 3131 if (load) 3132 sa->load_sum += load * contrib; 3133 if (runnable) 3134 sa->runnable_load_sum += runnable * contrib; 3135 if (running) 3136 sa->util_sum += contrib * scale_cpu; 3137 3138 return periods; 3139 } 3140 3141 /* 3142 * We can represent the historical contribution to runnable average as the 3143 * coefficients of a geometric series. To do this we sub-divide our runnable 3144 * history into segments of approximately 1ms (1024us); label the segment that 3145 * occurred N-ms ago p_N, with p_0 corresponding to the current period, e.g. 3146 * 3147 * [<- 1024us ->|<- 1024us ->|<- 1024us ->| ... 3148 * p0 p1 p2 3149 * (now) (~1ms ago) (~2ms ago) 3150 * 3151 * Let u_i denote the fraction of p_i that the entity was runnable. 3152 * 3153 * We then designate the fractions u_i as our co-efficients, yielding the 3154 * following representation of historical load: 3155 * u_0 + u_1*y + u_2*y^2 + u_3*y^3 + ... 3156 * 3157 * We choose y based on the with of a reasonably scheduling period, fixing: 3158 * y^32 = 0.5 3159 * 3160 * This means that the contribution to load ~32ms ago (u_32) will be weighted 3161 * approximately half as much as the contribution to load within the last ms 3162 * (u_0). 3163 * 3164 * When a period "rolls over" and we have new u_0`, multiplying the previous 3165 * sum again by y is sufficient to update: 3166 * load_avg = u_0` + y*(u_0 + u_1*y + u_2*y^2 + ... ) 3167 * = u_0 + u_1*y + u_2*y^2 + ... [re-labeling u_i --> u_{i+1}] 3168 */ 3169 static __always_inline int 3170 ___update_load_sum(u64 now, int cpu, struct sched_avg *sa, 3171 unsigned long load, unsigned long runnable, int running) 3172 { 3173 u64 delta; 3174 3175 delta = now - sa->last_update_time; 3176 /* 3177 * This should only happen when time goes backwards, which it 3178 * unfortunately does during sched clock init when we swap over to TSC. 3179 */ 3180 if ((s64)delta < 0) { 3181 sa->last_update_time = now; 3182 return 0; 3183 } 3184 3185 /* 3186 * Use 1024ns as the unit of measurement since it's a reasonable 3187 * approximation of 1us and fast to compute. 3188 */ 3189 delta >>= 10; 3190 if (!delta) 3191 return 0; 3192 3193 sa->last_update_time += delta << 10; 3194 3195 /* 3196 * running is a subset of runnable (weight) so running can't be set if 3197 * runnable is clear. But there are some corner cases where the current 3198 * se has been already dequeued but cfs_rq->curr still points to it. 3199 * This means that weight will be 0 but not running for a sched_entity 3200 * but also for a cfs_rq if the latter becomes idle. As an example, 3201 * this happens during idle_balance() which calls 3202 * update_blocked_averages() 3203 */ 3204 if (!load) 3205 runnable = running = 0; 3206 3207 /* 3208 * Now we know we crossed measurement unit boundaries. The *_avg 3209 * accrues by two steps: 3210 * 3211 * Step 1: accumulate *_sum since last_update_time. If we haven't 3212 * crossed period boundaries, finish. 3213 */ 3214 if (!accumulate_sum(delta, cpu, sa, load, runnable, running)) 3215 return 0; 3216 3217 return 1; 3218 } 3219 3220 static __always_inline void 3221 ___update_load_avg(struct sched_avg *sa, unsigned long load, unsigned long runnable) 3222 { 3223 u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 3224 3225 /* 3226 * Step 2: update *_avg. 3227 */ 3228 sa->load_avg = div_u64(load * sa->load_sum, divider); 3229 sa->runnable_load_avg = div_u64(runnable * sa->runnable_load_sum, divider); 3230 sa->util_avg = sa->util_sum / divider; 3231 } 3232 3233 /* 3234 * When a task is dequeued, its estimated utilization should not be update if 3235 * its util_avg has not been updated at least once. 3236 * This flag is used to synchronize util_avg updates with util_est updates. 3237 * We map this information into the LSB bit of the utilization saved at 3238 * dequeue time (i.e. util_est.dequeued). 3239 */ 3240 #define UTIL_AVG_UNCHANGED 0x1 3241 3242 static inline void cfs_se_util_change(struct sched_avg *avg) 3243 { 3244 unsigned int enqueued; 3245 3246 if (!sched_feat(UTIL_EST)) 3247 return; 3248 3249 /* Avoid store if the flag has been already set */ 3250 enqueued = avg->util_est.enqueued; 3251 if (!(enqueued & UTIL_AVG_UNCHANGED)) 3252 return; 3253 3254 /* Reset flag to report util_avg has been updated */ 3255 enqueued &= ~UTIL_AVG_UNCHANGED; 3256 WRITE_ONCE(avg->util_est.enqueued, enqueued); 3257 } 3258 3259 /* 3260 * sched_entity: 3261 * 3262 * task: 3263 * se_runnable() == se_weight() 3264 * 3265 * group: [ see update_cfs_group() ] 3266 * se_weight() = tg->weight * grq->load_avg / tg->load_avg 3267 * se_runnable() = se_weight(se) * grq->runnable_load_avg / grq->load_avg 3268 * 3269 * load_sum := runnable_sum 3270 * load_avg = se_weight(se) * runnable_avg 3271 * 3272 * runnable_load_sum := runnable_sum 3273 * runnable_load_avg = se_runnable(se) * runnable_avg 3274 * 3275 * XXX collapse load_sum and runnable_load_sum 3276 * 3277 * cfq_rs: 3278 * 3279 * load_sum = \Sum se_weight(se) * se->avg.load_sum 3280 * load_avg = \Sum se->avg.load_avg 3281 * 3282 * runnable_load_sum = \Sum se_runnable(se) * se->avg.runnable_load_sum 3283 * runnable_load_avg = \Sum se->avg.runable_load_avg 3284 */ 3285 3286 static int 3287 __update_load_avg_blocked_se(u64 now, int cpu, struct sched_entity *se) 3288 { 3289 if (entity_is_task(se)) 3290 se->runnable_weight = se->load.weight; 3291 3292 if (___update_load_sum(now, cpu, &se->avg, 0, 0, 0)) { 3293 ___update_load_avg(&se->avg, se_weight(se), se_runnable(se)); 3294 return 1; 3295 } 3296 3297 return 0; 3298 } 3299 3300 static int 3301 __update_load_avg_se(u64 now, int cpu, struct cfs_rq *cfs_rq, struct sched_entity *se) 3302 { 3303 if (entity_is_task(se)) 3304 se->runnable_weight = se->load.weight; 3305 3306 if (___update_load_sum(now, cpu, &se->avg, !!se->on_rq, !!se->on_rq, 3307 cfs_rq->curr == se)) { 3308 3309 ___update_load_avg(&se->avg, se_weight(se), se_runnable(se)); 3310 cfs_se_util_change(&se->avg); 3311 return 1; 3312 } 3313 3314 return 0; 3315 } 3316 3317 static int 3318 __update_load_avg_cfs_rq(u64 now, int cpu, struct cfs_rq *cfs_rq) 3319 { 3320 if (___update_load_sum(now, cpu, &cfs_rq->avg, 3321 scale_load_down(cfs_rq->load.weight), 3322 scale_load_down(cfs_rq->runnable_weight), 3323 cfs_rq->curr != NULL)) { 3324 3325 ___update_load_avg(&cfs_rq->avg, 1, 1); 3326 return 1; 3327 } 3328 3329 return 0; 3330 } 3331 3332 #ifdef CONFIG_FAIR_GROUP_SCHED 3333 /** 3334 * update_tg_load_avg - update the tg's load avg 3335 * @cfs_rq: the cfs_rq whose avg changed 3336 * @force: update regardless of how small the difference 3337 * 3338 * This function 'ensures': tg->load_avg := \Sum tg->cfs_rq[]->avg.load. 3339 * However, because tg->load_avg is a global value there are performance 3340 * considerations. 3341 * 3342 * In order to avoid having to look at the other cfs_rq's, we use a 3343 * differential update where we store the last value we propagated. This in 3344 * turn allows skipping updates if the differential is 'small'. 3345 * 3346 * Updating tg's load_avg is necessary before update_cfs_share(). 3347 */ 3348 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) 3349 { 3350 long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; 3351 3352 /* 3353 * No need to update load_avg for root_task_group as it is not used. 3354 */ 3355 if (cfs_rq->tg == &root_task_group) 3356 return; 3357 3358 if (force || abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { 3359 atomic_long_add(delta, &cfs_rq->tg->load_avg); 3360 cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; 3361 } 3362 } 3363 3364 /* 3365 * Called within set_task_rq() right before setting a task's CPU. The 3366 * caller only guarantees p->pi_lock is held; no other assumptions, 3367 * including the state of rq->lock, should be made. 3368 */ 3369 void set_task_rq_fair(struct sched_entity *se, 3370 struct cfs_rq *prev, struct cfs_rq *next) 3371 { 3372 u64 p_last_update_time; 3373 u64 n_last_update_time; 3374 3375 if (!sched_feat(ATTACH_AGE_LOAD)) 3376 return; 3377 3378 /* 3379 * We are supposed to update the task to "current" time, then its up to 3380 * date and ready to go to new CPU/cfs_rq. But we have difficulty in 3381 * getting what current time is, so simply throw away the out-of-date 3382 * time. This will result in the wakee task is less decayed, but giving 3383 * the wakee more load sounds not bad. 3384 */ 3385 if (!(se->avg.last_update_time && prev)) 3386 return; 3387 3388 #ifndef CONFIG_64BIT 3389 { 3390 u64 p_last_update_time_copy; 3391 u64 n_last_update_time_copy; 3392 3393 do { 3394 p_last_update_time_copy = prev->load_last_update_time_copy; 3395 n_last_update_time_copy = next->load_last_update_time_copy; 3396 3397 smp_rmb(); 3398 3399 p_last_update_time = prev->avg.last_update_time; 3400 n_last_update_time = next->avg.last_update_time; 3401 3402 } while (p_last_update_time != p_last_update_time_copy || 3403 n_last_update_time != n_last_update_time_copy); 3404 } 3405 #else 3406 p_last_update_time = prev->avg.last_update_time; 3407 n_last_update_time = next->avg.last_update_time; 3408 #endif 3409 __update_load_avg_blocked_se(p_last_update_time, cpu_of(rq_of(prev)), se); 3410 se->avg.last_update_time = n_last_update_time; 3411 } 3412 3413 3414 /* 3415 * When on migration a sched_entity joins/leaves the PELT hierarchy, we need to 3416 * propagate its contribution. The key to this propagation is the invariant 3417 * that for each group: 3418 * 3419 * ge->avg == grq->avg (1) 3420 * 3421 * _IFF_ we look at the pure running and runnable sums. Because they 3422 * represent the very same entity, just at different points in the hierarchy. 3423 * 3424 * Per the above update_tg_cfs_util() is trivial and simply copies the running 3425 * sum over (but still wrong, because the group entity and group rq do not have 3426 * their PELT windows aligned). 3427 * 3428 * However, update_tg_cfs_runnable() is more complex. So we have: 3429 * 3430 * ge->avg.load_avg = ge->load.weight * ge->avg.runnable_avg (2) 3431 * 3432 * And since, like util, the runnable part should be directly transferable, 3433 * the following would _appear_ to be the straight forward approach: 3434 * 3435 * grq->avg.load_avg = grq->load.weight * grq->avg.runnable_avg (3) 3436 * 3437 * And per (1) we have: 3438 * 3439 * ge->avg.runnable_avg == grq->avg.runnable_avg 3440 * 3441 * Which gives: 3442 * 3443 * ge->load.weight * grq->avg.load_avg 3444 * ge->avg.load_avg = ----------------------------------- (4) 3445 * grq->load.weight 3446 * 3447 * Except that is wrong! 3448 * 3449 * Because while for entities historical weight is not important and we 3450 * really only care about our future and therefore can consider a pure 3451 * runnable sum, runqueues can NOT do this. 3452 * 3453 * We specifically want runqueues to have a load_avg that includes 3454 * historical weights. Those represent the blocked load, the load we expect 3455 * to (shortly) return to us. This only works by keeping the weights as 3456 * integral part of the sum. We therefore cannot decompose as per (3). 3457 * 3458 * Another reason this doesn't work is that runnable isn't a 0-sum entity. 3459 * Imagine a rq with 2 tasks that each are runnable 2/3 of the time. Then the 3460 * rq itself is runnable anywhere between 2/3 and 1 depending on how the 3461 * runnable section of these tasks overlap (or not). If they were to perfectly 3462 * align the rq as a whole would be runnable 2/3 of the time. If however we 3463 * always have at least 1 runnable task, the rq as a whole is always runnable. 3464 * 3465 * So we'll have to approximate.. :/ 3466 * 3467 * Given the constraint: 3468 * 3469 * ge->avg.running_sum <= ge->avg.runnable_sum <= LOAD_AVG_MAX 3470 * 3471 * We can construct a rule that adds runnable to a rq by assuming minimal 3472 * overlap. 3473 * 3474 * On removal, we'll assume each task is equally runnable; which yields: 3475 * 3476 * grq->avg.runnable_sum = grq->avg.load_sum / grq->load.weight 3477 * 3478 * XXX: only do this for the part of runnable > running ? 3479 * 3480 */ 3481 3482 static inline void 3483 update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 3484 { 3485 long delta = gcfs_rq->avg.util_avg - se->avg.util_avg; 3486 3487 /* Nothing to update */ 3488 if (!delta) 3489 return; 3490 3491 /* 3492 * The relation between sum and avg is: 3493 * 3494 * LOAD_AVG_MAX - 1024 + sa->period_contrib 3495 * 3496 * however, the PELT windows are not aligned between grq and gse. 3497 */ 3498 3499 /* Set new sched_entity's utilization */ 3500 se->avg.util_avg = gcfs_rq->avg.util_avg; 3501 se->avg.util_sum = se->avg.util_avg * LOAD_AVG_MAX; 3502 3503 /* Update parent cfs_rq utilization */ 3504 add_positive(&cfs_rq->avg.util_avg, delta); 3505 cfs_rq->avg.util_sum = cfs_rq->avg.util_avg * LOAD_AVG_MAX; 3506 } 3507 3508 static inline void 3509 update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 3510 { 3511 long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum; 3512 unsigned long runnable_load_avg, load_avg; 3513 u64 runnable_load_sum, load_sum = 0; 3514 s64 delta_sum; 3515 3516 if (!runnable_sum) 3517 return; 3518 3519 gcfs_rq->prop_runnable_sum = 0; 3520 3521 if (runnable_sum >= 0) { 3522 /* 3523 * Add runnable; clip at LOAD_AVG_MAX. Reflects that until 3524 * the CPU is saturated running == runnable. 3525 */ 3526 runnable_sum += se->avg.load_sum; 3527 runnable_sum = min(runnable_sum, (long)LOAD_AVG_MAX); 3528 } else { 3529 /* 3530 * Estimate the new unweighted runnable_sum of the gcfs_rq by 3531 * assuming all tasks are equally runnable. 3532 */ 3533 if (scale_load_down(gcfs_rq->load.weight)) { 3534 load_sum = div_s64(gcfs_rq->avg.load_sum, 3535 scale_load_down(gcfs_rq->load.weight)); 3536 } 3537 3538 /* But make sure to not inflate se's runnable */ 3539 runnable_sum = min(se->avg.load_sum, load_sum); 3540 } 3541 3542 /* 3543 * runnable_sum can't be lower than running_sum 3544 * As running sum is scale with CPU capacity wehreas the runnable sum 3545 * is not we rescale running_sum 1st 3546 */ 3547 running_sum = se->avg.util_sum / 3548 arch_scale_cpu_capacity(NULL, cpu_of(rq_of(cfs_rq))); 3549 runnable_sum = max(runnable_sum, running_sum); 3550 3551 load_sum = (s64)se_weight(se) * runnable_sum; 3552 load_avg = div_s64(load_sum, LOAD_AVG_MAX); 3553 3554 delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum; 3555 delta_avg = load_avg - se->avg.load_avg; 3556 3557 se->avg.load_sum = runnable_sum; 3558 se->avg.load_avg = load_avg; 3559 add_positive(&cfs_rq->avg.load_avg, delta_avg); 3560 add_positive(&cfs_rq->avg.load_sum, delta_sum); 3561 3562 runnable_load_sum = (s64)se_runnable(se) * runnable_sum; 3563 runnable_load_avg = div_s64(runnable_load_sum, LOAD_AVG_MAX); 3564 delta_sum = runnable_load_sum - se_weight(se) * se->avg.runnable_load_sum; 3565 delta_avg = runnable_load_avg - se->avg.runnable_load_avg; 3566 3567 se->avg.runnable_load_sum = runnable_sum; 3568 se->avg.runnable_load_avg = runnable_load_avg; 3569 3570 if (se->on_rq) { 3571 add_positive(&cfs_rq->avg.runnable_load_avg, delta_avg); 3572 add_positive(&cfs_rq->avg.runnable_load_sum, delta_sum); 3573 } 3574 } 3575 3576 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) 3577 { 3578 cfs_rq->propagate = 1; 3579 cfs_rq->prop_runnable_sum += runnable_sum; 3580 } 3581 3582 /* Update task and its cfs_rq load average */ 3583 static inline int propagate_entity_load_avg(struct sched_entity *se) 3584 { 3585 struct cfs_rq *cfs_rq, *gcfs_rq; 3586 3587 if (entity_is_task(se)) 3588 return 0; 3589 3590 gcfs_rq = group_cfs_rq(se); 3591 if (!gcfs_rq->propagate) 3592 return 0; 3593 3594 gcfs_rq->propagate = 0; 3595 3596 cfs_rq = cfs_rq_of(se); 3597 3598 add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum); 3599 3600 update_tg_cfs_util(cfs_rq, se, gcfs_rq); 3601 update_tg_cfs_runnable(cfs_rq, se, gcfs_rq); 3602 3603 return 1; 3604 } 3605 3606 /* 3607 * Check if we need to update the load and the utilization of a blocked 3608 * group_entity: 3609 */ 3610 static inline bool skip_blocked_update(struct sched_entity *se) 3611 { 3612 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 3613 3614 /* 3615 * If sched_entity still have not zero load or utilization, we have to 3616 * decay it: 3617 */ 3618 if (se->avg.load_avg || se->avg.util_avg) 3619 return false; 3620 3621 /* 3622 * If there is a pending propagation, we have to update the load and 3623 * the utilization of the sched_entity: 3624 */ 3625 if (gcfs_rq->propagate) 3626 return false; 3627 3628 /* 3629 * Otherwise, the load and the utilization of the sched_entity is 3630 * already zero and there is no pending propagation, so it will be a 3631 * waste of time to try to decay it: 3632 */ 3633 return true; 3634 } 3635 3636 #else /* CONFIG_FAIR_GROUP_SCHED */ 3637 3638 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) {} 3639 3640 static inline int propagate_entity_load_avg(struct sched_entity *se) 3641 { 3642 return 0; 3643 } 3644 3645 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) {} 3646 3647 #endif /* CONFIG_FAIR_GROUP_SCHED */ 3648 3649 /** 3650 * update_cfs_rq_load_avg - update the cfs_rq's load/util averages 3651 * @now: current time, as per cfs_rq_clock_task() 3652 * @cfs_rq: cfs_rq to update 3653 * 3654 * The cfs_rq avg is the direct sum of all its entities (blocked and runnable) 3655 * avg. The immediate corollary is that all (fair) tasks must be attached, see 3656 * post_init_entity_util_avg(). 3657 * 3658 * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example. 3659 * 3660 * Returns true if the load decayed or we removed load. 3661 * 3662 * Since both these conditions indicate a changed cfs_rq->avg.load we should 3663 * call update_tg_load_avg() when this function returns true. 3664 */ 3665 static inline int 3666 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) 3667 { 3668 unsigned long removed_load = 0, removed_util = 0, removed_runnable_sum = 0; 3669 struct sched_avg *sa = &cfs_rq->avg; 3670 int decayed = 0; 3671 3672 if (cfs_rq->removed.nr) { 3673 unsigned long r; 3674 u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 3675 3676 raw_spin_lock(&cfs_rq->removed.lock); 3677 swap(cfs_rq->removed.util_avg, removed_util); 3678 swap(cfs_rq->removed.load_avg, removed_load); 3679 swap(cfs_rq->removed.runnable_sum, removed_runnable_sum); 3680 cfs_rq->removed.nr = 0; 3681 raw_spin_unlock(&cfs_rq->removed.lock); 3682 3683 r = removed_load; 3684 sub_positive(&sa->load_avg, r); 3685 sub_positive(&sa->load_sum, r * divider); 3686 3687 r = removed_util; 3688 sub_positive(&sa->util_avg, r); 3689 sub_positive(&sa->util_sum, r * divider); 3690 3691 add_tg_cfs_propagate(cfs_rq, -(long)removed_runnable_sum); 3692 3693 decayed = 1; 3694 } 3695 3696 decayed |= __update_load_avg_cfs_rq(now, cpu_of(rq_of(cfs_rq)), cfs_rq); 3697 3698 #ifndef CONFIG_64BIT 3699 smp_wmb(); 3700 cfs_rq->load_last_update_time_copy = sa->last_update_time; 3701 #endif 3702 3703 if (decayed) 3704 cfs_rq_util_change(cfs_rq, 0); 3705 3706 return decayed; 3707 } 3708 3709 /** 3710 * attach_entity_load_avg - attach this entity to its cfs_rq load avg 3711 * @cfs_rq: cfs_rq to attach to 3712 * @se: sched_entity to attach 3713 * 3714 * Must call update_cfs_rq_load_avg() before this, since we rely on 3715 * cfs_rq->avg.last_update_time being current. 3716 */ 3717 static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 3718 { 3719 u32 divider = LOAD_AVG_MAX - 1024 + cfs_rq->avg.period_contrib; 3720 3721 /* 3722 * When we attach the @se to the @cfs_rq, we must align the decay 3723 * window because without that, really weird and wonderful things can 3724 * happen. 3725 * 3726 * XXX illustrate 3727 */ 3728 se->avg.last_update_time = cfs_rq->avg.last_update_time; 3729 se->avg.period_contrib = cfs_rq->avg.period_contrib; 3730 3731 /* 3732 * Hell(o) Nasty stuff.. we need to recompute _sum based on the new 3733 * period_contrib. This isn't strictly correct, but since we're 3734 * entirely outside of the PELT hierarchy, nobody cares if we truncate 3735 * _sum a little. 3736 */ 3737 se->avg.util_sum = se->avg.util_avg * divider; 3738 3739 se->avg.load_sum = divider; 3740 if (se_weight(se)) { 3741 se->avg.load_sum = 3742 div_u64(se->avg.load_avg * se->avg.load_sum, se_weight(se)); 3743 } 3744 3745 se->avg.runnable_load_sum = se->avg.load_sum; 3746 3747 enqueue_load_avg(cfs_rq, se); 3748 cfs_rq->avg.util_avg += se->avg.util_avg; 3749 cfs_rq->avg.util_sum += se->avg.util_sum; 3750 3751 add_tg_cfs_propagate(cfs_rq, se->avg.load_sum); 3752 3753 cfs_rq_util_change(cfs_rq, flags); 3754 } 3755 3756 /** 3757 * detach_entity_load_avg - detach this entity from its cfs_rq load avg 3758 * @cfs_rq: cfs_rq to detach from 3759 * @se: sched_entity to detach 3760 * 3761 * Must call update_cfs_rq_load_avg() before this, since we rely on 3762 * cfs_rq->avg.last_update_time being current. 3763 */ 3764 static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 3765 { 3766 dequeue_load_avg(cfs_rq, se); 3767 sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg); 3768 sub_positive(&cfs_rq->avg.util_sum, se->avg.util_sum); 3769 3770 add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum); 3771 3772 cfs_rq_util_change(cfs_rq, 0); 3773 } 3774 3775 /* 3776 * Optional action to be done while updating the load average 3777 */ 3778 #define UPDATE_TG 0x1 3779 #define SKIP_AGE_LOAD 0x2 3780 #define DO_ATTACH 0x4 3781 3782 /* Update task and its cfs_rq load average */ 3783 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 3784 { 3785 u64 now = cfs_rq_clock_task(cfs_rq); 3786 struct rq *rq = rq_of(cfs_rq); 3787 int cpu = cpu_of(rq); 3788 int decayed; 3789 3790 /* 3791 * Track task load average for carrying it to new CPU after migrated, and 3792 * track group sched_entity load average for task_h_load calc in migration 3793 */ 3794 if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) 3795 __update_load_avg_se(now, cpu, cfs_rq, se); 3796 3797 decayed = update_cfs_rq_load_avg(now, cfs_rq); 3798 decayed |= propagate_entity_load_avg(se); 3799 3800 if (!se->avg.last_update_time && (flags & DO_ATTACH)) { 3801 3802 /* 3803 * DO_ATTACH means we're here from enqueue_entity(). 3804 * !last_update_time means we've passed through 3805 * migrate_task_rq_fair() indicating we migrated. 3806 * 3807 * IOW we're enqueueing a task on a new CPU. 3808 */ 3809 attach_entity_load_avg(cfs_rq, se, SCHED_CPUFREQ_MIGRATION); 3810 update_tg_load_avg(cfs_rq, 0); 3811 3812 } else if (decayed && (flags & UPDATE_TG)) 3813 update_tg_load_avg(cfs_rq, 0); 3814 } 3815 3816 #ifndef CONFIG_64BIT 3817 static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) 3818 { 3819 u64 last_update_time_copy; 3820 u64 last_update_time; 3821 3822 do { 3823 last_update_time_copy = cfs_rq->load_last_update_time_copy; 3824 smp_rmb(); 3825 last_update_time = cfs_rq->avg.last_update_time; 3826 } while (last_update_time != last_update_time_copy); 3827 3828 return last_update_time; 3829 } 3830 #else 3831 static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) 3832 { 3833 return cfs_rq->avg.last_update_time; 3834 } 3835 #endif 3836 3837 /* 3838 * Synchronize entity load avg of dequeued entity without locking 3839 * the previous rq. 3840 */ 3841 void sync_entity_load_avg(struct sched_entity *se) 3842 { 3843 struct cfs_rq *cfs_rq = cfs_rq_of(se); 3844 u64 last_update_time; 3845 3846 last_update_time = cfs_rq_last_update_time(cfs_rq); 3847 __update_load_avg_blocked_se(last_update_time, cpu_of(rq_of(cfs_rq)), se); 3848 } 3849 3850 /* 3851 * Task first catches up with cfs_rq, and then subtract 3852 * itself from the cfs_rq (task must be off the queue now). 3853 */ 3854 void remove_entity_load_avg(struct sched_entity *se) 3855 { 3856 struct cfs_rq *cfs_rq = cfs_rq_of(se); 3857 unsigned long flags; 3858 3859 /* 3860 * tasks cannot exit without having gone through wake_up_new_task() -> 3861 * post_init_entity_util_avg() which will have added things to the 3862 * cfs_rq, so we can remove unconditionally. 3863 * 3864 * Similarly for groups, they will have passed through 3865 * post_init_entity_util_avg() before unregister_sched_fair_group() 3866 * calls this. 3867 */ 3868 3869 sync_entity_load_avg(se); 3870 3871 raw_spin_lock_irqsave(&cfs_rq->removed.lock, flags); 3872 ++cfs_rq->removed.nr; 3873 cfs_rq->removed.util_avg += se->avg.util_avg; 3874 cfs_rq->removed.load_avg += se->avg.load_avg; 3875 cfs_rq->removed.runnable_sum += se->avg.load_sum; /* == runnable_sum */ 3876 raw_spin_unlock_irqrestore(&cfs_rq->removed.lock, flags); 3877 } 3878 3879 static inline unsigned long cfs_rq_runnable_load_avg(struct cfs_rq *cfs_rq) 3880 { 3881 return cfs_rq->avg.runnable_load_avg; 3882 } 3883 3884 static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) 3885 { 3886 return cfs_rq->avg.load_avg; 3887 } 3888 3889 static int idle_balance(struct rq *this_rq, struct rq_flags *rf); 3890 3891 static inline unsigned long task_util(struct task_struct *p) 3892 { 3893 return READ_ONCE(p->se.avg.util_avg); 3894 } 3895 3896 static inline unsigned long _task_util_est(struct task_struct *p) 3897 { 3898 struct util_est ue = READ_ONCE(p->se.avg.util_est); 3899 3900 return max(ue.ewma, ue.enqueued); 3901 } 3902 3903 static inline unsigned long task_util_est(struct task_struct *p) 3904 { 3905 return max(task_util(p), _task_util_est(p)); 3906 } 3907 3908 static inline void util_est_enqueue(struct cfs_rq *cfs_rq, 3909 struct task_struct *p) 3910 { 3911 unsigned int enqueued; 3912 3913 if (!sched_feat(UTIL_EST)) 3914 return; 3915 3916 /* Update root cfs_rq's estimated utilization */ 3917 enqueued = cfs_rq->avg.util_est.enqueued; 3918 enqueued += (_task_util_est(p) | UTIL_AVG_UNCHANGED); 3919 WRITE_ONCE(cfs_rq->avg.util_est.enqueued, enqueued); 3920 } 3921 3922 /* 3923 * Check if a (signed) value is within a specified (unsigned) margin, 3924 * based on the observation that: 3925 * 3926 * abs(x) < y := (unsigned)(x + y - 1) < (2 * y - 1) 3927 * 3928 * NOTE: this only works when value + maring < INT_MAX. 3929 */ 3930 static inline bool within_margin(int value, int margin) 3931 { 3932 return ((unsigned int)(value + margin - 1) < (2 * margin - 1)); 3933 } 3934 3935 static void 3936 util_est_dequeue(struct cfs_rq *cfs_rq, struct task_struct *p, bool task_sleep) 3937 { 3938 long last_ewma_diff; 3939 struct util_est ue; 3940 3941 if (!sched_feat(UTIL_EST)) 3942 return; 3943 3944 /* Update root cfs_rq's estimated utilization */ 3945 ue.enqueued = cfs_rq->avg.util_est.enqueued; 3946 ue.enqueued -= min_t(unsigned int, ue.enqueued, 3947 (_task_util_est(p) | UTIL_AVG_UNCHANGED)); 3948 WRITE_ONCE(cfs_rq->avg.util_est.enqueued, ue.enqueued); 3949 3950 /* 3951 * Skip update of task's estimated utilization when the task has not 3952 * yet completed an activation, e.g. being migrated. 3953 */ 3954 if (!task_sleep) 3955 return; 3956 3957 /* 3958 * If the PELT values haven't changed since enqueue time, 3959 * skip the util_est update. 3960 */ 3961 ue = p->se.avg.util_est; 3962 if (ue.enqueued & UTIL_AVG_UNCHANGED) 3963 return; 3964 3965 /* 3966 * Skip update of task's estimated utilization when its EWMA is 3967 * already ~1% close to its last activation value. 3968 */ 3969 ue.enqueued = (task_util(p) | UTIL_AVG_UNCHANGED); 3970 last_ewma_diff = ue.enqueued - ue.ewma; 3971 if (within_margin(last_ewma_diff, (SCHED_CAPACITY_SCALE / 100))) 3972 return; 3973 3974 /* 3975 * Update Task's estimated utilization 3976 * 3977 * When *p completes an activation we can consolidate another sample 3978 * of the task size. This is done by storing the current PELT value 3979 * as ue.enqueued and by using this value to update the Exponential 3980 * Weighted Moving Average (EWMA): 3981 * 3982 * ewma(t) = w * task_util(p) + (1-w) * ewma(t-1) 3983 * = w * task_util(p) + ewma(t-1) - w * ewma(t-1) 3984 * = w * (task_util(p) - ewma(t-1)) + ewma(t-1) 3985 * = w * ( last_ewma_diff ) + ewma(t-1) 3986 * = w * (last_ewma_diff + ewma(t-1) / w) 3987 * 3988 * Where 'w' is the weight of new samples, which is configured to be 3989 * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT) 3990 */ 3991 ue.ewma <<= UTIL_EST_WEIGHT_SHIFT; 3992 ue.ewma += last_ewma_diff; 3993 ue.ewma >>= UTIL_EST_WEIGHT_SHIFT; 3994 WRITE_ONCE(p->se.avg.util_est, ue); 3995 } 3996 3997 #else /* CONFIG_SMP */ 3998 3999 static inline int 4000 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) 4001 { 4002 return 0; 4003 } 4004 4005 #define UPDATE_TG 0x0 4006 #define SKIP_AGE_LOAD 0x0 4007 #define DO_ATTACH 0x0 4008 4009 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int not_used1) 4010 { 4011 cfs_rq_util_change(cfs_rq, 0); 4012 } 4013 4014 static inline void remove_entity_load_avg(struct sched_entity *se) {} 4015 4016 static inline void 4017 attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) {} 4018 static inline void 4019 detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} 4020 4021 static inline int idle_balance(struct rq *rq, struct rq_flags *rf) 4022 { 4023 return 0; 4024 } 4025 4026 static inline void 4027 util_est_enqueue(struct cfs_rq *cfs_rq, struct task_struct *p) {} 4028 4029 static inline void 4030 util_est_dequeue(struct cfs_rq *cfs_rq, struct task_struct *p, 4031 bool task_sleep) {} 4032 4033 #endif /* CONFIG_SMP */ 4034 4035 static void check_spread(struct cfs_rq *cfs_rq, struct sched_entity *se) 4036 { 4037 #ifdef CONFIG_SCHED_DEBUG 4038 s64 d = se->vruntime - cfs_rq->min_vruntime; 4039 4040 if (d < 0) 4041 d = -d; 4042 4043 if (d > 3*sysctl_sched_latency) 4044 schedstat_inc(cfs_rq->nr_spread_over); 4045 #endif 4046 } 4047 4048 static void 4049 place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) 4050 { 4051 u64 vruntime = cfs_rq->min_vruntime; 4052 4053 /* 4054 * The 'current' period is already promised to the current tasks, 4055 * however the extra weight of the new task will slow them down a 4056 * little, place the new task so that it fits in the slot that 4057 * stays open at the end. 4058 */ 4059 if (initial && sched_feat(START_DEBIT)) 4060 vruntime += sched_vslice(cfs_rq, se); 4061 4062 /* sleeps up to a single latency don't count. */ 4063 if (!initial) { 4064 unsigned long thresh = sysctl_sched_latency; 4065 4066 /* 4067 * Halve their sleep time's effect, to allow 4068 * for a gentler effect of sleepers: 4069 */ 4070 if (sched_feat(GENTLE_FAIR_SLEEPERS)) 4071 thresh >>= 1; 4072 4073 vruntime -= thresh; 4074 } 4075 4076 /* ensure we never gain time by being placed backwards. */ 4077 se->vruntime = max_vruntime(se->vruntime, vruntime); 4078 } 4079 4080 static void check_enqueue_throttle(struct cfs_rq *cfs_rq); 4081 4082 static inline void check_schedstat_required(void) 4083 { 4084 #ifdef CONFIG_SCHEDSTATS 4085 if (schedstat_enabled()) 4086 return; 4087 4088 /* Force schedstat enabled if a dependent tracepoint is active */ 4089 if (trace_sched_stat_wait_enabled() || 4090 trace_sched_stat_sleep_enabled() || 4091 trace_sched_stat_iowait_enabled() || 4092 trace_sched_stat_blocked_enabled() || 4093 trace_sched_stat_runtime_enabled()) { 4094 printk_deferred_once("Scheduler tracepoints stat_sleep, stat_iowait, " 4095 "stat_blocked and stat_runtime require the " 4096 "kernel parameter schedstats=enable or " 4097 "kernel.sched_schedstats=1\n"); 4098 } 4099 #endif 4100 } 4101 4102 4103 /* 4104 * MIGRATION 4105 * 4106 * dequeue 4107 * update_curr() 4108 * update_min_vruntime() 4109 * vruntime -= min_vruntime 4110 * 4111 * enqueue 4112 * update_curr() 4113 * update_min_vruntime() 4114 * vruntime += min_vruntime 4115 * 4116 * this way the vruntime transition between RQs is done when both 4117 * min_vruntime are up-to-date. 4118 * 4119 * WAKEUP (remote) 4120 * 4121 * ->migrate_task_rq_fair() (p->state == TASK_WAKING) 4122 * vruntime -= min_vruntime 4123 * 4124 * enqueue 4125 * update_curr() 4126 * update_min_vruntime() 4127 * vruntime += min_vruntime 4128 * 4129 * this way we don't have the most up-to-date min_vruntime on the originating 4130 * CPU and an up-to-date min_vruntime on the destination CPU. 4131 */ 4132 4133 static void 4134 enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 4135 { 4136 bool renorm = !(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_MIGRATED); 4137 bool curr = cfs_rq->curr == se; 4138 4139 /* 4140 * If we're the current task, we must renormalise before calling 4141 * update_curr(). 4142 */ 4143 if (renorm && curr) 4144 se->vruntime += cfs_rq->min_vruntime; 4145 4146 update_curr(cfs_rq); 4147 4148 /* 4149 * Otherwise, renormalise after, such that we're placed at the current 4150 * moment in time, instead of some random moment in the past. Being 4151 * placed in the past could significantly boost this task to the 4152 * fairness detriment of existing tasks. 4153 */ 4154 if (renorm && !curr) 4155 se->vruntime += cfs_rq->min_vruntime; 4156 4157 /* 4158 * When enqueuing a sched_entity, we must: 4159 * - Update loads to have both entity and cfs_rq synced with now. 4160 * - Add its load to cfs_rq->runnable_avg 4161 * - For group_entity, update its weight to reflect the new share of 4162 * its group cfs_rq 4163 * - Add its new weight to cfs_rq->load.weight 4164 */ 4165 update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH); 4166 update_cfs_group(se); 4167 enqueue_runnable_load_avg(cfs_rq, se); 4168 account_entity_enqueue(cfs_rq, se); 4169 4170 if (flags & ENQUEUE_WAKEUP) 4171 place_entity(cfs_rq, se, 0); 4172 4173 check_schedstat_required(); 4174 update_stats_enqueue(cfs_rq, se, flags); 4175 check_spread(cfs_rq, se); 4176 if (!curr) 4177 __enqueue_entity(cfs_rq, se); 4178 se->on_rq = 1; 4179 4180 if (cfs_rq->nr_running == 1) { 4181 list_add_leaf_cfs_rq(cfs_rq); 4182 check_enqueue_throttle(cfs_rq); 4183 } 4184 } 4185 4186 static void __clear_buddies_last(struct sched_entity *se) 4187 { 4188 for_each_sched_entity(se) { 4189 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4190 if (cfs_rq->last != se) 4191 break; 4192 4193 cfs_rq->last = NULL; 4194 } 4195 } 4196 4197 static void __clear_buddies_next(struct sched_entity *se) 4198 { 4199 for_each_sched_entity(se) { 4200 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4201 if (cfs_rq->next != se) 4202 break; 4203 4204 cfs_rq->next = NULL; 4205 } 4206 } 4207 4208 static void __clear_buddies_skip(struct sched_entity *se) 4209 { 4210 for_each_sched_entity(se) { 4211 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4212 if (cfs_rq->skip != se) 4213 break; 4214 4215 cfs_rq->skip = NULL; 4216 } 4217 } 4218 4219 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) 4220 { 4221 if (cfs_rq->last == se) 4222 __clear_buddies_last(se); 4223 4224 if (cfs_rq->next == se) 4225 __clear_buddies_next(se); 4226 4227 if (cfs_rq->skip == se) 4228 __clear_buddies_skip(se); 4229 } 4230 4231 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); 4232 4233 static void 4234 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 4235 { 4236 /* 4237 * Update run-time statistics of the 'current'. 4238 */ 4239 update_curr(cfs_rq); 4240 4241 /* 4242 * When dequeuing a sched_entity, we must: 4243 * - Update loads to have both entity and cfs_rq synced with now. 4244 * - Substract its load from the cfs_rq->runnable_avg. 4245 * - Substract its previous weight from cfs_rq->load.weight. 4246 * - For group entity, update its weight to reflect the new share 4247 * of its group cfs_rq. 4248 */ 4249 update_load_avg(cfs_rq, se, UPDATE_TG); 4250 dequeue_runnable_load_avg(cfs_rq, se); 4251 4252 update_stats_dequeue(cfs_rq, se, flags); 4253 4254 clear_buddies(cfs_rq, se); 4255 4256 if (se != cfs_rq->curr) 4257 __dequeue_entity(cfs_rq, se); 4258 se->on_rq = 0; 4259 account_entity_dequeue(cfs_rq, se); 4260 4261 /* 4262 * Normalize after update_curr(); which will also have moved 4263 * min_vruntime if @se is the one holding it back. But before doing 4264 * update_min_vruntime() again, which will discount @se's position and 4265 * can move min_vruntime forward still more. 4266 */ 4267 if (!(flags & DEQUEUE_SLEEP)) 4268 se->vruntime -= cfs_rq->min_vruntime; 4269 4270 /* return excess runtime on last dequeue */ 4271 return_cfs_rq_runtime(cfs_rq); 4272 4273 update_cfs_group(se); 4274 4275 /* 4276 * Now advance min_vruntime if @se was the entity holding it back, 4277 * except when: DEQUEUE_SAVE && !DEQUEUE_MOVE, in this case we'll be 4278 * put back on, and if we advance min_vruntime, we'll be placed back 4279 * further than we started -- ie. we'll be penalized. 4280 */ 4281 if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) == DEQUEUE_SAVE) 4282 update_min_vruntime(cfs_rq); 4283 } 4284 4285 /* 4286 * Preempt the current task with a newly woken task if needed: 4287 */ 4288 static void 4289 check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr) 4290 { 4291 unsigned long ideal_runtime, delta_exec; 4292 struct sched_entity *se; 4293 s64 delta; 4294 4295 ideal_runtime = sched_slice(cfs_rq, curr); 4296 delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; 4297 if (delta_exec > ideal_runtime) { 4298 resched_curr(rq_of(cfs_rq)); 4299 /* 4300 * The current task ran long enough, ensure it doesn't get 4301 * re-elected due to buddy favours. 4302 */ 4303 clear_buddies(cfs_rq, curr); 4304 return; 4305 } 4306 4307 /* 4308 * Ensure that a task that missed wakeup preemption by a 4309 * narrow margin doesn't have to wait for a full slice. 4310 * This also mitigates buddy induced latencies under load. 4311 */ 4312 if (delta_exec < sysctl_sched_min_granularity) 4313 return; 4314 4315 se = __pick_first_entity(cfs_rq); 4316 delta = curr->vruntime - se->vruntime; 4317 4318 if (delta < 0) 4319 return; 4320 4321 if (delta > ideal_runtime) 4322 resched_curr(rq_of(cfs_rq)); 4323 } 4324 4325 static void 4326 set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 4327 { 4328 /* 'current' is not kept within the tree. */ 4329 if (se->on_rq) { 4330 /* 4331 * Any task has to be enqueued before it get to execute on 4332 * a CPU. So account for the time it spent waiting on the 4333 * runqueue. 4334 */ 4335 update_stats_wait_end(cfs_rq, se); 4336 __dequeue_entity(cfs_rq, se); 4337 update_load_avg(cfs_rq, se, UPDATE_TG); 4338 } 4339 4340 update_stats_curr_start(cfs_rq, se); 4341 cfs_rq->curr = se; 4342 4343 /* 4344 * Track our maximum slice length, if the CPU's load is at 4345 * least twice that of our own weight (i.e. dont track it 4346 * when there are only lesser-weight tasks around): 4347 */ 4348 if (schedstat_enabled() && rq_of(cfs_rq)->load.weight >= 2*se->load.weight) { 4349 schedstat_set(se->statistics.slice_max, 4350 max((u64)schedstat_val(se->statistics.slice_max), 4351 se->sum_exec_runtime - se->prev_sum_exec_runtime)); 4352 } 4353 4354 se->prev_sum_exec_runtime = se->sum_exec_runtime; 4355 } 4356 4357 static int 4358 wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se); 4359 4360 /* 4361 * Pick the next process, keeping these things in mind, in this order: 4362 * 1) keep things fair between processes/task groups 4363 * 2) pick the "next" process, since someone really wants that to run 4364 * 3) pick the "last" process, for cache locality 4365 * 4) do not run the "skip" process, if something else is available 4366 */ 4367 static struct sched_entity * 4368 pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr) 4369 { 4370 struct sched_entity *left = __pick_first_entity(cfs_rq); 4371 struct sched_entity *se; 4372 4373 /* 4374 * If curr is set we have to see if its left of the leftmost entity 4375 * still in the tree, provided there was anything in the tree at all. 4376 */ 4377 if (!left || (curr && entity_before(curr, left))) 4378 left = curr; 4379 4380 se = left; /* ideally we run the leftmost entity */ 4381 4382 /* 4383 * Avoid running the skip buddy, if running something else can 4384 * be done without getting too unfair. 4385 */ 4386 if (cfs_rq->skip == se) { 4387 struct sched_entity *second; 4388 4389 if (se == curr) { 4390 second = __pick_first_entity(cfs_rq); 4391 } else { 4392 second = __pick_next_entity(se); 4393 if (!second || (curr && entity_before(curr, second))) 4394 second = curr; 4395 } 4396 4397 if (second && wakeup_preempt_entity(second, left) < 1) 4398 se = second; 4399 } 4400 4401 /* 4402 * Prefer last buddy, try to return the CPU to a preempted task. 4403 */ 4404 if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1) 4405 se = cfs_rq->last; 4406 4407 /* 4408 * Someone really wants this to run. If it's not unfair, run it. 4409 */ 4410 if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1) 4411 se = cfs_rq->next; 4412 4413 clear_buddies(cfs_rq, se); 4414 4415 return se; 4416 } 4417 4418 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq); 4419 4420 static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) 4421 { 4422 /* 4423 * If still on the runqueue then deactivate_task() 4424 * was not called and update_curr() has to be done: 4425 */ 4426 if (prev->on_rq) 4427 update_curr(cfs_rq); 4428 4429 /* throttle cfs_rqs exceeding runtime */ 4430 check_cfs_rq_runtime(cfs_rq); 4431 4432 check_spread(cfs_rq, prev); 4433 4434 if (prev->on_rq) { 4435 update_stats_wait_start(cfs_rq, prev); 4436 /* Put 'current' back into the tree. */ 4437 __enqueue_entity(cfs_rq, prev); 4438 /* in !on_rq case, update occurred at dequeue */ 4439 update_load_avg(cfs_rq, prev, 0); 4440 } 4441 cfs_rq->curr = NULL; 4442 } 4443 4444 static void 4445 entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) 4446 { 4447 /* 4448 * Update run-time statistics of the 'current'. 4449 */ 4450 update_curr(cfs_rq); 4451 4452 /* 4453 * Ensure that runnable average is periodically updated. 4454 */ 4455 update_load_avg(cfs_rq, curr, UPDATE_TG); 4456 update_cfs_group(curr); 4457 4458 #ifdef CONFIG_SCHED_HRTICK 4459 /* 4460 * queued ticks are scheduled to match the slice, so don't bother 4461 * validating it and just reschedule. 4462 */ 4463 if (queued) { 4464 resched_curr(rq_of(cfs_rq)); 4465 return; 4466 } 4467 /* 4468 * don't let the period tick interfere with the hrtick preemption 4469 */ 4470 if (!sched_feat(DOUBLE_TICK) && 4471 hrtimer_active(&rq_of(cfs_rq)->hrtick_timer)) 4472 return; 4473 #endif 4474 4475 if (cfs_rq->nr_running > 1) 4476 check_preempt_tick(cfs_rq, curr); 4477 } 4478 4479 4480 /************************************************** 4481 * CFS bandwidth control machinery 4482 */ 4483 4484 #ifdef CONFIG_CFS_BANDWIDTH 4485 4486 #ifdef HAVE_JUMP_LABEL 4487 static struct static_key __cfs_bandwidth_used; 4488 4489 static inline bool cfs_bandwidth_used(void) 4490 { 4491 return static_key_false(&__cfs_bandwidth_used); 4492 } 4493 4494 void cfs_bandwidth_usage_inc(void) 4495 { 4496 static_key_slow_inc_cpuslocked(&__cfs_bandwidth_used); 4497 } 4498 4499 void cfs_bandwidth_usage_dec(void) 4500 { 4501 static_key_slow_dec_cpuslocked(&__cfs_bandwidth_used); 4502 } 4503 #else /* HAVE_JUMP_LABEL */ 4504 static bool cfs_bandwidth_used(void) 4505 { 4506 return true; 4507 } 4508 4509 void cfs_bandwidth_usage_inc(void) {} 4510 void cfs_bandwidth_usage_dec(void) {} 4511 #endif /* HAVE_JUMP_LABEL */ 4512 4513 /* 4514 * default period for cfs group bandwidth. 4515 * default: 0.1s, units: nanoseconds 4516 */ 4517 static inline u64 default_cfs_period(void) 4518 { 4519 return 100000000ULL; 4520 } 4521 4522 static inline u64 sched_cfs_bandwidth_slice(void) 4523 { 4524 return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC; 4525 } 4526 4527 /* 4528 * Replenish runtime according to assigned quota and update expiration time. 4529 * We use sched_clock_cpu directly instead of rq->clock to avoid adding 4530 * additional synchronization around rq->lock. 4531 * 4532 * requires cfs_b->lock 4533 */ 4534 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b) 4535 { 4536 u64 now; 4537 4538 if (cfs_b->quota == RUNTIME_INF) 4539 return; 4540 4541 now = sched_clock_cpu(smp_processor_id()); 4542 cfs_b->runtime = cfs_b->quota; 4543 cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period); 4544 cfs_b->expires_seq++; 4545 } 4546 4547 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 4548 { 4549 return &tg->cfs_bandwidth; 4550 } 4551 4552 /* rq->task_clock normalized against any time this cfs_rq has spent throttled */ 4553 static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq) 4554 { 4555 if (unlikely(cfs_rq->throttle_count)) 4556 return cfs_rq->throttled_clock_task - cfs_rq->throttled_clock_task_time; 4557 4558 return rq_clock_task(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time; 4559 } 4560 4561 /* returns 0 on failure to allocate runtime */ 4562 static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) 4563 { 4564 struct task_group *tg = cfs_rq->tg; 4565 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg); 4566 u64 amount = 0, min_amount, expires; 4567 int expires_seq; 4568 4569 /* note: this is a positive sum as runtime_remaining <= 0 */ 4570 min_amount = sched_cfs_bandwidth_slice() - cfs_rq->runtime_remaining; 4571 4572 raw_spin_lock(&cfs_b->lock); 4573 if (cfs_b->quota == RUNTIME_INF) 4574 amount = min_amount; 4575 else { 4576 start_cfs_bandwidth(cfs_b); 4577 4578 if (cfs_b->runtime > 0) { 4579 amount = min(cfs_b->runtime, min_amount); 4580 cfs_b->runtime -= amount; 4581 cfs_b->idle = 0; 4582 } 4583 } 4584 expires_seq = cfs_b->expires_seq; 4585 expires = cfs_b->runtime_expires; 4586 raw_spin_unlock(&cfs_b->lock); 4587 4588 cfs_rq->runtime_remaining += amount; 4589 /* 4590 * we may have advanced our local expiration to account for allowed 4591 * spread between our sched_clock and the one on which runtime was 4592 * issued. 4593 */ 4594 if (cfs_rq->expires_seq != expires_seq) { 4595 cfs_rq->expires_seq = expires_seq; 4596 cfs_rq->runtime_expires = expires; 4597 } 4598 4599 return cfs_rq->runtime_remaining > 0; 4600 } 4601 4602 /* 4603 * Note: This depends on the synchronization provided by sched_clock and the 4604 * fact that rq->clock snapshots this value. 4605 */ 4606 static void expire_cfs_rq_runtime(struct cfs_rq *cfs_rq) 4607 { 4608 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 4609 4610 /* if the deadline is ahead of our clock, nothing to do */ 4611 if (likely((s64)(rq_clock(rq_of(cfs_rq)) - cfs_rq->runtime_expires) < 0)) 4612 return; 4613 4614 if (cfs_rq->runtime_remaining < 0) 4615 return; 4616 4617 /* 4618 * If the local deadline has passed we have to consider the 4619 * possibility that our sched_clock is 'fast' and the global deadline 4620 * has not truly expired. 4621 * 4622 * Fortunately we can check determine whether this the case by checking 4623 * whether the global deadline(cfs_b->expires_seq) has advanced. 4624 */ 4625 if (cfs_rq->expires_seq == cfs_b->expires_seq) { 4626 /* extend local deadline, drift is bounded above by 2 ticks */ 4627 cfs_rq->runtime_expires += TICK_NSEC; 4628 } else { 4629 /* global deadline is ahead, expiration has passed */ 4630 cfs_rq->runtime_remaining = 0; 4631 } 4632 } 4633 4634 static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 4635 { 4636 /* dock delta_exec before expiring quota (as it could span periods) */ 4637 cfs_rq->runtime_remaining -= delta_exec; 4638 expire_cfs_rq_runtime(cfs_rq); 4639 4640 if (likely(cfs_rq->runtime_remaining > 0)) 4641 return; 4642 4643 /* 4644 * if we're unable to extend our runtime we resched so that the active 4645 * hierarchy can be throttled 4646 */ 4647 if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) 4648 resched_curr(rq_of(cfs_rq)); 4649 } 4650 4651 static __always_inline 4652 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 4653 { 4654 if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled) 4655 return; 4656 4657 __account_cfs_rq_runtime(cfs_rq, delta_exec); 4658 } 4659 4660 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 4661 { 4662 return cfs_bandwidth_used() && cfs_rq->throttled; 4663 } 4664 4665 /* check whether cfs_rq, or any parent, is throttled */ 4666 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 4667 { 4668 return cfs_bandwidth_used() && cfs_rq->throttle_count; 4669 } 4670 4671 /* 4672 * Ensure that neither of the group entities corresponding to src_cpu or 4673 * dest_cpu are members of a throttled hierarchy when performing group 4674 * load-balance operations. 4675 */ 4676 static inline int throttled_lb_pair(struct task_group *tg, 4677 int src_cpu, int dest_cpu) 4678 { 4679 struct cfs_rq *src_cfs_rq, *dest_cfs_rq; 4680 4681 src_cfs_rq = tg->cfs_rq[src_cpu]; 4682 dest_cfs_rq = tg->cfs_rq[dest_cpu]; 4683 4684 return throttled_hierarchy(src_cfs_rq) || 4685 throttled_hierarchy(dest_cfs_rq); 4686 } 4687 4688 /* updated child weight may affect parent so we have to do this bottom up */ 4689 static int tg_unthrottle_up(struct task_group *tg, void *data) 4690 { 4691 struct rq *rq = data; 4692 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 4693 4694 cfs_rq->throttle_count--; 4695 if (!cfs_rq->throttle_count) { 4696 /* adjust cfs_rq_clock_task() */ 4697 cfs_rq->throttled_clock_task_time += rq_clock_task(rq) - 4698 cfs_rq->throttled_clock_task; 4699 } 4700 4701 return 0; 4702 } 4703 4704 static int tg_throttle_down(struct task_group *tg, void *data) 4705 { 4706 struct rq *rq = data; 4707 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 4708 4709 /* group is entering throttled state, stop time */ 4710 if (!cfs_rq->throttle_count) 4711 cfs_rq->throttled_clock_task = rq_clock_task(rq); 4712 cfs_rq->throttle_count++; 4713 4714 return 0; 4715 } 4716 4717 static void throttle_cfs_rq(struct cfs_rq *cfs_rq) 4718 { 4719 struct rq *rq = rq_of(cfs_rq); 4720 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 4721 struct sched_entity *se; 4722 long task_delta, dequeue = 1; 4723 bool empty; 4724 4725 se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))]; 4726 4727 /* freeze hierarchy runnable averages while throttled */ 4728 rcu_read_lock(); 4729 walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq); 4730 rcu_read_unlock(); 4731 4732 task_delta = cfs_rq->h_nr_running; 4733 for_each_sched_entity(se) { 4734 struct cfs_rq *qcfs_rq = cfs_rq_of(se); 4735 /* throttled entity or throttle-on-deactivate */ 4736 if (!se->on_rq) 4737 break; 4738 4739 if (dequeue) 4740 dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP); 4741 qcfs_rq->h_nr_running -= task_delta; 4742 4743 if (qcfs_rq->load.weight) 4744 dequeue = 0; 4745 } 4746 4747 if (!se) 4748 sub_nr_running(rq, task_delta); 4749 4750 cfs_rq->throttled = 1; 4751 cfs_rq->throttled_clock = rq_clock(rq); 4752 raw_spin_lock(&cfs_b->lock); 4753 empty = list_empty(&cfs_b->throttled_cfs_rq); 4754 4755 /* 4756 * Add to the _head_ of the list, so that an already-started 4757 * distribute_cfs_runtime will not see us 4758 */ 4759 list_add_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq); 4760 4761 /* 4762 * If we're the first throttled task, make sure the bandwidth 4763 * timer is running. 4764 */ 4765 if (empty) 4766 start_cfs_bandwidth(cfs_b); 4767 4768 raw_spin_unlock(&cfs_b->lock); 4769 } 4770 4771 void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) 4772 { 4773 struct rq *rq = rq_of(cfs_rq); 4774 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 4775 struct sched_entity *se; 4776 int enqueue = 1; 4777 long task_delta; 4778 4779 se = cfs_rq->tg->se[cpu_of(rq)]; 4780 4781 cfs_rq->throttled = 0; 4782 4783 update_rq_clock(rq); 4784 4785 raw_spin_lock(&cfs_b->lock); 4786 cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock; 4787 list_del_rcu(&cfs_rq->throttled_list); 4788 raw_spin_unlock(&cfs_b->lock); 4789 4790 /* update hierarchical throttle state */ 4791 walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); 4792 4793 if (!cfs_rq->load.weight) 4794 return; 4795 4796 task_delta = cfs_rq->h_nr_running; 4797 for_each_sched_entity(se) { 4798 if (se->on_rq) 4799 enqueue = 0; 4800 4801 cfs_rq = cfs_rq_of(se); 4802 if (enqueue) 4803 enqueue_entity(cfs_rq, se, ENQUEUE_WAKEUP); 4804 cfs_rq->h_nr_running += task_delta; 4805 4806 if (cfs_rq_throttled(cfs_rq)) 4807 break; 4808 } 4809 4810 if (!se) 4811 add_nr_running(rq, task_delta); 4812 4813 /* Determine whether we need to wake up potentially idle CPU: */ 4814 if (rq->curr == rq->idle && rq->cfs.nr_running) 4815 resched_curr(rq); 4816 } 4817 4818 static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b, 4819 u64 remaining, u64 expires) 4820 { 4821 struct cfs_rq *cfs_rq; 4822 u64 runtime; 4823 u64 starting_runtime = remaining; 4824 4825 rcu_read_lock(); 4826 list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, 4827 throttled_list) { 4828 struct rq *rq = rq_of(cfs_rq); 4829 struct rq_flags rf; 4830 4831 rq_lock(rq, &rf); 4832 if (!cfs_rq_throttled(cfs_rq)) 4833 goto next; 4834 4835 runtime = -cfs_rq->runtime_remaining + 1; 4836 if (runtime > remaining) 4837 runtime = remaining; 4838 remaining -= runtime; 4839 4840 cfs_rq->runtime_remaining += runtime; 4841 cfs_rq->runtime_expires = expires; 4842 4843 /* we check whether we're throttled above */ 4844 if (cfs_rq->runtime_remaining > 0) 4845 unthrottle_cfs_rq(cfs_rq); 4846 4847 next: 4848 rq_unlock(rq, &rf); 4849 4850 if (!remaining) 4851 break; 4852 } 4853 rcu_read_unlock(); 4854 4855 return starting_runtime - remaining; 4856 } 4857 4858 /* 4859 * Responsible for refilling a task_group's bandwidth and unthrottling its 4860 * cfs_rqs as appropriate. If there has been no activity within the last 4861 * period the timer is deactivated until scheduling resumes; cfs_b->idle is 4862 * used to track this state. 4863 */ 4864 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun) 4865 { 4866 u64 runtime, runtime_expires; 4867 int throttled; 4868 4869 /* no need to continue the timer with no bandwidth constraint */ 4870 if (cfs_b->quota == RUNTIME_INF) 4871 goto out_deactivate; 4872 4873 throttled = !list_empty(&cfs_b->throttled_cfs_rq); 4874 cfs_b->nr_periods += overrun; 4875 4876 /* 4877 * idle depends on !throttled (for the case of a large deficit), and if 4878 * we're going inactive then everything else can be deferred 4879 */ 4880 if (cfs_b->idle && !throttled) 4881 goto out_deactivate; 4882 4883 __refill_cfs_bandwidth_runtime(cfs_b); 4884 4885 if (!throttled) { 4886 /* mark as potentially idle for the upcoming period */ 4887 cfs_b->idle = 1; 4888 return 0; 4889 } 4890 4891 /* account preceding periods in which throttling occurred */ 4892 cfs_b->nr_throttled += overrun; 4893 4894 runtime_expires = cfs_b->runtime_expires; 4895 4896 /* 4897 * This check is repeated as we are holding onto the new bandwidth while 4898 * we unthrottle. This can potentially race with an unthrottled group 4899 * trying to acquire new bandwidth from the global pool. This can result 4900 * in us over-using our runtime if it is all used during this loop, but 4901 * only by limited amounts in that extreme case. 4902 */ 4903 while (throttled && cfs_b->runtime > 0) { 4904 runtime = cfs_b->runtime; 4905 raw_spin_unlock(&cfs_b->lock); 4906 /* we can't nest cfs_b->lock while distributing bandwidth */ 4907 runtime = distribute_cfs_runtime(cfs_b, runtime, 4908 runtime_expires); 4909 raw_spin_lock(&cfs_b->lock); 4910 4911 throttled = !list_empty(&cfs_b->throttled_cfs_rq); 4912 4913 cfs_b->runtime -= min(runtime, cfs_b->runtime); 4914 } 4915 4916 /* 4917 * While we are ensured activity in the period following an 4918 * unthrottle, this also covers the case in which the new bandwidth is 4919 * insufficient to cover the existing bandwidth deficit. (Forcing the 4920 * timer to remain active while there are any throttled entities.) 4921 */ 4922 cfs_b->idle = 0; 4923 4924 return 0; 4925 4926 out_deactivate: 4927 return 1; 4928 } 4929 4930 /* a cfs_rq won't donate quota below this amount */ 4931 static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC; 4932 /* minimum remaining period time to redistribute slack quota */ 4933 static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC; 4934 /* how long we wait to gather additional slack before distributing */ 4935 static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC; 4936 4937 /* 4938 * Are we near the end of the current quota period? 4939 * 4940 * Requires cfs_b->lock for hrtimer_expires_remaining to be safe against the 4941 * hrtimer base being cleared by hrtimer_start. In the case of 4942 * migrate_hrtimers, base is never cleared, so we are fine. 4943 */ 4944 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire) 4945 { 4946 struct hrtimer *refresh_timer = &cfs_b->period_timer; 4947 u64 remaining; 4948 4949 /* if the call-back is running a quota refresh is already occurring */ 4950 if (hrtimer_callback_running(refresh_timer)) 4951 return 1; 4952 4953 /* is a quota refresh about to occur? */ 4954 remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer)); 4955 if (remaining < min_expire) 4956 return 1; 4957 4958 return 0; 4959 } 4960 4961 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b) 4962 { 4963 u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration; 4964 4965 /* if there's a quota refresh soon don't bother with slack */ 4966 if (runtime_refresh_within(cfs_b, min_left)) 4967 return; 4968 4969 hrtimer_start(&cfs_b->slack_timer, 4970 ns_to_ktime(cfs_bandwidth_slack_period), 4971 HRTIMER_MODE_REL); 4972 } 4973 4974 /* we know any runtime found here is valid as update_curr() precedes return */ 4975 static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 4976 { 4977 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 4978 s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime; 4979 4980 if (slack_runtime <= 0) 4981 return; 4982 4983 raw_spin_lock(&cfs_b->lock); 4984 if (cfs_b->quota != RUNTIME_INF && 4985 cfs_rq->runtime_expires == cfs_b->runtime_expires) { 4986 cfs_b->runtime += slack_runtime; 4987 4988 /* we are under rq->lock, defer unthrottling using a timer */ 4989 if (cfs_b->runtime > sched_cfs_bandwidth_slice() && 4990 !list_empty(&cfs_b->throttled_cfs_rq)) 4991 start_cfs_slack_bandwidth(cfs_b); 4992 } 4993 raw_spin_unlock(&cfs_b->lock); 4994 4995 /* even if it's not valid for return we don't want to try again */ 4996 cfs_rq->runtime_remaining -= slack_runtime; 4997 } 4998 4999 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 5000 { 5001 if (!cfs_bandwidth_used()) 5002 return; 5003 5004 if (!cfs_rq->runtime_enabled || cfs_rq->nr_running) 5005 return; 5006 5007 __return_cfs_rq_runtime(cfs_rq); 5008 } 5009 5010 /* 5011 * This is done with a timer (instead of inline with bandwidth return) since 5012 * it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs. 5013 */ 5014 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) 5015 { 5016 u64 runtime = 0, slice = sched_cfs_bandwidth_slice(); 5017 u64 expires; 5018 5019 /* confirm we're still not at a refresh boundary */ 5020 raw_spin_lock(&cfs_b->lock); 5021 if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) { 5022 raw_spin_unlock(&cfs_b->lock); 5023 return; 5024 } 5025 5026 if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) 5027 runtime = cfs_b->runtime; 5028 5029 expires = cfs_b->runtime_expires; 5030 raw_spin_unlock(&cfs_b->lock); 5031 5032 if (!runtime) 5033 return; 5034 5035 runtime = distribute_cfs_runtime(cfs_b, runtime, expires); 5036 5037 raw_spin_lock(&cfs_b->lock); 5038 if (expires == cfs_b->runtime_expires) 5039 cfs_b->runtime -= min(runtime, cfs_b->runtime); 5040 raw_spin_unlock(&cfs_b->lock); 5041 } 5042 5043 /* 5044 * When a group wakes up we want to make sure that its quota is not already 5045 * expired/exceeded, otherwise it may be allowed to steal additional ticks of 5046 * runtime as update_curr() throttling can not not trigger until it's on-rq. 5047 */ 5048 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) 5049 { 5050 if (!cfs_bandwidth_used()) 5051 return; 5052 5053 /* an active group must be handled by the update_curr()->put() path */ 5054 if (!cfs_rq->runtime_enabled || cfs_rq->curr) 5055 return; 5056 5057 /* ensure the group is not already throttled */ 5058 if (cfs_rq_throttled(cfs_rq)) 5059 return; 5060 5061 /* update runtime allocation */ 5062 account_cfs_rq_runtime(cfs_rq, 0); 5063 if (cfs_rq->runtime_remaining <= 0) 5064 throttle_cfs_rq(cfs_rq); 5065 } 5066 5067 static void sync_throttle(struct task_group *tg, int cpu) 5068 { 5069 struct cfs_rq *pcfs_rq, *cfs_rq; 5070 5071 if (!cfs_bandwidth_used()) 5072 return; 5073 5074 if (!tg->parent) 5075 return; 5076 5077 cfs_rq = tg->cfs_rq[cpu]; 5078 pcfs_rq = tg->parent->cfs_rq[cpu]; 5079 5080 cfs_rq->throttle_count = pcfs_rq->throttle_count; 5081 cfs_rq->throttled_clock_task = rq_clock_task(cpu_rq(cpu)); 5082 } 5083 5084 /* conditionally throttle active cfs_rq's from put_prev_entity() */ 5085 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) 5086 { 5087 if (!cfs_bandwidth_used()) 5088 return false; 5089 5090 if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0)) 5091 return false; 5092 5093 /* 5094 * it's possible for a throttled entity to be forced into a running 5095 * state (e.g. set_curr_task), in this case we're finished. 5096 */ 5097 if (cfs_rq_throttled(cfs_rq)) 5098 return true; 5099 5100 throttle_cfs_rq(cfs_rq); 5101 return true; 5102 } 5103 5104 static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer) 5105 { 5106 struct cfs_bandwidth *cfs_b = 5107 container_of(timer, struct cfs_bandwidth, slack_timer); 5108 5109 do_sched_cfs_slack_timer(cfs_b); 5110 5111 return HRTIMER_NORESTART; 5112 } 5113 5114 static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) 5115 { 5116 struct cfs_bandwidth *cfs_b = 5117 container_of(timer, struct cfs_bandwidth, period_timer); 5118 int overrun; 5119 int idle = 0; 5120 5121 raw_spin_lock(&cfs_b->lock); 5122 for (;;) { 5123 overrun = hrtimer_forward_now(timer, cfs_b->period); 5124 if (!overrun) 5125 break; 5126 5127 idle = do_sched_cfs_period_timer(cfs_b, overrun); 5128 } 5129 if (idle) 5130 cfs_b->period_active = 0; 5131 raw_spin_unlock(&cfs_b->lock); 5132 5133 return idle ? HRTIMER_NORESTART : HRTIMER_RESTART; 5134 } 5135 5136 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 5137 { 5138 raw_spin_lock_init(&cfs_b->lock); 5139 cfs_b->runtime = 0; 5140 cfs_b->quota = RUNTIME_INF; 5141 cfs_b->period = ns_to_ktime(default_cfs_period()); 5142 5143 INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq); 5144 hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED); 5145 cfs_b->period_timer.function = sched_cfs_period_timer; 5146 hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 5147 cfs_b->slack_timer.function = sched_cfs_slack_timer; 5148 } 5149 5150 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) 5151 { 5152 cfs_rq->runtime_enabled = 0; 5153 INIT_LIST_HEAD(&cfs_rq->throttled_list); 5154 } 5155 5156 void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 5157 { 5158 lockdep_assert_held(&cfs_b->lock); 5159 5160 if (!cfs_b->period_active) { 5161 cfs_b->period_active = 1; 5162 hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period); 5163 hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED); 5164 } 5165 } 5166 5167 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 5168 { 5169 /* init_cfs_bandwidth() was not called */ 5170 if (!cfs_b->throttled_cfs_rq.next) 5171 return; 5172 5173 hrtimer_cancel(&cfs_b->period_timer); 5174 hrtimer_cancel(&cfs_b->slack_timer); 5175 } 5176 5177 /* 5178 * Both these CPU hotplug callbacks race against unregister_fair_sched_group() 5179 * 5180 * The race is harmless, since modifying bandwidth settings of unhooked group 5181 * bits doesn't do much. 5182 */ 5183 5184 /* cpu online calback */ 5185 static void __maybe_unused update_runtime_enabled(struct rq *rq) 5186 { 5187 struct task_group *tg; 5188 5189 lockdep_assert_held(&rq->lock); 5190 5191 rcu_read_lock(); 5192 list_for_each_entry_rcu(tg, &task_groups, list) { 5193 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 5194 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 5195 5196 raw_spin_lock(&cfs_b->lock); 5197 cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; 5198 raw_spin_unlock(&cfs_b->lock); 5199 } 5200 rcu_read_unlock(); 5201 } 5202 5203 /* cpu offline callback */ 5204 static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) 5205 { 5206 struct task_group *tg; 5207 5208 lockdep_assert_held(&rq->lock); 5209 5210 rcu_read_lock(); 5211 list_for_each_entry_rcu(tg, &task_groups, list) { 5212 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 5213 5214 if (!cfs_rq->runtime_enabled) 5215 continue; 5216 5217 /* 5218 * clock_task is not advancing so we just need to make sure 5219 * there's some valid quota amount 5220 */ 5221 cfs_rq->runtime_remaining = 1; 5222 /* 5223 * Offline rq is schedulable till CPU is completely disabled 5224 * in take_cpu_down(), so we prevent new cfs throttling here. 5225 */ 5226 cfs_rq->runtime_enabled = 0; 5227 5228 if (cfs_rq_throttled(cfs_rq)) 5229 unthrottle_cfs_rq(cfs_rq); 5230 } 5231 rcu_read_unlock(); 5232 } 5233 5234 #else /* CONFIG_CFS_BANDWIDTH */ 5235 static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq) 5236 { 5237 return rq_clock_task(rq_of(cfs_rq)); 5238 } 5239 5240 static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {} 5241 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; } 5242 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {} 5243 static inline void sync_throttle(struct task_group *tg, int cpu) {} 5244 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 5245 5246 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 5247 { 5248 return 0; 5249 } 5250 5251 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 5252 { 5253 return 0; 5254 } 5255 5256 static inline int throttled_lb_pair(struct task_group *tg, 5257 int src_cpu, int dest_cpu) 5258 { 5259 return 0; 5260 } 5261 5262 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} 5263 5264 #ifdef CONFIG_FAIR_GROUP_SCHED 5265 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 5266 #endif 5267 5268 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 5269 { 5270 return NULL; 5271 } 5272 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} 5273 static inline void update_runtime_enabled(struct rq *rq) {} 5274 static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {} 5275 5276 #endif /* CONFIG_CFS_BANDWIDTH */ 5277 5278 /************************************************** 5279 * CFS operations on tasks: 5280 */ 5281 5282 #ifdef CONFIG_SCHED_HRTICK 5283 static void hrtick_start_fair(struct rq *rq, struct task_struct *p) 5284 { 5285 struct sched_entity *se = &p->se; 5286 struct cfs_rq *cfs_rq = cfs_rq_of(se); 5287 5288 SCHED_WARN_ON(task_rq(p) != rq); 5289 5290 if (rq->cfs.h_nr_running > 1) { 5291 u64 slice = sched_slice(cfs_rq, se); 5292 u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime; 5293 s64 delta = slice - ran; 5294 5295 if (delta < 0) { 5296 if (rq->curr == p) 5297 resched_curr(rq); 5298 return; 5299 } 5300 hrtick_start(rq, delta); 5301 } 5302 } 5303 5304 /* 5305 * called from enqueue/dequeue and updates the hrtick when the 5306 * current task is from our class and nr_running is low enough 5307 * to matter. 5308 */ 5309 static void hrtick_update(struct rq *rq) 5310 { 5311 struct task_struct *curr = rq->curr; 5312 5313 if (!hrtick_enabled(rq) || curr->sched_class != &fair_sched_class) 5314 return; 5315 5316 if (cfs_rq_of(&curr->se)->nr_running < sched_nr_latency) 5317 hrtick_start_fair(rq, curr); 5318 } 5319 #else /* !CONFIG_SCHED_HRTICK */ 5320 static inline void 5321 hrtick_start_fair(struct rq *rq, struct task_struct *p) 5322 { 5323 } 5324 5325 static inline void hrtick_update(struct rq *rq) 5326 { 5327 } 5328 #endif 5329 5330 /* 5331 * The enqueue_task method is called before nr_running is 5332 * increased. Here we update the fair scheduling stats and 5333 * then put the task into the rbtree: 5334 */ 5335 static void 5336 enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) 5337 { 5338 struct cfs_rq *cfs_rq; 5339 struct sched_entity *se = &p->se; 5340 5341 /* 5342 * If in_iowait is set, the code below may not trigger any cpufreq 5343 * utilization updates, so do it here explicitly with the IOWAIT flag 5344 * passed. 5345 */ 5346 if (p->in_iowait) 5347 cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT); 5348 5349 for_each_sched_entity(se) { 5350 if (se->on_rq) 5351 break; 5352 cfs_rq = cfs_rq_of(se); 5353 enqueue_entity(cfs_rq, se, flags); 5354 5355 /* 5356 * end evaluation on encountering a throttled cfs_rq 5357 * 5358 * note: in the case of encountering a throttled cfs_rq we will 5359 * post the final h_nr_running increment below. 5360 */ 5361 if (cfs_rq_throttled(cfs_rq)) 5362 break; 5363 cfs_rq->h_nr_running++; 5364 5365 flags = ENQUEUE_WAKEUP; 5366 } 5367 5368 for_each_sched_entity(se) { 5369 cfs_rq = cfs_rq_of(se); 5370 cfs_rq->h_nr_running++; 5371 5372 if (cfs_rq_throttled(cfs_rq)) 5373 break; 5374 5375 update_load_avg(cfs_rq, se, UPDATE_TG); 5376 update_cfs_group(se); 5377 } 5378 5379 if (!se) 5380 add_nr_running(rq, 1); 5381 5382 util_est_enqueue(&rq->cfs, p); 5383 hrtick_update(rq); 5384 } 5385 5386 static void set_next_buddy(struct sched_entity *se); 5387 5388 /* 5389 * The dequeue_task method is called before nr_running is 5390 * decreased. We remove the task from the rbtree and 5391 * update the fair scheduling stats: 5392 */ 5393 static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) 5394 { 5395 struct cfs_rq *cfs_rq; 5396 struct sched_entity *se = &p->se; 5397 int task_sleep = flags & DEQUEUE_SLEEP; 5398 5399 for_each_sched_entity(se) { 5400 cfs_rq = cfs_rq_of(se); 5401 dequeue_entity(cfs_rq, se, flags); 5402 5403 /* 5404 * end evaluation on encountering a throttled cfs_rq 5405 * 5406 * note: in the case of encountering a throttled cfs_rq we will 5407 * post the final h_nr_running decrement below. 5408 */ 5409 if (cfs_rq_throttled(cfs_rq)) 5410 break; 5411 cfs_rq->h_nr_running--; 5412 5413 /* Don't dequeue parent if it has other entities besides us */ 5414 if (cfs_rq->load.weight) { 5415 /* Avoid re-evaluating load for this entity: */ 5416 se = parent_entity(se); 5417 /* 5418 * Bias pick_next to pick a task from this cfs_rq, as 5419 * p is sleeping when it is within its sched_slice. 5420 */ 5421 if (task_sleep && se && !throttled_hierarchy(cfs_rq)) 5422 set_next_buddy(se); 5423 break; 5424 } 5425 flags |= DEQUEUE_SLEEP; 5426 } 5427 5428 for_each_sched_entity(se) { 5429 cfs_rq = cfs_rq_of(se); 5430 cfs_rq->h_nr_running--; 5431 5432 if (cfs_rq_throttled(cfs_rq)) 5433 break; 5434 5435 update_load_avg(cfs_rq, se, UPDATE_TG); 5436 update_cfs_group(se); 5437 } 5438 5439 if (!se) 5440 sub_nr_running(rq, 1); 5441 5442 util_est_dequeue(&rq->cfs, p, task_sleep); 5443 hrtick_update(rq); 5444 } 5445 5446 #ifdef CONFIG_SMP 5447 5448 /* Working cpumask for: load_balance, load_balance_newidle. */ 5449 DEFINE_PER_CPU(cpumask_var_t, load_balance_mask); 5450 DEFINE_PER_CPU(cpumask_var_t, select_idle_mask); 5451 5452 #ifdef CONFIG_NO_HZ_COMMON 5453 /* 5454 * per rq 'load' arrray crap; XXX kill this. 5455 */ 5456 5457 /* 5458 * The exact cpuload calculated at every tick would be: 5459 * 5460 * load' = (1 - 1/2^i) * load + (1/2^i) * cur_load 5461 * 5462 * If a CPU misses updates for n ticks (as it was idle) and update gets 5463 * called on the n+1-th tick when CPU may be busy, then we have: 5464 * 5465 * load_n = (1 - 1/2^i)^n * load_0 5466 * load_n+1 = (1 - 1/2^i) * load_n + (1/2^i) * cur_load 5467 * 5468 * decay_load_missed() below does efficient calculation of 5469 * 5470 * load' = (1 - 1/2^i)^n * load 5471 * 5472 * Because x^(n+m) := x^n * x^m we can decompose any x^n in power-of-2 factors. 5473 * This allows us to precompute the above in said factors, thereby allowing the 5474 * reduction of an arbitrary n in O(log_2 n) steps. (See also 5475 * fixed_power_int()) 5476 * 5477 * The calculation is approximated on a 128 point scale. 5478 */ 5479 #define DEGRADE_SHIFT 7 5480 5481 static const u8 degrade_zero_ticks[CPU_LOAD_IDX_MAX] = {0, 8, 32, 64, 128}; 5482 static const u8 degrade_factor[CPU_LOAD_IDX_MAX][DEGRADE_SHIFT + 1] = { 5483 { 0, 0, 0, 0, 0, 0, 0, 0 }, 5484 { 64, 32, 8, 0, 0, 0, 0, 0 }, 5485 { 96, 72, 40, 12, 1, 0, 0, 0 }, 5486 { 112, 98, 75, 43, 15, 1, 0, 0 }, 5487 { 120, 112, 98, 76, 45, 16, 2, 0 } 5488 }; 5489 5490 /* 5491 * Update cpu_load for any missed ticks, due to tickless idle. The backlog 5492 * would be when CPU is idle and so we just decay the old load without 5493 * adding any new load. 5494 */ 5495 static unsigned long 5496 decay_load_missed(unsigned long load, unsigned long missed_updates, int idx) 5497 { 5498 int j = 0; 5499 5500 if (!missed_updates) 5501 return load; 5502 5503 if (missed_updates >= degrade_zero_ticks[idx]) 5504 return 0; 5505 5506 if (idx == 1) 5507 return load >> missed_updates; 5508 5509 while (missed_updates) { 5510 if (missed_updates % 2) 5511 load = (load * degrade_factor[idx][j]) >> DEGRADE_SHIFT; 5512 5513 missed_updates >>= 1; 5514 j++; 5515 } 5516 return load; 5517 } 5518 5519 static struct { 5520 cpumask_var_t idle_cpus_mask; 5521 atomic_t nr_cpus; 5522 int has_blocked; /* Idle CPUS has blocked load */ 5523 unsigned long next_balance; /* in jiffy units */ 5524 unsigned long next_blocked; /* Next update of blocked load in jiffies */ 5525 } nohz ____cacheline_aligned; 5526 5527 #endif /* CONFIG_NO_HZ_COMMON */ 5528 5529 /** 5530 * __cpu_load_update - update the rq->cpu_load[] statistics 5531 * @this_rq: The rq to update statistics for 5532 * @this_load: The current load 5533 * @pending_updates: The number of missed updates 5534 * 5535 * Update rq->cpu_load[] statistics. This function is usually called every 5536 * scheduler tick (TICK_NSEC). 5537 * 5538 * This function computes a decaying average: 5539 * 5540 * load[i]' = (1 - 1/2^i) * load[i] + (1/2^i) * load 5541 * 5542 * Because of NOHZ it might not get called on every tick which gives need for 5543 * the @pending_updates argument. 5544 * 5545 * load[i]_n = (1 - 1/2^i) * load[i]_n-1 + (1/2^i) * load_n-1 5546 * = A * load[i]_n-1 + B ; A := (1 - 1/2^i), B := (1/2^i) * load 5547 * = A * (A * load[i]_n-2 + B) + B 5548 * = A * (A * (A * load[i]_n-3 + B) + B) + B 5549 * = A^3 * load[i]_n-3 + (A^2 + A + 1) * B 5550 * = A^n * load[i]_0 + (A^(n-1) + A^(n-2) + ... + 1) * B 5551 * = A^n * load[i]_0 + ((1 - A^n) / (1 - A)) * B 5552 * = (1 - 1/2^i)^n * (load[i]_0 - load) + load 5553 * 5554 * In the above we've assumed load_n := load, which is true for NOHZ_FULL as 5555 * any change in load would have resulted in the tick being turned back on. 5556 * 5557 * For regular NOHZ, this reduces to: 5558 * 5559 * load[i]_n = (1 - 1/2^i)^n * load[i]_0 5560 * 5561 * see decay_load_misses(). For NOHZ_FULL we get to subtract and add the extra 5562 * term. 5563 */ 5564 static void cpu_load_update(struct rq *this_rq, unsigned long this_load, 5565 unsigned long pending_updates) 5566 { 5567 unsigned long __maybe_unused tickless_load = this_rq->cpu_load[0]; 5568 int i, scale; 5569 5570 this_rq->nr_load_updates++; 5571 5572 /* Update our load: */ 5573 this_rq->cpu_load[0] = this_load; /* Fasttrack for idx 0 */ 5574 for (i = 1, scale = 2; i < CPU_LOAD_IDX_MAX; i++, scale += scale) { 5575 unsigned long old_load, new_load; 5576 5577 /* scale is effectively 1 << i now, and >> i divides by scale */ 5578 5579 old_load = this_rq->cpu_load[i]; 5580 #ifdef CONFIG_NO_HZ_COMMON 5581 old_load = decay_load_missed(old_load, pending_updates - 1, i); 5582 if (tickless_load) { 5583 old_load -= decay_load_missed(tickless_load, pending_updates - 1, i); 5584 /* 5585 * old_load can never be a negative value because a 5586 * decayed tickless_load cannot be greater than the 5587 * original tickless_load. 5588 */ 5589 old_load += tickless_load; 5590 } 5591 #endif 5592 new_load = this_load; 5593 /* 5594 * Round up the averaging division if load is increasing. This 5595 * prevents us from getting stuck on 9 if the load is 10, for 5596 * example. 5597 */ 5598 if (new_load > old_load) 5599 new_load += scale - 1; 5600 5601 this_rq->cpu_load[i] = (old_load * (scale - 1) + new_load) >> i; 5602 } 5603 5604 sched_avg_update(this_rq); 5605 } 5606 5607 /* Used instead of source_load when we know the type == 0 */ 5608 static unsigned long weighted_cpuload(struct rq *rq) 5609 { 5610 return cfs_rq_runnable_load_avg(&rq->cfs); 5611 } 5612 5613 #ifdef CONFIG_NO_HZ_COMMON 5614 /* 5615 * There is no sane way to deal with nohz on smp when using jiffies because the 5616 * CPU doing the jiffies update might drift wrt the CPU doing the jiffy reading 5617 * causing off-by-one errors in observed deltas; {0,2} instead of {1,1}. 5618 * 5619 * Therefore we need to avoid the delta approach from the regular tick when 5620 * possible since that would seriously skew the load calculation. This is why we 5621 * use cpu_load_update_periodic() for CPUs out of nohz. However we'll rely on 5622 * jiffies deltas for updates happening while in nohz mode (idle ticks, idle 5623 * loop exit, nohz_idle_balance, nohz full exit...) 5624 * 5625 * This means we might still be one tick off for nohz periods. 5626 */ 5627 5628 static void cpu_load_update_nohz(struct rq *this_rq, 5629 unsigned long curr_jiffies, 5630 unsigned long load) 5631 { 5632 unsigned long pending_updates; 5633 5634 pending_updates = curr_jiffies - this_rq->last_load_update_tick; 5635 if (pending_updates) { 5636 this_rq->last_load_update_tick = curr_jiffies; 5637 /* 5638 * In the regular NOHZ case, we were idle, this means load 0. 5639 * In the NOHZ_FULL case, we were non-idle, we should consider 5640 * its weighted load. 5641 */ 5642 cpu_load_update(this_rq, load, pending_updates); 5643 } 5644 } 5645 5646 /* 5647 * Called from nohz_idle_balance() to update the load ratings before doing the 5648 * idle balance. 5649 */ 5650 static void cpu_load_update_idle(struct rq *this_rq) 5651 { 5652 /* 5653 * bail if there's load or we're actually up-to-date. 5654 */ 5655 if (weighted_cpuload(this_rq)) 5656 return; 5657 5658 cpu_load_update_nohz(this_rq, READ_ONCE(jiffies), 0); 5659 } 5660 5661 /* 5662 * Record CPU load on nohz entry so we know the tickless load to account 5663 * on nohz exit. cpu_load[0] happens then to be updated more frequently 5664 * than other cpu_load[idx] but it should be fine as cpu_load readers 5665 * shouldn't rely into synchronized cpu_load[*] updates. 5666 */ 5667 void cpu_load_update_nohz_start(void) 5668 { 5669 struct rq *this_rq = this_rq(); 5670 5671 /* 5672 * This is all lockless but should be fine. If weighted_cpuload changes 5673 * concurrently we'll exit nohz. And cpu_load write can race with 5674 * cpu_load_update_idle() but both updater would be writing the same. 5675 */ 5676 this_rq->cpu_load[0] = weighted_cpuload(this_rq); 5677 } 5678 5679 /* 5680 * Account the tickless load in the end of a nohz frame. 5681 */ 5682 void cpu_load_update_nohz_stop(void) 5683 { 5684 unsigned long curr_jiffies = READ_ONCE(jiffies); 5685 struct rq *this_rq = this_rq(); 5686 unsigned long load; 5687 struct rq_flags rf; 5688 5689 if (curr_jiffies == this_rq->last_load_update_tick) 5690 return; 5691 5692 load = weighted_cpuload(this_rq); 5693 rq_lock(this_rq, &rf); 5694 update_rq_clock(this_rq); 5695 cpu_load_update_nohz(this_rq, curr_jiffies, load); 5696 rq_unlock(this_rq, &rf); 5697 } 5698 #else /* !CONFIG_NO_HZ_COMMON */ 5699 static inline void cpu_load_update_nohz(struct rq *this_rq, 5700 unsigned long curr_jiffies, 5701 unsigned long load) { } 5702 #endif /* CONFIG_NO_HZ_COMMON */ 5703 5704 static void cpu_load_update_periodic(struct rq *this_rq, unsigned long load) 5705 { 5706 #ifdef CONFIG_NO_HZ_COMMON 5707 /* See the mess around cpu_load_update_nohz(). */ 5708 this_rq->last_load_update_tick = READ_ONCE(jiffies); 5709 #endif 5710 cpu_load_update(this_rq, load, 1); 5711 } 5712 5713 /* 5714 * Called from scheduler_tick() 5715 */ 5716 void cpu_load_update_active(struct rq *this_rq) 5717 { 5718 unsigned long load = weighted_cpuload(this_rq); 5719 5720 if (tick_nohz_tick_stopped()) 5721 cpu_load_update_nohz(this_rq, READ_ONCE(jiffies), load); 5722 else 5723 cpu_load_update_periodic(this_rq, load); 5724 } 5725 5726 /* 5727 * Return a low guess at the load of a migration-source CPU weighted 5728 * according to the scheduling class and "nice" value. 5729 * 5730 * We want to under-estimate the load of migration sources, to 5731 * balance conservatively. 5732 */ 5733 static unsigned long source_load(int cpu, int type) 5734 { 5735 struct rq *rq = cpu_rq(cpu); 5736 unsigned long total = weighted_cpuload(rq); 5737 5738 if (type == 0 || !sched_feat(LB_BIAS)) 5739 return total; 5740 5741 return min(rq->cpu_load[type-1], total); 5742 } 5743 5744 /* 5745 * Return a high guess at the load of a migration-target CPU weighted 5746 * according to the scheduling class and "nice" value. 5747 */ 5748 static unsigned long target_load(int cpu, int type) 5749 { 5750 struct rq *rq = cpu_rq(cpu); 5751 unsigned long total = weighted_cpuload(rq); 5752 5753 if (type == 0 || !sched_feat(LB_BIAS)) 5754 return total; 5755 5756 return max(rq->cpu_load[type-1], total); 5757 } 5758 5759 static unsigned long capacity_of(int cpu) 5760 { 5761 return cpu_rq(cpu)->cpu_capacity; 5762 } 5763 5764 static unsigned long capacity_orig_of(int cpu) 5765 { 5766 return cpu_rq(cpu)->cpu_capacity_orig; 5767 } 5768 5769 static unsigned long cpu_avg_load_per_task(int cpu) 5770 { 5771 struct rq *rq = cpu_rq(cpu); 5772 unsigned long nr_running = READ_ONCE(rq->cfs.h_nr_running); 5773 unsigned long load_avg = weighted_cpuload(rq); 5774 5775 if (nr_running) 5776 return load_avg / nr_running; 5777 5778 return 0; 5779 } 5780 5781 static void record_wakee(struct task_struct *p) 5782 { 5783 /* 5784 * Only decay a single time; tasks that have less then 1 wakeup per 5785 * jiffy will not have built up many flips. 5786 */ 5787 if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) { 5788 current->wakee_flips >>= 1; 5789 current->wakee_flip_decay_ts = jiffies; 5790 } 5791 5792 if (current->last_wakee != p) { 5793 current->last_wakee = p; 5794 current->wakee_flips++; 5795 } 5796 } 5797 5798 /* 5799 * Detect M:N waker/wakee relationships via a switching-frequency heuristic. 5800 * 5801 * A waker of many should wake a different task than the one last awakened 5802 * at a frequency roughly N times higher than one of its wakees. 5803 * 5804 * In order to determine whether we should let the load spread vs consolidating 5805 * to shared cache, we look for a minimum 'flip' frequency of llc_size in one 5806 * partner, and a factor of lls_size higher frequency in the other. 5807 * 5808 * With both conditions met, we can be relatively sure that the relationship is 5809 * non-monogamous, with partner count exceeding socket size. 5810 * 5811 * Waker/wakee being client/server, worker/dispatcher, interrupt source or 5812 * whatever is irrelevant, spread criteria is apparent partner count exceeds 5813 * socket size. 5814 */ 5815 static int wake_wide(struct task_struct *p) 5816 { 5817 unsigned int master = current->wakee_flips; 5818 unsigned int slave = p->wakee_flips; 5819 int factor = this_cpu_read(sd_llc_size); 5820 5821 if (master < slave) 5822 swap(master, slave); 5823 if (slave < factor || master < slave * factor) 5824 return 0; 5825 return 1; 5826 } 5827 5828 /* 5829 * The purpose of wake_affine() is to quickly determine on which CPU we can run 5830 * soonest. For the purpose of speed we only consider the waking and previous 5831 * CPU. 5832 * 5833 * wake_affine_idle() - only considers 'now', it check if the waking CPU is 5834 * cache-affine and is (or will be) idle. 5835 * 5836 * wake_affine_weight() - considers the weight to reflect the average 5837 * scheduling latency of the CPUs. This seems to work 5838 * for the overloaded case. 5839 */ 5840 static int 5841 wake_affine_idle(int this_cpu, int prev_cpu, int sync) 5842 { 5843 /* 5844 * If this_cpu is idle, it implies the wakeup is from interrupt 5845 * context. Only allow the move if cache is shared. Otherwise an 5846 * interrupt intensive workload could force all tasks onto one 5847 * node depending on the IO topology or IRQ affinity settings. 5848 * 5849 * If the prev_cpu is idle and cache affine then avoid a migration. 5850 * There is no guarantee that the cache hot data from an interrupt 5851 * is more important than cache hot data on the prev_cpu and from 5852 * a cpufreq perspective, it's better to have higher utilisation 5853 * on one CPU. 5854 */ 5855 if (idle_cpu(this_cpu) && cpus_share_cache(this_cpu, prev_cpu)) 5856 return idle_cpu(prev_cpu) ? prev_cpu : this_cpu; 5857 5858 if (sync && cpu_rq(this_cpu)->nr_running == 1) 5859 return this_cpu; 5860 5861 return nr_cpumask_bits; 5862 } 5863 5864 static int 5865 wake_affine_weight(struct sched_domain *sd, struct task_struct *p, 5866 int this_cpu, int prev_cpu, int sync) 5867 { 5868 s64 this_eff_load, prev_eff_load; 5869 unsigned long task_load; 5870 5871 this_eff_load = target_load(this_cpu, sd->wake_idx); 5872 5873 if (sync) { 5874 unsigned long current_load = task_h_load(current); 5875 5876 if (current_load > this_eff_load) 5877 return this_cpu; 5878 5879 this_eff_load -= current_load; 5880 } 5881 5882 task_load = task_h_load(p); 5883 5884 this_eff_load += task_load; 5885 if (sched_feat(WA_BIAS)) 5886 this_eff_load *= 100; 5887 this_eff_load *= capacity_of(prev_cpu); 5888 5889 prev_eff_load = source_load(prev_cpu, sd->wake_idx); 5890 prev_eff_load -= task_load; 5891 if (sched_feat(WA_BIAS)) 5892 prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2; 5893 prev_eff_load *= capacity_of(this_cpu); 5894 5895 /* 5896 * If sync, adjust the weight of prev_eff_load such that if 5897 * prev_eff == this_eff that select_idle_sibling() will consider 5898 * stacking the wakee on top of the waker if no other CPU is 5899 * idle. 5900 */ 5901 if (sync) 5902 prev_eff_load += 1; 5903 5904 return this_eff_load < prev_eff_load ? this_cpu : nr_cpumask_bits; 5905 } 5906 5907 static int wake_affine(struct sched_domain *sd, struct task_struct *p, 5908 int this_cpu, int prev_cpu, int sync) 5909 { 5910 int target = nr_cpumask_bits; 5911 5912 if (sched_feat(WA_IDLE)) 5913 target = wake_affine_idle(this_cpu, prev_cpu, sync); 5914 5915 if (sched_feat(WA_WEIGHT) && target == nr_cpumask_bits) 5916 target = wake_affine_weight(sd, p, this_cpu, prev_cpu, sync); 5917 5918 schedstat_inc(p->se.statistics.nr_wakeups_affine_attempts); 5919 if (target == nr_cpumask_bits) 5920 return prev_cpu; 5921 5922 schedstat_inc(sd->ttwu_move_affine); 5923 schedstat_inc(p->se.statistics.nr_wakeups_affine); 5924 return target; 5925 } 5926 5927 static unsigned long cpu_util_wake(int cpu, struct task_struct *p); 5928 5929 static unsigned long capacity_spare_wake(int cpu, struct task_struct *p) 5930 { 5931 return max_t(long, capacity_of(cpu) - cpu_util_wake(cpu, p), 0); 5932 } 5933 5934 /* 5935 * find_idlest_group finds and returns the least busy CPU group within the 5936 * domain. 5937 * 5938 * Assumes p is allowed on at least one CPU in sd. 5939 */ 5940 static struct sched_group * 5941 find_idlest_group(struct sched_domain *sd, struct task_struct *p, 5942 int this_cpu, int sd_flag) 5943 { 5944 struct sched_group *idlest = NULL, *group = sd->groups; 5945 struct sched_group *most_spare_sg = NULL; 5946 unsigned long min_runnable_load = ULONG_MAX; 5947 unsigned long this_runnable_load = ULONG_MAX; 5948 unsigned long min_avg_load = ULONG_MAX, this_avg_load = ULONG_MAX; 5949 unsigned long most_spare = 0, this_spare = 0; 5950 int load_idx = sd->forkexec_idx; 5951 int imbalance_scale = 100 + (sd->imbalance_pct-100)/2; 5952 unsigned long imbalance = scale_load_down(NICE_0_LOAD) * 5953 (sd->imbalance_pct-100) / 100; 5954 5955 if (sd_flag & SD_BALANCE_WAKE) 5956 load_idx = sd->wake_idx; 5957 5958 do { 5959 unsigned long load, avg_load, runnable_load; 5960 unsigned long spare_cap, max_spare_cap; 5961 int local_group; 5962 int i; 5963 5964 /* Skip over this group if it has no CPUs allowed */ 5965 if (!cpumask_intersects(sched_group_span(group), 5966 &p->cpus_allowed)) 5967 continue; 5968 5969 local_group = cpumask_test_cpu(this_cpu, 5970 sched_group_span(group)); 5971 5972 /* 5973 * Tally up the load of all CPUs in the group and find 5974 * the group containing the CPU with most spare capacity. 5975 */ 5976 avg_load = 0; 5977 runnable_load = 0; 5978 max_spare_cap = 0; 5979 5980 for_each_cpu(i, sched_group_span(group)) { 5981 /* Bias balancing toward CPUs of our domain */ 5982 if (local_group) 5983 load = source_load(i, load_idx); 5984 else 5985 load = target_load(i, load_idx); 5986 5987 runnable_load += load; 5988 5989 avg_load += cfs_rq_load_avg(&cpu_rq(i)->cfs); 5990 5991 spare_cap = capacity_spare_wake(i, p); 5992 5993 if (spare_cap > max_spare_cap) 5994 max_spare_cap = spare_cap; 5995 } 5996 5997 /* Adjust by relative CPU capacity of the group */ 5998 avg_load = (avg_load * SCHED_CAPACITY_SCALE) / 5999 group->sgc->capacity; 6000 runnable_load = (runnable_load * SCHED_CAPACITY_SCALE) / 6001 group->sgc->capacity; 6002 6003 if (local_group) { 6004 this_runnable_load = runnable_load; 6005 this_avg_load = avg_load; 6006 this_spare = max_spare_cap; 6007 } else { 6008 if (min_runnable_load > (runnable_load + imbalance)) { 6009 /* 6010 * The runnable load is significantly smaller 6011 * so we can pick this new CPU: 6012 */ 6013 min_runnable_load = runnable_load; 6014 min_avg_load = avg_load; 6015 idlest = group; 6016 } else if ((runnable_load < (min_runnable_load + imbalance)) && 6017 (100*min_avg_load > imbalance_scale*avg_load)) { 6018 /* 6019 * The runnable loads are close so take the 6020 * blocked load into account through avg_load: 6021 */ 6022 min_avg_load = avg_load; 6023 idlest = group; 6024 } 6025 6026 if (most_spare < max_spare_cap) { 6027 most_spare = max_spare_cap; 6028 most_spare_sg = group; 6029 } 6030 } 6031 } while (group = group->next, group != sd->groups); 6032 6033 /* 6034 * The cross-over point between using spare capacity or least load 6035 * is too conservative for high utilization tasks on partially 6036 * utilized systems if we require spare_capacity > task_util(p), 6037 * so we allow for some task stuffing by using 6038 * spare_capacity > task_util(p)/2. 6039 * 6040 * Spare capacity can't be used for fork because the utilization has 6041 * not been set yet, we must first select a rq to compute the initial 6042 * utilization. 6043 */ 6044 if (sd_flag & SD_BALANCE_FORK) 6045 goto skip_spare; 6046 6047 if (this_spare > task_util(p) / 2 && 6048 imbalance_scale*this_spare > 100*most_spare) 6049 return NULL; 6050 6051 if (most_spare > task_util(p) / 2) 6052 return most_spare_sg; 6053 6054 skip_spare: 6055 if (!idlest) 6056 return NULL; 6057 6058 /* 6059 * When comparing groups across NUMA domains, it's possible for the 6060 * local domain to be very lightly loaded relative to the remote 6061 * domains but "imbalance" skews the comparison making remote CPUs 6062 * look much more favourable. When considering cross-domain, add 6063 * imbalance to the runnable load on the remote node and consider 6064 * staying local. 6065 */ 6066 if ((sd->flags & SD_NUMA) && 6067 min_runnable_load + imbalance >= this_runnable_load) 6068 return NULL; 6069 6070 if (min_runnable_load > (this_runnable_load + imbalance)) 6071 return NULL; 6072 6073 if ((this_runnable_load < (min_runnable_load + imbalance)) && 6074 (100*this_avg_load < imbalance_scale*min_avg_load)) 6075 return NULL; 6076 6077 return idlest; 6078 } 6079 6080 /* 6081 * find_idlest_group_cpu - find the idlest CPU among the CPUs in the group. 6082 */ 6083 static int 6084 find_idlest_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) 6085 { 6086 unsigned long load, min_load = ULONG_MAX; 6087 unsigned int min_exit_latency = UINT_MAX; 6088 u64 latest_idle_timestamp = 0; 6089 int least_loaded_cpu = this_cpu; 6090 int shallowest_idle_cpu = -1; 6091 int i; 6092 6093 /* Check if we have any choice: */ 6094 if (group->group_weight == 1) 6095 return cpumask_first(sched_group_span(group)); 6096 6097 /* Traverse only the allowed CPUs */ 6098 for_each_cpu_and(i, sched_group_span(group), &p->cpus_allowed) { 6099 if (idle_cpu(i)) { 6100 struct rq *rq = cpu_rq(i); 6101 struct cpuidle_state *idle = idle_get_state(rq); 6102 if (idle && idle->exit_latency < min_exit_latency) { 6103 /* 6104 * We give priority to a CPU whose idle state 6105 * has the smallest exit latency irrespective 6106 * of any idle timestamp. 6107 */ 6108 min_exit_latency = idle->exit_latency; 6109 latest_idle_timestamp = rq->idle_stamp; 6110 shallowest_idle_cpu = i; 6111 } else if ((!idle || idle->exit_latency == min_exit_latency) && 6112 rq->idle_stamp > latest_idle_timestamp) { 6113 /* 6114 * If equal or no active idle state, then 6115 * the most recently idled CPU might have 6116 * a warmer cache. 6117 */ 6118 latest_idle_timestamp = rq->idle_stamp; 6119 shallowest_idle_cpu = i; 6120 } 6121 } else if (shallowest_idle_cpu == -1) { 6122 load = weighted_cpuload(cpu_rq(i)); 6123 if (load < min_load) { 6124 min_load = load; 6125 least_loaded_cpu = i; 6126 } 6127 } 6128 } 6129 6130 return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu; 6131 } 6132 6133 static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *p, 6134 int cpu, int prev_cpu, int sd_flag) 6135 { 6136 int new_cpu = cpu; 6137 6138 if (!cpumask_intersects(sched_domain_span(sd), &p->cpus_allowed)) 6139 return prev_cpu; 6140 6141 while (sd) { 6142 struct sched_group *group; 6143 struct sched_domain *tmp; 6144 int weight; 6145 6146 if (!(sd->flags & sd_flag)) { 6147 sd = sd->child; 6148 continue; 6149 } 6150 6151 group = find_idlest_group(sd, p, cpu, sd_flag); 6152 if (!group) { 6153 sd = sd->child; 6154 continue; 6155 } 6156 6157 new_cpu = find_idlest_group_cpu(group, p, cpu); 6158 if (new_cpu == cpu) { 6159 /* Now try balancing at a lower domain level of 'cpu': */ 6160 sd = sd->child; 6161 continue; 6162 } 6163 6164 /* Now try balancing at a lower domain level of 'new_cpu': */ 6165 cpu = new_cpu; 6166 weight = sd->span_weight; 6167 sd = NULL; 6168 for_each_domain(cpu, tmp) { 6169 if (weight <= tmp->span_weight) 6170 break; 6171 if (tmp->flags & sd_flag) 6172 sd = tmp; 6173 } 6174 } 6175 6176 return new_cpu; 6177 } 6178 6179 #ifdef CONFIG_SCHED_SMT 6180 DEFINE_STATIC_KEY_FALSE(sched_smt_present); 6181 6182 static inline void set_idle_cores(int cpu, int val) 6183 { 6184 struct sched_domain_shared *sds; 6185 6186 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 6187 if (sds) 6188 WRITE_ONCE(sds->has_idle_cores, val); 6189 } 6190 6191 static inline bool test_idle_cores(int cpu, bool def) 6192 { 6193 struct sched_domain_shared *sds; 6194 6195 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 6196 if (sds) 6197 return READ_ONCE(sds->has_idle_cores); 6198 6199 return def; 6200 } 6201 6202 /* 6203 * Scans the local SMT mask to see if the entire core is idle, and records this 6204 * information in sd_llc_shared->has_idle_cores. 6205 * 6206 * Since SMT siblings share all cache levels, inspecting this limited remote 6207 * state should be fairly cheap. 6208 */ 6209 void __update_idle_core(struct rq *rq) 6210 { 6211 int core = cpu_of(rq); 6212 int cpu; 6213 6214 rcu_read_lock(); 6215 if (test_idle_cores(core, true)) 6216 goto unlock; 6217 6218 for_each_cpu(cpu, cpu_smt_mask(core)) { 6219 if (cpu == core) 6220 continue; 6221 6222 if (!idle_cpu(cpu)) 6223 goto unlock; 6224 } 6225 6226 set_idle_cores(core, 1); 6227 unlock: 6228 rcu_read_unlock(); 6229 } 6230 6231 /* 6232 * Scan the entire LLC domain for idle cores; this dynamically switches off if 6233 * there are no idle cores left in the system; tracked through 6234 * sd_llc->shared->has_idle_cores and enabled through update_idle_core() above. 6235 */ 6236 static int select_idle_core(struct task_struct *p, struct sched_domain *sd, int target) 6237 { 6238 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_idle_mask); 6239 int core, cpu; 6240 6241 if (!static_branch_likely(&sched_smt_present)) 6242 return -1; 6243 6244 if (!test_idle_cores(target, false)) 6245 return -1; 6246 6247 cpumask_and(cpus, sched_domain_span(sd), &p->cpus_allowed); 6248 6249 for_each_cpu_wrap(core, cpus, target) { 6250 bool idle = true; 6251 6252 for_each_cpu(cpu, cpu_smt_mask(core)) { 6253 cpumask_clear_cpu(cpu, cpus); 6254 if (!idle_cpu(cpu)) 6255 idle = false; 6256 } 6257 6258 if (idle) 6259 return core; 6260 } 6261 6262 /* 6263 * Failed to find an idle core; stop looking for one. 6264 */ 6265 set_idle_cores(target, 0); 6266 6267 return -1; 6268 } 6269 6270 /* 6271 * Scan the local SMT mask for idle CPUs. 6272 */ 6273 static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) 6274 { 6275 int cpu; 6276 6277 if (!static_branch_likely(&sched_smt_present)) 6278 return -1; 6279 6280 for_each_cpu(cpu, cpu_smt_mask(target)) { 6281 if (!cpumask_test_cpu(cpu, &p->cpus_allowed)) 6282 continue; 6283 if (idle_cpu(cpu)) 6284 return cpu; 6285 } 6286 6287 return -1; 6288 } 6289 6290 #else /* CONFIG_SCHED_SMT */ 6291 6292 static inline int select_idle_core(struct task_struct *p, struct sched_domain *sd, int target) 6293 { 6294 return -1; 6295 } 6296 6297 static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) 6298 { 6299 return -1; 6300 } 6301 6302 #endif /* CONFIG_SCHED_SMT */ 6303 6304 /* 6305 * Scan the LLC domain for idle CPUs; this is dynamically regulated by 6306 * comparing the average scan cost (tracked in sd->avg_scan_cost) against the 6307 * average idle time for this rq (as found in rq->avg_idle). 6308 */ 6309 static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int target) 6310 { 6311 struct sched_domain *this_sd; 6312 u64 avg_cost, avg_idle; 6313 u64 time, cost; 6314 s64 delta; 6315 int cpu, nr = INT_MAX; 6316 6317 this_sd = rcu_dereference(*this_cpu_ptr(&sd_llc)); 6318 if (!this_sd) 6319 return -1; 6320 6321 /* 6322 * Due to large variance we need a large fuzz factor; hackbench in 6323 * particularly is sensitive here. 6324 */ 6325 avg_idle = this_rq()->avg_idle / 512; 6326 avg_cost = this_sd->avg_scan_cost + 1; 6327 6328 if (sched_feat(SIS_AVG_CPU) && avg_idle < avg_cost) 6329 return -1; 6330 6331 if (sched_feat(SIS_PROP)) { 6332 u64 span_avg = sd->span_weight * avg_idle; 6333 if (span_avg > 4*avg_cost) 6334 nr = div_u64(span_avg, avg_cost); 6335 else 6336 nr = 4; 6337 } 6338 6339 time = local_clock(); 6340 6341 for_each_cpu_wrap(cpu, sched_domain_span(sd), target) { 6342 if (!--nr) 6343 return -1; 6344 if (!cpumask_test_cpu(cpu, &p->cpus_allowed)) 6345 continue; 6346 if (idle_cpu(cpu)) 6347 break; 6348 } 6349 6350 time = local_clock() - time; 6351 cost = this_sd->avg_scan_cost; 6352 delta = (s64)(time - cost) / 8; 6353 this_sd->avg_scan_cost += delta; 6354 6355 return cpu; 6356 } 6357 6358 /* 6359 * Try and locate an idle core/thread in the LLC cache domain. 6360 */ 6361 static int select_idle_sibling(struct task_struct *p, int prev, int target) 6362 { 6363 struct sched_domain *sd; 6364 int i, recent_used_cpu; 6365 6366 if (idle_cpu(target)) 6367 return target; 6368 6369 /* 6370 * If the previous CPU is cache affine and idle, don't be stupid: 6371 */ 6372 if (prev != target && cpus_share_cache(prev, target) && idle_cpu(prev)) 6373 return prev; 6374 6375 /* Check a recently used CPU as a potential idle candidate: */ 6376 recent_used_cpu = p->recent_used_cpu; 6377 if (recent_used_cpu != prev && 6378 recent_used_cpu != target && 6379 cpus_share_cache(recent_used_cpu, target) && 6380 idle_cpu(recent_used_cpu) && 6381 cpumask_test_cpu(p->recent_used_cpu, &p->cpus_allowed)) { 6382 /* 6383 * Replace recent_used_cpu with prev as it is a potential 6384 * candidate for the next wake: 6385 */ 6386 p->recent_used_cpu = prev; 6387 return recent_used_cpu; 6388 } 6389 6390 sd = rcu_dereference(per_cpu(sd_llc, target)); 6391 if (!sd) 6392 return target; 6393 6394 i = select_idle_core(p, sd, target); 6395 if ((unsigned)i < nr_cpumask_bits) 6396 return i; 6397 6398 i = select_idle_cpu(p, sd, target); 6399 if ((unsigned)i < nr_cpumask_bits) 6400 return i; 6401 6402 i = select_idle_smt(p, sd, target);