~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/time/tick-sched.c

Version: ~ [ linux-5.15-rc1 ] ~ [ linux-5.14.5 ] ~ [ linux-5.13.18 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.66 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.147 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.206 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.246 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.282 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.283 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *  linux/kernel/time/tick-sched.c
  3  *
  4  *  Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
  5  *  Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
  6  *  Copyright(C) 2006-2007  Timesys Corp., Thomas Gleixner
  7  *
  8  *  No idle tick implementation for low and high resolution timers
  9  *
 10  *  Started by: Thomas Gleixner and Ingo Molnar
 11  *
 12  *  Distribute under GPLv2.
 13  */
 14 #include <linux/cpu.h>
 15 #include <linux/err.h>
 16 #include <linux/hrtimer.h>
 17 #include <linux/interrupt.h>
 18 #include <linux/kernel_stat.h>
 19 #include <linux/percpu.h>
 20 #include <linux/nmi.h>
 21 #include <linux/profile.h>
 22 #include <linux/sched/signal.h>
 23 #include <linux/sched/clock.h>
 24 #include <linux/sched/stat.h>
 25 #include <linux/sched/nohz.h>
 26 #include <linux/module.h>
 27 #include <linux/irq_work.h>
 28 #include <linux/posix-timers.h>
 29 #include <linux/context_tracking.h>
 30 #include <linux/mm.h>
 31 
 32 #include <asm/irq_regs.h>
 33 
 34 #include "tick-internal.h"
 35 
 36 #include <trace/events/timer.h>
 37 
 38 /*
 39  * Per-CPU nohz control structure
 40  */
 41 static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
 42 
 43 struct tick_sched *tick_get_tick_sched(int cpu)
 44 {
 45         return &per_cpu(tick_cpu_sched, cpu);
 46 }
 47 
 48 #if defined(CONFIG_NO_HZ_COMMON) || defined(CONFIG_HIGH_RES_TIMERS)
 49 /*
 50  * The time, when the last jiffy update happened. Protected by jiffies_lock.
 51  */
 52 static ktime_t last_jiffies_update;
 53 
 54 /*
 55  * Must be called with interrupts disabled !
 56  */
 57 static void tick_do_update_jiffies64(ktime_t now)
 58 {
 59         unsigned long ticks = 0;
 60         ktime_t delta;
 61 
 62         /*
 63          * Do a quick check without holding jiffies_lock:
 64          */
 65         delta = ktime_sub(now, last_jiffies_update);
 66         if (delta < tick_period)
 67                 return;
 68 
 69         /* Reevaluate with jiffies_lock held */
 70         write_seqlock(&jiffies_lock);
 71 
 72         delta = ktime_sub(now, last_jiffies_update);
 73         if (delta >= tick_period) {
 74 
 75                 delta = ktime_sub(delta, tick_period);
 76                 last_jiffies_update = ktime_add(last_jiffies_update,
 77                                                 tick_period);
 78 
 79                 /* Slow path for long timeouts */
 80                 if (unlikely(delta >= tick_period)) {
 81                         s64 incr = ktime_to_ns(tick_period);
 82 
 83                         ticks = ktime_divns(delta, incr);
 84 
 85                         last_jiffies_update = ktime_add_ns(last_jiffies_update,
 86                                                            incr * ticks);
 87                 }
 88                 do_timer(++ticks);
 89 
 90                 /* Keep the tick_next_period variable up to date */
 91                 tick_next_period = ktime_add(last_jiffies_update, tick_period);
 92         } else {
 93                 write_sequnlock(&jiffies_lock);
 94                 return;
 95         }
 96         write_sequnlock(&jiffies_lock);
 97         update_wall_time();
 98 }
 99 
100 /*
101  * Initialize and return retrieve the jiffies update.
102  */
103 static ktime_t tick_init_jiffy_update(void)
104 {
105         ktime_t period;
106 
107         write_seqlock(&jiffies_lock);
108         /* Did we start the jiffies update yet ? */
109         if (last_jiffies_update == 0)
110                 last_jiffies_update = tick_next_period;
111         period = last_jiffies_update;
112         write_sequnlock(&jiffies_lock);
113         return period;
114 }
115 
116 
117 static void tick_sched_do_timer(ktime_t now)
118 {
119         int cpu = smp_processor_id();
120 
121 #ifdef CONFIG_NO_HZ_COMMON
122         /*
123          * Check if the do_timer duty was dropped. We don't care about
124          * concurrency: This happens only when the CPU in charge went
125          * into a long sleep. If two CPUs happen to assign themselves to
126          * this duty, then the jiffies update is still serialized by
127          * jiffies_lock.
128          */
129         if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE)
130             && !tick_nohz_full_cpu(cpu))
131                 tick_do_timer_cpu = cpu;
132 #endif
133 
134         /* Check, if the jiffies need an update */
135         if (tick_do_timer_cpu == cpu)
136                 tick_do_update_jiffies64(now);
137 }
138 
139 static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)
140 {
141 #ifdef CONFIG_NO_HZ_COMMON
142         /*
143          * When we are idle and the tick is stopped, we have to touch
144          * the watchdog as we might not schedule for a really long
145          * time. This happens on complete idle SMP systems while
146          * waiting on the login prompt. We also increment the "start of
147          * idle" jiffy stamp so the idle accounting adjustment we do
148          * when we go busy again does not account too much ticks.
149          */
150         if (ts->tick_stopped) {
151                 touch_softlockup_watchdog_sched();
152                 if (is_idle_task(current))
153                         ts->idle_jiffies++;
154                 /*
155                  * In case the current tick fired too early past its expected
156                  * expiration, make sure we don't bypass the next clock reprogramming
157                  * to the same deadline.
158                  */
159                 ts->next_tick = 0;
160         }
161 #endif
162         update_process_times(user_mode(regs));
163         profile_tick(CPU_PROFILING);
164 }
165 #endif
166 
167 #ifdef CONFIG_NO_HZ_FULL
168 cpumask_var_t tick_nohz_full_mask;
169 bool tick_nohz_full_running;
170 static atomic_t tick_dep_mask;
171 
172 static bool check_tick_dependency(atomic_t *dep)
173 {
174         int val = atomic_read(dep);
175 
176         if (val & TICK_DEP_MASK_POSIX_TIMER) {
177                 trace_tick_stop(0, TICK_DEP_MASK_POSIX_TIMER);
178                 return true;
179         }
180 
181         if (val & TICK_DEP_MASK_PERF_EVENTS) {
182                 trace_tick_stop(0, TICK_DEP_MASK_PERF_EVENTS);
183                 return true;
184         }
185 
186         if (val & TICK_DEP_MASK_SCHED) {
187                 trace_tick_stop(0, TICK_DEP_MASK_SCHED);
188                 return true;
189         }
190 
191         if (val & TICK_DEP_MASK_CLOCK_UNSTABLE) {
192                 trace_tick_stop(0, TICK_DEP_MASK_CLOCK_UNSTABLE);
193                 return true;
194         }
195 
196         return false;
197 }
198 
199 static bool can_stop_full_tick(int cpu, struct tick_sched *ts)
200 {
201         lockdep_assert_irqs_disabled();
202 
203         if (unlikely(!cpu_online(cpu)))
204                 return false;
205 
206         if (check_tick_dependency(&tick_dep_mask))
207                 return false;
208 
209         if (check_tick_dependency(&ts->tick_dep_mask))
210                 return false;
211 
212         if (check_tick_dependency(&current->tick_dep_mask))
213                 return false;
214 
215         if (check_tick_dependency(&current->signal->tick_dep_mask))
216                 return false;
217 
218         return true;
219 }
220 
221 static void nohz_full_kick_func(struct irq_work *work)
222 {
223         /* Empty, the tick restart happens on tick_nohz_irq_exit() */
224 }
225 
226 static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) = {
227         .func = nohz_full_kick_func,
228 };
229 
230 /*
231  * Kick this CPU if it's full dynticks in order to force it to
232  * re-evaluate its dependency on the tick and restart it if necessary.
233  * This kick, unlike tick_nohz_full_kick_cpu() and tick_nohz_full_kick_all(),
234  * is NMI safe.
235  */
236 static void tick_nohz_full_kick(void)
237 {
238         if (!tick_nohz_full_cpu(smp_processor_id()))
239                 return;
240 
241         irq_work_queue(this_cpu_ptr(&nohz_full_kick_work));
242 }
243 
244 /*
245  * Kick the CPU if it's full dynticks in order to force it to
246  * re-evaluate its dependency on the tick and restart it if necessary.
247  */
248 void tick_nohz_full_kick_cpu(int cpu)
249 {
250         if (!tick_nohz_full_cpu(cpu))
251                 return;
252 
253         irq_work_queue_on(&per_cpu(nohz_full_kick_work, cpu), cpu);
254 }
255 
256 /*
257  * Kick all full dynticks CPUs in order to force these to re-evaluate
258  * their dependency on the tick and restart it if necessary.
259  */
260 static void tick_nohz_full_kick_all(void)
261 {
262         int cpu;
263 
264         if (!tick_nohz_full_running)
265                 return;
266 
267         preempt_disable();
268         for_each_cpu_and(cpu, tick_nohz_full_mask, cpu_online_mask)
269                 tick_nohz_full_kick_cpu(cpu);
270         preempt_enable();
271 }
272 
273 static void tick_nohz_dep_set_all(atomic_t *dep,
274                                   enum tick_dep_bits bit)
275 {
276         int prev;
277 
278         prev = atomic_fetch_or(BIT(bit), dep);
279         if (!prev)
280                 tick_nohz_full_kick_all();
281 }
282 
283 /*
284  * Set a global tick dependency. Used by perf events that rely on freq and
285  * by unstable clock.
286  */
287 void tick_nohz_dep_set(enum tick_dep_bits bit)
288 {
289         tick_nohz_dep_set_all(&tick_dep_mask, bit);
290 }
291 
292 void tick_nohz_dep_clear(enum tick_dep_bits bit)
293 {
294         atomic_andnot(BIT(bit), &tick_dep_mask);
295 }
296 
297 /*
298  * Set per-CPU tick dependency. Used by scheduler and perf events in order to
299  * manage events throttling.
300  */
301 void tick_nohz_dep_set_cpu(int cpu, enum tick_dep_bits bit)
302 {
303         int prev;
304         struct tick_sched *ts;
305 
306         ts = per_cpu_ptr(&tick_cpu_sched, cpu);
307 
308         prev = atomic_fetch_or(BIT(bit), &ts->tick_dep_mask);
309         if (!prev) {
310                 preempt_disable();
311                 /* Perf needs local kick that is NMI safe */
312                 if (cpu == smp_processor_id()) {
313                         tick_nohz_full_kick();
314                 } else {
315                         /* Remote irq work not NMI-safe */
316                         if (!WARN_ON_ONCE(in_nmi()))
317                                 tick_nohz_full_kick_cpu(cpu);
318                 }
319                 preempt_enable();
320         }
321 }
322 
323 void tick_nohz_dep_clear_cpu(int cpu, enum tick_dep_bits bit)
324 {
325         struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);
326 
327         atomic_andnot(BIT(bit), &ts->tick_dep_mask);
328 }
329 
330 /*
331  * Set a per-task tick dependency. Posix CPU timers need this in order to elapse
332  * per task timers.
333  */
334 void tick_nohz_dep_set_task(struct task_struct *tsk, enum tick_dep_bits bit)
335 {
336         /*
337          * We could optimize this with just kicking the target running the task
338          * if that noise matters for nohz full users.
339          */
340         tick_nohz_dep_set_all(&tsk->tick_dep_mask, bit);
341 }
342 
343 void tick_nohz_dep_clear_task(struct task_struct *tsk, enum tick_dep_bits bit)
344 {
345         atomic_andnot(BIT(bit), &tsk->tick_dep_mask);
346 }
347 
348 /*
349  * Set a per-taskgroup tick dependency. Posix CPU timers need this in order to elapse
350  * per process timers.
351  */
352 void tick_nohz_dep_set_signal(struct signal_struct *sig, enum tick_dep_bits bit)
353 {
354         tick_nohz_dep_set_all(&sig->tick_dep_mask, bit);
355 }
356 
357 void tick_nohz_dep_clear_signal(struct signal_struct *sig, enum tick_dep_bits bit)
358 {
359         atomic_andnot(BIT(bit), &sig->tick_dep_mask);
360 }
361 
362 /*
363  * Re-evaluate the need for the tick as we switch the current task.
364  * It might need the tick due to per task/process properties:
365  * perf events, posix CPU timers, ...
366  */
367 void __tick_nohz_task_switch(void)
368 {
369         unsigned long flags;
370         struct tick_sched *ts;
371 
372         local_irq_save(flags);
373 
374         if (!tick_nohz_full_cpu(smp_processor_id()))
375                 goto out;
376 
377         ts = this_cpu_ptr(&tick_cpu_sched);
378 
379         if (ts->tick_stopped) {
380                 if (atomic_read(&current->tick_dep_mask) ||
381                     atomic_read(&current->signal->tick_dep_mask))
382                         tick_nohz_full_kick();
383         }
384 out:
385         local_irq_restore(flags);
386 }
387 
388 /* Get the boot-time nohz CPU list from the kernel parameters. */
389 void __init tick_nohz_full_setup(cpumask_var_t cpumask)
390 {
391         alloc_bootmem_cpumask_var(&tick_nohz_full_mask);
392         cpumask_copy(tick_nohz_full_mask, cpumask);
393         tick_nohz_full_running = true;
394 }
395 
396 static int tick_nohz_cpu_down(unsigned int cpu)
397 {
398         /*
399          * The boot CPU handles housekeeping duty (unbound timers,
400          * workqueues, timekeeping, ...) on behalf of full dynticks
401          * CPUs. It must remain online when nohz full is enabled.
402          */
403         if (tick_nohz_full_running && tick_do_timer_cpu == cpu)
404                 return -EBUSY;
405         return 0;
406 }
407 
408 static int tick_nohz_init_all(void)
409 {
410         int err = -1;
411 
412 #ifdef CONFIG_NO_HZ_FULL_ALL
413         if (!alloc_cpumask_var(&tick_nohz_full_mask, GFP_KERNEL)) {
414                 WARN(1, "NO_HZ: Can't allocate full dynticks cpumask\n");
415                 return err;
416         }
417         err = 0;
418         cpumask_setall(tick_nohz_full_mask);
419         tick_nohz_full_running = true;
420 #endif
421         return err;
422 }
423 
424 void __init tick_nohz_init(void)
425 {
426         int cpu, ret;
427 
428         if (!tick_nohz_full_running) {
429                 if (tick_nohz_init_all() < 0)
430                         return;
431         }
432 
433         /*
434          * Full dynticks uses irq work to drive the tick rescheduling on safe
435          * locking contexts. But then we need irq work to raise its own
436          * interrupts to avoid circular dependency on the tick
437          */
438         if (!arch_irq_work_has_interrupt()) {
439                 pr_warn("NO_HZ: Can't run full dynticks because arch doesn't support irq work self-IPIs\n");
440                 cpumask_clear(tick_nohz_full_mask);
441                 tick_nohz_full_running = false;
442                 return;
443         }
444 
445         cpu = smp_processor_id();
446 
447         if (cpumask_test_cpu(cpu, tick_nohz_full_mask)) {
448                 pr_warn("NO_HZ: Clearing %d from nohz_full range for timekeeping\n",
449                         cpu);
450                 cpumask_clear_cpu(cpu, tick_nohz_full_mask);
451         }
452 
453         for_each_cpu(cpu, tick_nohz_full_mask)
454                 context_tracking_cpu_set(cpu);
455 
456         ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
457                                         "kernel/nohz:predown", NULL,
458                                         tick_nohz_cpu_down);
459         WARN_ON(ret < 0);
460         pr_info("NO_HZ: Full dynticks CPUs: %*pbl.\n",
461                 cpumask_pr_args(tick_nohz_full_mask));
462 }
463 #endif
464 
465 /*
466  * NOHZ - aka dynamic tick functionality
467  */
468 #ifdef CONFIG_NO_HZ_COMMON
469 /*
470  * NO HZ enabled ?
471  */
472 bool tick_nohz_enabled __read_mostly  = true;
473 unsigned long tick_nohz_active  __read_mostly;
474 /*
475  * Enable / Disable tickless mode
476  */
477 static int __init setup_tick_nohz(char *str)
478 {
479         return (kstrtobool(str, &tick_nohz_enabled) == 0);
480 }
481 
482 __setup("nohz=", setup_tick_nohz);
483 
484 int tick_nohz_tick_stopped(void)
485 {
486         return __this_cpu_read(tick_cpu_sched.tick_stopped);
487 }
488 
489 /**
490  * tick_nohz_update_jiffies - update jiffies when idle was interrupted
491  *
492  * Called from interrupt entry when the CPU was idle
493  *
494  * In case the sched_tick was stopped on this CPU, we have to check if jiffies
495  * must be updated. Otherwise an interrupt handler could use a stale jiffy
496  * value. We do this unconditionally on any CPU, as we don't know whether the
497  * CPU, which has the update task assigned is in a long sleep.
498  */
499 static void tick_nohz_update_jiffies(ktime_t now)
500 {
501         unsigned long flags;
502 
503         __this_cpu_write(tick_cpu_sched.idle_waketime, now);
504 
505         local_irq_save(flags);
506         tick_do_update_jiffies64(now);
507         local_irq_restore(flags);
508 
509         touch_softlockup_watchdog_sched();
510 }
511 
512 /*
513  * Updates the per-CPU time idle statistics counters
514  */
515 static void
516 update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time)
517 {
518         ktime_t delta;
519 
520         if (ts->idle_active) {
521                 delta = ktime_sub(now, ts->idle_entrytime);
522                 if (nr_iowait_cpu(cpu) > 0)
523                         ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);
524                 else
525                         ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
526                 ts->idle_entrytime = now;
527         }
528 
529         if (last_update_time)
530                 *last_update_time = ktime_to_us(now);
531 
532 }
533 
534 static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)
535 {
536         update_ts_time_stats(smp_processor_id(), ts, now, NULL);
537         ts->idle_active = 0;
538 
539         sched_clock_idle_wakeup_event();
540 }
541 
542 static ktime_t tick_nohz_start_idle(struct tick_sched *ts)
543 {
544         ktime_t now = ktime_get();
545 
546         ts->idle_entrytime = now;
547         ts->idle_active = 1;
548         sched_clock_idle_sleep_event();
549         return now;
550 }
551 
552 /**
553  * get_cpu_idle_time_us - get the total idle time of a CPU
554  * @cpu: CPU number to query
555  * @last_update_time: variable to store update time in. Do not update
556  * counters if NULL.
557  *
558  * Return the cumulative idle time (since boot) for a given
559  * CPU, in microseconds.
560  *
561  * This time is measured via accounting rather than sampling,
562  * and is as accurate as ktime_get() is.
563  *
564  * This function returns -1 if NOHZ is not enabled.
565  */
566 u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
567 {
568         struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
569         ktime_t now, idle;
570 
571         if (!tick_nohz_active)
572                 return -1;
573 
574         now = ktime_get();
575         if (last_update_time) {
576                 update_ts_time_stats(cpu, ts, now, last_update_time);
577                 idle = ts->idle_sleeptime;
578         } else {
579                 if (ts->idle_active && !nr_iowait_cpu(cpu)) {
580                         ktime_t delta = ktime_sub(now, ts->idle_entrytime);
581 
582                         idle = ktime_add(ts->idle_sleeptime, delta);
583                 } else {
584                         idle = ts->idle_sleeptime;
585                 }
586         }
587 
588         return ktime_to_us(idle);
589 
590 }
591 EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
592 
593 /**
594  * get_cpu_iowait_time_us - get the total iowait time of a CPU
595  * @cpu: CPU number to query
596  * @last_update_time: variable to store update time in. Do not update
597  * counters if NULL.
598  *
599  * Return the cumulative iowait time (since boot) for a given
600  * CPU, in microseconds.
601  *
602  * This time is measured via accounting rather than sampling,
603  * and is as accurate as ktime_get() is.
604  *
605  * This function returns -1 if NOHZ is not enabled.
606  */
607 u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
608 {
609         struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
610         ktime_t now, iowait;
611 
612         if (!tick_nohz_active)
613                 return -1;
614 
615         now = ktime_get();
616         if (last_update_time) {
617                 update_ts_time_stats(cpu, ts, now, last_update_time);
618                 iowait = ts->iowait_sleeptime;
619         } else {
620                 if (ts->idle_active && nr_iowait_cpu(cpu) > 0) {
621                         ktime_t delta = ktime_sub(now, ts->idle_entrytime);
622 
623                         iowait = ktime_add(ts->iowait_sleeptime, delta);
624                 } else {
625                         iowait = ts->iowait_sleeptime;
626                 }
627         }
628 
629         return ktime_to_us(iowait);
630 }
631 EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
632 
633 static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
634 {
635         hrtimer_cancel(&ts->sched_timer);
636         hrtimer_set_expires(&ts->sched_timer, ts->last_tick);
637 
638         /* Forward the time to expire in the future */
639         hrtimer_forward(&ts->sched_timer, now, tick_period);
640 
641         if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
642                 hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED);
643         else
644                 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
645 
646         /*
647          * Reset to make sure next tick stop doesn't get fooled by past
648          * cached clock deadline.
649          */
650         ts->next_tick = 0;
651 }
652 
653 static inline bool local_timer_softirq_pending(void)
654 {
655         return local_softirq_pending() & TIMER_SOFTIRQ;
656 }
657 
658 static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
659                                          ktime_t now, int cpu)
660 {
661         struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
662         u64 basemono, next_tick, next_tmr, next_rcu, delta, expires;
663         unsigned long seq, basejiff;
664         ktime_t tick;
665 
666         /* Read jiffies and the time when jiffies were updated last */
667         do {
668                 seq = read_seqbegin(&jiffies_lock);
669                 basemono = last_jiffies_update;
670                 basejiff = jiffies;
671         } while (read_seqretry(&jiffies_lock, seq));
672         ts->last_jiffies = basejiff;
673 
674         /*
675          * Keep the periodic tick, when RCU, architecture or irq_work
676          * requests it.
677          * Aside of that check whether the local timer softirq is
678          * pending. If so its a bad idea to call get_next_timer_interrupt()
679          * because there is an already expired timer, so it will request
680          * immeditate expiry, which rearms the hardware timer with a
681          * minimal delta which brings us back to this place
682          * immediately. Lather, rinse and repeat...
683          */
684         if (rcu_needs_cpu(basemono, &next_rcu) || arch_needs_cpu() ||
685             irq_work_needs_cpu() || local_timer_softirq_pending()) {
686                 next_tick = basemono + TICK_NSEC;
687         } else {
688                 /*
689                  * Get the next pending timer. If high resolution
690                  * timers are enabled this only takes the timer wheel
691                  * timers into account. If high resolution timers are
692                  * disabled this also looks at the next expiring
693                  * hrtimer.
694                  */
695                 next_tmr = get_next_timer_interrupt(basejiff, basemono);
696                 ts->next_timer = next_tmr;
697                 /* Take the next rcu event into account */
698                 next_tick = next_rcu < next_tmr ? next_rcu : next_tmr;
699         }
700 
701         /*
702          * If the tick is due in the next period, keep it ticking or
703          * force prod the timer.
704          */
705         delta = next_tick - basemono;
706         if (delta <= (u64)TICK_NSEC) {
707                 /*
708                  * Tell the timer code that the base is not idle, i.e. undo
709                  * the effect of get_next_timer_interrupt():
710                  */
711                 timer_clear_idle();
712                 /*
713                  * We've not stopped the tick yet, and there's a timer in the
714                  * next period, so no point in stopping it either, bail.
715                  */
716                 if (!ts->tick_stopped) {
717                         tick = 0;
718                         goto out;
719                 }
720         }
721 
722         /*
723          * If this CPU is the one which updates jiffies, then give up
724          * the assignment and let it be taken by the CPU which runs
725          * the tick timer next, which might be this CPU as well. If we
726          * don't drop this here the jiffies might be stale and
727          * do_timer() never invoked. Keep track of the fact that it
728          * was the one which had the do_timer() duty last. If this CPU
729          * is the one which had the do_timer() duty last, we limit the
730          * sleep time to the timekeeping max_deferment value.
731          * Otherwise we can sleep as long as we want.
732          */
733         delta = timekeeping_max_deferment();
734         if (cpu == tick_do_timer_cpu) {
735                 tick_do_timer_cpu = TICK_DO_TIMER_NONE;
736                 ts->do_timer_last = 1;
737         } else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) {
738                 delta = KTIME_MAX;
739                 ts->do_timer_last = 0;
740         } else if (!ts->do_timer_last) {
741                 delta = KTIME_MAX;
742         }
743 
744 #ifdef CONFIG_NO_HZ_FULL
745         /* Limit the tick delta to the maximum scheduler deferment */
746         if (!ts->inidle)
747                 delta = min(delta, scheduler_tick_max_deferment());
748 #endif
749 
750         /* Calculate the next expiry time */
751         if (delta < (KTIME_MAX - basemono))
752                 expires = basemono + delta;
753         else
754                 expires = KTIME_MAX;
755 
756         expires = min_t(u64, expires, next_tick);
757         tick = expires;
758 
759         /* Skip reprogram of event if its not changed */
760         if (ts->tick_stopped && (expires == ts->next_tick)) {
761                 /* Sanity check: make sure clockevent is actually programmed */
762                 if (tick == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer))
763                         goto out;
764 
765                 WARN_ON_ONCE(1);
766                 printk_once("basemono: %llu ts->next_tick: %llu dev->next_event: %llu timer->active: %d timer->expires: %llu\n",
767                             basemono, ts->next_tick, dev->next_event,
768                             hrtimer_active(&ts->sched_timer), hrtimer_get_expires(&ts->sched_timer));
769         }
770 
771         /*
772          * nohz_stop_sched_tick can be called several times before
773          * the nohz_restart_sched_tick is called. This happens when
774          * interrupts arrive which do not cause a reschedule. In the
775          * first call we save the current tick time, so we can restart
776          * the scheduler tick in nohz_restart_sched_tick.
777          */
778         if (!ts->tick_stopped) {
779                 calc_load_nohz_start();
780                 cpu_load_update_nohz_start();
781                 quiet_vmstat();
782 
783                 ts->last_tick = hrtimer_get_expires(&ts->sched_timer);
784                 ts->tick_stopped = 1;
785                 trace_tick_stop(1, TICK_DEP_MASK_NONE);
786         }
787 
788         ts->next_tick = tick;
789 
790         /*
791          * If the expiration time == KTIME_MAX, then we simply stop
792          * the tick timer.
793          */
794         if (unlikely(expires == KTIME_MAX)) {
795                 if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
796                         hrtimer_cancel(&ts->sched_timer);
797                 goto out;
798         }
799 
800         if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
801                 hrtimer_start(&ts->sched_timer, tick, HRTIMER_MODE_ABS_PINNED);
802         } else {
803                 hrtimer_set_expires(&ts->sched_timer, tick);
804                 tick_program_event(tick, 1);
805         }
806 
807 out:
808         /*
809          * Update the estimated sleep length until the next timer
810          * (not only the tick).
811          */
812         ts->sleep_length = ktime_sub(dev->next_event, now);
813         return tick;
814 }
815 
816 static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
817 {
818         /* Update jiffies first */
819         tick_do_update_jiffies64(now);
820         cpu_load_update_nohz_stop();
821 
822         /*
823          * Clear the timer idle flag, so we avoid IPIs on remote queueing and
824          * the clock forward checks in the enqueue path:
825          */
826         timer_clear_idle();
827 
828         calc_load_nohz_stop();
829         touch_softlockup_watchdog_sched();
830         /*
831          * Cancel the scheduled timer and restore the tick
832          */
833         ts->tick_stopped  = 0;
834         ts->idle_exittime = now;
835 
836         tick_nohz_restart(ts, now);
837 }
838 
839 static void tick_nohz_full_update_tick(struct tick_sched *ts)
840 {
841 #ifdef CONFIG_NO_HZ_FULL
842         int cpu = smp_processor_id();
843 
844         if (!tick_nohz_full_cpu(cpu))
845                 return;
846 
847         if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE)
848                 return;
849 
850         if (can_stop_full_tick(cpu, ts))
851                 tick_nohz_stop_sched_tick(ts, ktime_get(), cpu);
852         else if (ts->tick_stopped)
853                 tick_nohz_restart_sched_tick(ts, ktime_get());
854 #endif
855 }
856 
857 static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
858 {
859         /*
860          * If this CPU is offline and it is the one which updates
861          * jiffies, then give up the assignment and let it be taken by
862          * the CPU which runs the tick timer next. If we don't drop
863          * this here the jiffies might be stale and do_timer() never
864          * invoked.
865          */
866         if (unlikely(!cpu_online(cpu))) {
867                 if (cpu == tick_do_timer_cpu)
868                         tick_do_timer_cpu = TICK_DO_TIMER_NONE;
869                 /*
870                  * Make sure the CPU doesn't get fooled by obsolete tick
871                  * deadline if it comes back online later.
872                  */
873                 ts->next_tick = 0;
874                 return false;
875         }
876 
877         if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) {
878                 ts->sleep_length = NSEC_PER_SEC / HZ;
879                 return false;
880         }
881 
882         if (need_resched())
883                 return false;
884 
885         if (unlikely(local_softirq_pending() && cpu_online(cpu))) {
886                 static int ratelimit;
887 
888                 if (ratelimit < 10 &&
889                     (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) {
890                         pr_warn("NOHZ: local_softirq_pending %02x\n",
891                                 (unsigned int) local_softirq_pending());
892                         ratelimit++;
893                 }
894                 return false;
895         }
896 
897         if (tick_nohz_full_enabled()) {
898                 /*
899                  * Keep the tick alive to guarantee timekeeping progression
900                  * if there are full dynticks CPUs around
901                  */
902                 if (tick_do_timer_cpu == cpu)
903                         return false;
904                 /*
905                  * Boot safety: make sure the timekeeping duty has been
906                  * assigned before entering dyntick-idle mode,
907                  */
908                 if (tick_do_timer_cpu == TICK_DO_TIMER_NONE)
909                         return false;
910         }
911 
912         return true;
913 }
914 
915 static void __tick_nohz_idle_enter(struct tick_sched *ts)
916 {
917         ktime_t now, expires;
918         int cpu = smp_processor_id();
919 
920         now = tick_nohz_start_idle(ts);
921 
922         if (can_stop_idle_tick(cpu, ts)) {
923                 int was_stopped = ts->tick_stopped;
924 
925                 ts->idle_calls++;
926 
927                 expires = tick_nohz_stop_sched_tick(ts, now, cpu);
928                 if (expires > 0LL) {
929                         ts->idle_sleeps++;
930                         ts->idle_expires = expires;
931                 }
932 
933                 if (!was_stopped && ts->tick_stopped) {
934                         ts->idle_jiffies = ts->last_jiffies;
935                         nohz_balance_enter_idle(cpu);
936                 }
937         }
938 }
939 
940 /**
941  * tick_nohz_idle_enter - stop the idle tick from the idle task
942  *
943  * When the next event is more than a tick into the future, stop the idle tick
944  * Called when we start the idle loop.
945  *
946  * The arch is responsible of calling:
947  *
948  * - rcu_idle_enter() after its last use of RCU before the CPU is put
949  *  to sleep.
950  * - rcu_idle_exit() before the first use of RCU after the CPU is woken up.
951  */
952 void tick_nohz_idle_enter(void)
953 {
954         struct tick_sched *ts;
955 
956         lockdep_assert_irqs_enabled();
957         /*
958          * Update the idle state in the scheduler domain hierarchy
959          * when tick_nohz_stop_sched_tick() is called from the idle loop.
960          * State will be updated to busy during the first busy tick after
961          * exiting idle.
962          */
963         set_cpu_sd_state_idle();
964 
965         local_irq_disable();
966 
967         ts = this_cpu_ptr(&tick_cpu_sched);
968         ts->inidle = 1;
969         __tick_nohz_idle_enter(ts);
970 
971         local_irq_enable();
972 }
973 
974 /**
975  * tick_nohz_irq_exit - update next tick event from interrupt exit
976  *
977  * When an interrupt fires while we are idle and it doesn't cause
978  * a reschedule, it may still add, modify or delete a timer, enqueue
979  * an RCU callback, etc...
980  * So we need to re-calculate and reprogram the next tick event.
981  */
982 void tick_nohz_irq_exit(void)
983 {
984         struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
985 
986         if (ts->inidle)
987                 __tick_nohz_idle_enter(ts);
988         else
989                 tick_nohz_full_update_tick(ts);
990 }
991 
992 /**
993  * tick_nohz_get_sleep_length - return the length of the current sleep
994  *
995  * Called from power state control code with interrupts disabled
996  */
997 ktime_t tick_nohz_get_sleep_length(void)
998 {
999         struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1000 
1001         return ts->sleep_length;
1002 }
1003 
1004 /**
1005  * tick_nohz_get_idle_calls_cpu - return the current idle calls counter value
1006  * for a particular CPU.
1007  *
1008  * Called from the schedutil frequency scaling governor in scheduler context.
1009  */
1010 unsigned long tick_nohz_get_idle_calls_cpu(int cpu)
1011 {
1012         struct tick_sched *ts = tick_get_tick_sched(cpu);
1013 
1014         return ts->idle_calls;
1015 }
1016 
1017 /**
1018  * tick_nohz_get_idle_calls - return the current idle calls counter value
1019  *
1020  * Called from the schedutil frequency scaling governor in scheduler context.
1021  */
1022 unsigned long tick_nohz_get_idle_calls(void)
1023 {
1024         struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1025 
1026         return ts->idle_calls;
1027 }
1028 
1029 static void tick_nohz_account_idle_ticks(struct tick_sched *ts)
1030 {
1031 #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
1032         unsigned long ticks;
1033 
1034         if (vtime_accounting_cpu_enabled())
1035                 return;
1036         /*
1037          * We stopped the tick in idle. Update process times would miss the
1038          * time we slept as update_process_times does only a 1 tick
1039          * accounting. Enforce that this is accounted to idle !
1040          */
1041         ticks = jiffies - ts->idle_jiffies;
1042         /*
1043          * We might be one off. Do not randomly account a huge number of ticks!
1044          */
1045         if (ticks && ticks < LONG_MAX)
1046                 account_idle_ticks(ticks);
1047 #endif
1048 }
1049 
1050 /**
1051  * tick_nohz_idle_exit - restart the idle tick from the idle task
1052  *
1053  * Restart the idle tick when the CPU is woken up from idle
1054  * This also exit the RCU extended quiescent state. The CPU
1055  * can use RCU again after this function is called.
1056  */
1057 void tick_nohz_idle_exit(void)
1058 {
1059         struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1060         ktime_t now;
1061 
1062         local_irq_disable();
1063 
1064         WARN_ON_ONCE(!ts->inidle);
1065 
1066         ts->inidle = 0;
1067 
1068         if (ts->idle_active || ts->tick_stopped)
1069                 now = ktime_get();
1070 
1071         if (ts->idle_active)
1072                 tick_nohz_stop_idle(ts, now);
1073 
1074         if (ts->tick_stopped) {
1075                 tick_nohz_restart_sched_tick(ts, now);
1076                 tick_nohz_account_idle_ticks(ts);
1077         }
1078 
1079         local_irq_enable();
1080 }
1081 
1082 /*
1083  * The nohz low res interrupt handler
1084  */
1085 static void tick_nohz_handler(struct clock_event_device *dev)
1086 {
1087         struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1088         struct pt_regs *regs = get_irq_regs();
1089         ktime_t now = ktime_get();
1090 
1091         dev->next_event = KTIME_MAX;
1092 
1093         tick_sched_do_timer(now);
1094         tick_sched_handle(ts, regs);
1095 
1096         /* No need to reprogram if we are running tickless  */
1097         if (unlikely(ts->tick_stopped))
1098                 return;
1099 
1100         hrtimer_forward(&ts->sched_timer, now, tick_period);
1101         tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
1102 }
1103 
1104 static inline void tick_nohz_activate(struct tick_sched *ts, int mode)
1105 {
1106         if (!tick_nohz_enabled)
1107                 return;
1108         ts->nohz_mode = mode;
1109         /* One update is enough */
1110         if (!test_and_set_bit(0, &tick_nohz_active))
1111                 timers_update_nohz();
1112 }
1113 
1114 /**
1115  * tick_nohz_switch_to_nohz - switch to nohz mode
1116  */
1117 static void tick_nohz_switch_to_nohz(void)
1118 {
1119         struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1120         ktime_t next;
1121 
1122         if (!tick_nohz_enabled)
1123                 return;
1124 
1125         if (tick_switch_to_oneshot(tick_nohz_handler))
1126                 return;
1127 
1128         /*
1129          * Recycle the hrtimer in ts, so we can share the
1130          * hrtimer_forward with the highres code.
1131          */
1132         hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1133         /* Get the next period */
1134         next = tick_init_jiffy_update();
1135 
1136         hrtimer_set_expires(&ts->sched_timer, next);
1137         hrtimer_forward_now(&ts->sched_timer, tick_period);
1138         tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
1139         tick_nohz_activate(ts, NOHZ_MODE_LOWRES);
1140 }
1141 
1142 static inline void tick_nohz_irq_enter(void)
1143 {
1144         struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1145         ktime_t now;
1146 
1147         if (!ts->idle_active && !ts->tick_stopped)
1148                 return;
1149         now = ktime_get();
1150         if (ts->idle_active)
1151                 tick_nohz_stop_idle(ts, now);
1152         if (ts->tick_stopped)
1153                 tick_nohz_update_jiffies(now);
1154 }
1155 
1156 #else
1157 
1158 static inline void tick_nohz_switch_to_nohz(void) { }
1159 static inline void tick_nohz_irq_enter(void) { }
1160 static inline void tick_nohz_activate(struct tick_sched *ts, int mode) { }
1161 
1162 #endif /* CONFIG_NO_HZ_COMMON */
1163 
1164 /*
1165  * Called from irq_enter to notify about the possible interruption of idle()
1166  */
1167 void tick_irq_enter(void)
1168 {
1169         tick_check_oneshot_broadcast_this_cpu();
1170         tick_nohz_irq_enter();
1171 }
1172 
1173 /*
1174  * High resolution timer specific code
1175  */
1176 #ifdef CONFIG_HIGH_RES_TIMERS
1177 /*
1178  * We rearm the timer until we get disabled by the idle code.
1179  * Called with interrupts disabled.
1180  */
1181 static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
1182 {
1183         struct tick_sched *ts =
1184                 container_of(timer, struct tick_sched, sched_timer);
1185         struct pt_regs *regs = get_irq_regs();
1186         ktime_t now = ktime_get();
1187 
1188         tick_sched_do_timer(now);
1189 
1190         /*
1191          * Do not call, when we are not in irq context and have
1192          * no valid regs pointer
1193          */
1194         if (regs)
1195                 tick_sched_handle(ts, regs);
1196         else
1197                 ts->next_tick = 0;
1198 
1199         /* No need to reprogram if we are in idle or full dynticks mode */
1200         if (unlikely(ts->tick_stopped))
1201                 return HRTIMER_NORESTART;
1202 
1203         hrtimer_forward(timer, now, tick_period);
1204 
1205         return HRTIMER_RESTART;
1206 }
1207 
1208 static int sched_skew_tick;
1209 
1210 static int __init skew_tick(char *str)
1211 {
1212         get_option(&str, &sched_skew_tick);
1213 
1214         return 0;
1215 }
1216 early_param("skew_tick", skew_tick);
1217 
1218 /**
1219  * tick_setup_sched_timer - setup the tick emulation timer
1220  */
1221 void tick_setup_sched_timer(void)
1222 {
1223         struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1224         ktime_t now = ktime_get();
1225 
1226         /*
1227          * Emulate tick processing via per-CPU hrtimers:
1228          */
1229         hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1230         ts->sched_timer.function = tick_sched_timer;
1231 
1232         /* Get the next period (per-CPU) */
1233         hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
1234 
1235         /* Offset the tick to avert jiffies_lock contention. */
1236         if (sched_skew_tick) {
1237                 u64 offset = ktime_to_ns(tick_period) >> 1;
1238                 do_div(offset, num_possible_cpus());
1239                 offset *= smp_processor_id();
1240                 hrtimer_add_expires_ns(&ts->sched_timer, offset);
1241         }
1242 
1243         hrtimer_forward(&ts->sched_timer, now, tick_period);
1244         hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED);
1245         tick_nohz_activate(ts, NOHZ_MODE_HIGHRES);
1246 }
1247 #endif /* HIGH_RES_TIMERS */
1248 
1249 #if defined CONFIG_NO_HZ_COMMON || defined CONFIG_HIGH_RES_TIMERS
1250 void tick_cancel_sched_timer(int cpu)
1251 {
1252         struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
1253 
1254 # ifdef CONFIG_HIGH_RES_TIMERS
1255         if (ts->sched_timer.base)
1256                 hrtimer_cancel(&ts->sched_timer);
1257 # endif
1258 
1259         memset(ts, 0, sizeof(*ts));
1260 }
1261 #endif
1262 
1263 /**
1264  * Async notification about clocksource changes
1265  */
1266 void tick_clock_notify(void)
1267 {
1268         int cpu;
1269 
1270         for_each_possible_cpu(cpu)
1271                 set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks);
1272 }
1273 
1274 /*
1275  * Async notification about clock event changes
1276  */
1277 void tick_oneshot_notify(void)
1278 {
1279         struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1280 
1281         set_bit(0, &ts->check_clocks);
1282 }
1283 
1284 /**
1285  * Check, if a change happened, which makes oneshot possible.
1286  *
1287  * Called cyclic from the hrtimer softirq (driven by the timer
1288  * softirq) allow_nohz signals, that we can switch into low-res nohz
1289  * mode, because high resolution timers are disabled (either compile
1290  * or runtime). Called with interrupts disabled.
1291  */
1292 int tick_check_oneshot_change(int allow_nohz)
1293 {
1294         struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1295 
1296         if (!test_and_clear_bit(0, &ts->check_clocks))
1297                 return 0;
1298 
1299         if (ts->nohz_mode != NOHZ_MODE_INACTIVE)
1300                 return 0;
1301 
1302         if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available())
1303                 return 0;
1304 
1305         if (!allow_nohz)
1306                 return 1;
1307 
1308         tick_nohz_switch_to_nohz();
1309         return 0;
1310 }
1311 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp