~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/time/hrtimer.c

Version: ~ [ linux-5.19-rc8 ] ~ [ linux-5.18.14 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.57 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.133 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.207 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.253 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.289 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.324 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  *  Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
  4  *  Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
  5  *  Copyright(C) 2006-2007  Timesys Corp., Thomas Gleixner
  6  *
  7  *  High-resolution kernel timers
  8  *
  9  *  In contrast to the low-resolution timeout API, aka timer wheel,
 10  *  hrtimers provide finer resolution and accuracy depending on system
 11  *  configuration and capabilities.
 12  *
 13  *  Started by: Thomas Gleixner and Ingo Molnar
 14  *
 15  *  Credits:
 16  *      Based on the original timer wheel code
 17  *
 18  *      Help, testing, suggestions, bugfixes, improvements were
 19  *      provided by:
 20  *
 21  *      George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel
 22  *      et. al.
 23  */
 24 
 25 #include <linux/cpu.h>
 26 #include <linux/export.h>
 27 #include <linux/percpu.h>
 28 #include <linux/hrtimer.h>
 29 #include <linux/notifier.h>
 30 #include <linux/syscalls.h>
 31 #include <linux/interrupt.h>
 32 #include <linux/tick.h>
 33 #include <linux/seq_file.h>
 34 #include <linux/err.h>
 35 #include <linux/debugobjects.h>
 36 #include <linux/sched/signal.h>
 37 #include <linux/sched/sysctl.h>
 38 #include <linux/sched/rt.h>
 39 #include <linux/sched/deadline.h>
 40 #include <linux/sched/nohz.h>
 41 #include <linux/sched/debug.h>
 42 #include <linux/timer.h>
 43 #include <linux/freezer.h>
 44 #include <linux/compat.h>
 45 
 46 #include <linux/uaccess.h>
 47 
 48 #include <trace/events/timer.h>
 49 
 50 #include "tick-internal.h"
 51 
 52 /*
 53  * Masks for selecting the soft and hard context timers from
 54  * cpu_base->active
 55  */
 56 #define MASK_SHIFT              (HRTIMER_BASE_MONOTONIC_SOFT)
 57 #define HRTIMER_ACTIVE_HARD     ((1U << MASK_SHIFT) - 1)
 58 #define HRTIMER_ACTIVE_SOFT     (HRTIMER_ACTIVE_HARD << MASK_SHIFT)
 59 #define HRTIMER_ACTIVE_ALL      (HRTIMER_ACTIVE_SOFT | HRTIMER_ACTIVE_HARD)
 60 
 61 /*
 62  * The timer bases:
 63  *
 64  * There are more clockids than hrtimer bases. Thus, we index
 65  * into the timer bases by the hrtimer_base_type enum. When trying
 66  * to reach a base using a clockid, hrtimer_clockid_to_base()
 67  * is used to convert from clockid to the proper hrtimer_base_type.
 68  */
 69 DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
 70 {
 71         .lock = __RAW_SPIN_LOCK_UNLOCKED(hrtimer_bases.lock),
 72         .clock_base =
 73         {
 74                 {
 75                         .index = HRTIMER_BASE_MONOTONIC,
 76                         .clockid = CLOCK_MONOTONIC,
 77                         .get_time = &ktime_get,
 78                 },
 79                 {
 80                         .index = HRTIMER_BASE_REALTIME,
 81                         .clockid = CLOCK_REALTIME,
 82                         .get_time = &ktime_get_real,
 83                 },
 84                 {
 85                         .index = HRTIMER_BASE_BOOTTIME,
 86                         .clockid = CLOCK_BOOTTIME,
 87                         .get_time = &ktime_get_boottime,
 88                 },
 89                 {
 90                         .index = HRTIMER_BASE_TAI,
 91                         .clockid = CLOCK_TAI,
 92                         .get_time = &ktime_get_clocktai,
 93                 },
 94                 {
 95                         .index = HRTIMER_BASE_MONOTONIC_SOFT,
 96                         .clockid = CLOCK_MONOTONIC,
 97                         .get_time = &ktime_get,
 98                 },
 99                 {
100                         .index = HRTIMER_BASE_REALTIME_SOFT,
101                         .clockid = CLOCK_REALTIME,
102                         .get_time = &ktime_get_real,
103                 },
104                 {
105                         .index = HRTIMER_BASE_BOOTTIME_SOFT,
106                         .clockid = CLOCK_BOOTTIME,
107                         .get_time = &ktime_get_boottime,
108                 },
109                 {
110                         .index = HRTIMER_BASE_TAI_SOFT,
111                         .clockid = CLOCK_TAI,
112                         .get_time = &ktime_get_clocktai,
113                 },
114         }
115 };
116 
117 static const int hrtimer_clock_to_base_table[MAX_CLOCKS] = {
118         /* Make sure we catch unsupported clockids */
119         [0 ... MAX_CLOCKS - 1]  = HRTIMER_MAX_CLOCK_BASES,
120 
121         [CLOCK_REALTIME]        = HRTIMER_BASE_REALTIME,
122         [CLOCK_MONOTONIC]       = HRTIMER_BASE_MONOTONIC,
123         [CLOCK_BOOTTIME]        = HRTIMER_BASE_BOOTTIME,
124         [CLOCK_TAI]             = HRTIMER_BASE_TAI,
125 };
126 
127 /*
128  * Functions and macros which are different for UP/SMP systems are kept in a
129  * single place
130  */
131 #ifdef CONFIG_SMP
132 
133 /*
134  * We require the migration_base for lock_hrtimer_base()/switch_hrtimer_base()
135  * such that hrtimer_callback_running() can unconditionally dereference
136  * timer->base->cpu_base
137  */
138 static struct hrtimer_cpu_base migration_cpu_base = {
139         .clock_base = { { .cpu_base = &migration_cpu_base, }, },
140 };
141 
142 #define migration_base  migration_cpu_base.clock_base[0]
143 
144 /*
145  * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock
146  * means that all timers which are tied to this base via timer->base are
147  * locked, and the base itself is locked too.
148  *
149  * So __run_timers/migrate_timers can safely modify all timers which could
150  * be found on the lists/queues.
151  *
152  * When the timer's base is locked, and the timer removed from list, it is
153  * possible to set timer->base = &migration_base and drop the lock: the timer
154  * remains locked.
155  */
156 static
157 struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
158                                              unsigned long *flags)
159 {
160         struct hrtimer_clock_base *base;
161 
162         for (;;) {
163                 base = timer->base;
164                 if (likely(base != &migration_base)) {
165                         raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
166                         if (likely(base == timer->base))
167                                 return base;
168                         /* The timer has migrated to another CPU: */
169                         raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags);
170                 }
171                 cpu_relax();
172         }
173 }
174 
175 /*
176  * We do not migrate the timer when it is expiring before the next
177  * event on the target cpu. When high resolution is enabled, we cannot
178  * reprogram the target cpu hardware and we would cause it to fire
179  * late. To keep it simple, we handle the high resolution enabled and
180  * disabled case similar.
181  *
182  * Called with cpu_base->lock of target cpu held.
183  */
184 static int
185 hrtimer_check_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base)
186 {
187         ktime_t expires;
188 
189         expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset);
190         return expires < new_base->cpu_base->expires_next;
191 }
192 
193 static inline
194 struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base,
195                                          int pinned)
196 {
197 #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
198         if (static_branch_likely(&timers_migration_enabled) && !pinned)
199                 return &per_cpu(hrtimer_bases, get_nohz_timer_target());
200 #endif
201         return base;
202 }
203 
204 /*
205  * We switch the timer base to a power-optimized selected CPU target,
206  * if:
207  *      - NO_HZ_COMMON is enabled
208  *      - timer migration is enabled
209  *      - the timer callback is not running
210  *      - the timer is not the first expiring timer on the new target
211  *
212  * If one of the above requirements is not fulfilled we move the timer
213  * to the current CPU or leave it on the previously assigned CPU if
214  * the timer callback is currently running.
215  */
216 static inline struct hrtimer_clock_base *
217 switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
218                     int pinned)
219 {
220         struct hrtimer_cpu_base *new_cpu_base, *this_cpu_base;
221         struct hrtimer_clock_base *new_base;
222         int basenum = base->index;
223 
224         this_cpu_base = this_cpu_ptr(&hrtimer_bases);
225         new_cpu_base = get_target_base(this_cpu_base, pinned);
226 again:
227         new_base = &new_cpu_base->clock_base[basenum];
228 
229         if (base != new_base) {
230                 /*
231                  * We are trying to move timer to new_base.
232                  * However we can't change timer's base while it is running,
233                  * so we keep it on the same CPU. No hassle vs. reprogramming
234                  * the event source in the high resolution case. The softirq
235                  * code will take care of this when the timer function has
236                  * completed. There is no conflict as we hold the lock until
237                  * the timer is enqueued.
238                  */
239                 if (unlikely(hrtimer_callback_running(timer)))
240                         return base;
241 
242                 /* See the comment in lock_hrtimer_base() */
243                 timer->base = &migration_base;
244                 raw_spin_unlock(&base->cpu_base->lock);
245                 raw_spin_lock(&new_base->cpu_base->lock);
246 
247                 if (new_cpu_base != this_cpu_base &&
248                     hrtimer_check_target(timer, new_base)) {
249                         raw_spin_unlock(&new_base->cpu_base->lock);
250                         raw_spin_lock(&base->cpu_base->lock);
251                         new_cpu_base = this_cpu_base;
252                         timer->base = base;
253                         goto again;
254                 }
255                 timer->base = new_base;
256         } else {
257                 if (new_cpu_base != this_cpu_base &&
258                     hrtimer_check_target(timer, new_base)) {
259                         new_cpu_base = this_cpu_base;
260                         goto again;
261                 }
262         }
263         return new_base;
264 }
265 
266 #else /* CONFIG_SMP */
267 
268 static inline struct hrtimer_clock_base *
269 lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
270 {
271         struct hrtimer_clock_base *base = timer->base;
272 
273         raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
274 
275         return base;
276 }
277 
278 # define switch_hrtimer_base(t, b, p)   (b)
279 
280 #endif  /* !CONFIG_SMP */
281 
282 /*
283  * Functions for the union type storage format of ktime_t which are
284  * too large for inlining:
285  */
286 #if BITS_PER_LONG < 64
287 /*
288  * Divide a ktime value by a nanosecond value
289  */
290 s64 __ktime_divns(const ktime_t kt, s64 div)
291 {
292         int sft = 0;
293         s64 dclc;
294         u64 tmp;
295 
296         dclc = ktime_to_ns(kt);
297         tmp = dclc < 0 ? -dclc : dclc;
298 
299         /* Make sure the divisor is less than 2^32: */
300         while (div >> 32) {
301                 sft++;
302                 div >>= 1;
303         }
304         tmp >>= sft;
305         do_div(tmp, (unsigned long) div);
306         return dclc < 0 ? -tmp : tmp;
307 }
308 EXPORT_SYMBOL_GPL(__ktime_divns);
309 #endif /* BITS_PER_LONG >= 64 */
310 
311 /*
312  * Add two ktime values and do a safety check for overflow:
313  */
314 ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs)
315 {
316         ktime_t res = ktime_add_unsafe(lhs, rhs);
317 
318         /*
319          * We use KTIME_SEC_MAX here, the maximum timeout which we can
320          * return to user space in a timespec:
321          */
322         if (res < 0 || res < lhs || res < rhs)
323                 res = ktime_set(KTIME_SEC_MAX, 0);
324 
325         return res;
326 }
327 
328 EXPORT_SYMBOL_GPL(ktime_add_safe);
329 
330 #ifdef CONFIG_DEBUG_OBJECTS_TIMERS
331 
332 static struct debug_obj_descr hrtimer_debug_descr;
333 
334 static void *hrtimer_debug_hint(void *addr)
335 {
336         return ((struct hrtimer *) addr)->function;
337 }
338 
339 /*
340  * fixup_init is called when:
341  * - an active object is initialized
342  */
343 static bool hrtimer_fixup_init(void *addr, enum debug_obj_state state)
344 {
345         struct hrtimer *timer = addr;
346 
347         switch (state) {
348         case ODEBUG_STATE_ACTIVE:
349                 hrtimer_cancel(timer);
350                 debug_object_init(timer, &hrtimer_debug_descr);
351                 return true;
352         default:
353                 return false;
354         }
355 }
356 
357 /*
358  * fixup_activate is called when:
359  * - an active object is activated
360  * - an unknown non-static object is activated
361  */
362 static bool hrtimer_fixup_activate(void *addr, enum debug_obj_state state)
363 {
364         switch (state) {
365         case ODEBUG_STATE_ACTIVE:
366                 WARN_ON(1);
367 
368         default:
369                 return false;
370         }
371 }
372 
373 /*
374  * fixup_free is called when:
375  * - an active object is freed
376  */
377 static bool hrtimer_fixup_free(void *addr, enum debug_obj_state state)
378 {
379         struct hrtimer *timer = addr;
380 
381         switch (state) {
382         case ODEBUG_STATE_ACTIVE:
383                 hrtimer_cancel(timer);
384                 debug_object_free(timer, &hrtimer_debug_descr);
385                 return true;
386         default:
387                 return false;
388         }
389 }
390 
391 static struct debug_obj_descr hrtimer_debug_descr = {
392         .name           = "hrtimer",
393         .debug_hint     = hrtimer_debug_hint,
394         .fixup_init     = hrtimer_fixup_init,
395         .fixup_activate = hrtimer_fixup_activate,
396         .fixup_free     = hrtimer_fixup_free,
397 };
398 
399 static inline void debug_hrtimer_init(struct hrtimer *timer)
400 {
401         debug_object_init(timer, &hrtimer_debug_descr);
402 }
403 
404 static inline void debug_hrtimer_activate(struct hrtimer *timer,
405                                           enum hrtimer_mode mode)
406 {
407         debug_object_activate(timer, &hrtimer_debug_descr);
408 }
409 
410 static inline void debug_hrtimer_deactivate(struct hrtimer *timer)
411 {
412         debug_object_deactivate(timer, &hrtimer_debug_descr);
413 }
414 
415 static inline void debug_hrtimer_free(struct hrtimer *timer)
416 {
417         debug_object_free(timer, &hrtimer_debug_descr);
418 }
419 
420 static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
421                            enum hrtimer_mode mode);
422 
423 void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id,
424                            enum hrtimer_mode mode)
425 {
426         debug_object_init_on_stack(timer, &hrtimer_debug_descr);
427         __hrtimer_init(timer, clock_id, mode);
428 }
429 EXPORT_SYMBOL_GPL(hrtimer_init_on_stack);
430 
431 void destroy_hrtimer_on_stack(struct hrtimer *timer)
432 {
433         debug_object_free(timer, &hrtimer_debug_descr);
434 }
435 EXPORT_SYMBOL_GPL(destroy_hrtimer_on_stack);
436 
437 #else
438 
439 static inline void debug_hrtimer_init(struct hrtimer *timer) { }
440 static inline void debug_hrtimer_activate(struct hrtimer *timer,
441                                           enum hrtimer_mode mode) { }
442 static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }
443 #endif
444 
445 static inline void
446 debug_init(struct hrtimer *timer, clockid_t clockid,
447            enum hrtimer_mode mode)
448 {
449         debug_hrtimer_init(timer);
450         trace_hrtimer_init(timer, clockid, mode);
451 }
452 
453 static inline void debug_activate(struct hrtimer *timer,
454                                   enum hrtimer_mode mode)
455 {
456         debug_hrtimer_activate(timer, mode);
457         trace_hrtimer_start(timer, mode);
458 }
459 
460 static inline void debug_deactivate(struct hrtimer *timer)
461 {
462         debug_hrtimer_deactivate(timer);
463         trace_hrtimer_cancel(timer);
464 }
465 
466 static struct hrtimer_clock_base *
467 __next_base(struct hrtimer_cpu_base *cpu_base, unsigned int *active)
468 {
469         unsigned int idx;
470 
471         if (!*active)
472                 return NULL;
473 
474         idx = __ffs(*active);
475         *active &= ~(1U << idx);
476 
477         return &cpu_base->clock_base[idx];
478 }
479 
480 #define for_each_active_base(base, cpu_base, active)    \
481         while ((base = __next_base((cpu_base), &(active))))
482 
483 static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base,
484                                          const struct hrtimer *exclude,
485                                          unsigned int active,
486                                          ktime_t expires_next)
487 {
488         struct hrtimer_clock_base *base;
489         ktime_t expires;
490 
491         for_each_active_base(base, cpu_base, active) {
492                 struct timerqueue_node *next;
493                 struct hrtimer *timer;
494 
495                 next = timerqueue_getnext(&base->active);
496                 timer = container_of(next, struct hrtimer, node);
497                 if (timer == exclude) {
498                         /* Get to the next timer in the queue. */
499                         next = timerqueue_iterate_next(next);
500                         if (!next)
501                                 continue;
502 
503                         timer = container_of(next, struct hrtimer, node);
504                 }
505                 expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
506                 if (expires < expires_next) {
507                         expires_next = expires;
508 
509                         /* Skip cpu_base update if a timer is being excluded. */
510                         if (exclude)
511                                 continue;
512 
513                         if (timer->is_soft)
514                                 cpu_base->softirq_next_timer = timer;
515                         else
516                                 cpu_base->next_timer = timer;
517                 }
518         }
519         /*
520          * clock_was_set() might have changed base->offset of any of
521          * the clock bases so the result might be negative. Fix it up
522          * to prevent a false positive in clockevents_program_event().
523          */
524         if (expires_next < 0)
525                 expires_next = 0;
526         return expires_next;
527 }
528 
529 /*
530  * Recomputes cpu_base::*next_timer and returns the earliest expires_next but
531  * does not set cpu_base::*expires_next, that is done by hrtimer_reprogram.
532  *
533  * When a softirq is pending, we can ignore the HRTIMER_ACTIVE_SOFT bases,
534  * those timers will get run whenever the softirq gets handled, at the end of
535  * hrtimer_run_softirq(), hrtimer_update_softirq_timer() will re-add these bases.
536  *
537  * Therefore softirq values are those from the HRTIMER_ACTIVE_SOFT clock bases.
538  * The !softirq values are the minima across HRTIMER_ACTIVE_ALL, unless an actual
539  * softirq is pending, in which case they're the minima of HRTIMER_ACTIVE_HARD.
540  *
541  * @active_mask must be one of:
542  *  - HRTIMER_ACTIVE_ALL,
543  *  - HRTIMER_ACTIVE_SOFT, or
544  *  - HRTIMER_ACTIVE_HARD.
545  */
546 static ktime_t
547 __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask)
548 {
549         unsigned int active;
550         struct hrtimer *next_timer = NULL;
551         ktime_t expires_next = KTIME_MAX;
552 
553         if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) {
554                 active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
555                 cpu_base->softirq_next_timer = NULL;
556                 expires_next = __hrtimer_next_event_base(cpu_base, NULL,
557                                                          active, KTIME_MAX);
558 
559                 next_timer = cpu_base->softirq_next_timer;
560         }
561 
562         if (active_mask & HRTIMER_ACTIVE_HARD) {
563                 active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
564                 cpu_base->next_timer = next_timer;
565                 expires_next = __hrtimer_next_event_base(cpu_base, NULL, active,
566                                                          expires_next);
567         }
568 
569         return expires_next;
570 }
571 
572 static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
573 {
574         ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset;
575         ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset;
576         ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset;
577 
578         ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq,
579                                             offs_real, offs_boot, offs_tai);
580 
581         base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real;
582         base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot;
583         base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai;
584 
585         return now;
586 }
587 
588 /*
589  * Is the high resolution mode active ?
590  */
591 static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)
592 {
593         return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
594                 cpu_base->hres_active : 0;
595 }
596 
597 static inline int hrtimer_hres_active(void)
598 {
599         return __hrtimer_hres_active(this_cpu_ptr(&hrtimer_bases));
600 }
601 
602 /*
603  * Reprogram the event source with checking both queues for the
604  * next event
605  * Called with interrupts disabled and base->lock held
606  */
607 static void
608 hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)
609 {
610         ktime_t expires_next;
611 
612         /*
613          * Find the current next expiration time.
614          */
615         expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
616 
617         if (cpu_base->next_timer && cpu_base->next_timer->is_soft) {
618                 /*
619                  * When the softirq is activated, hrtimer has to be
620                  * programmed with the first hard hrtimer because soft
621                  * timer interrupt could occur too late.
622                  */
623                 if (cpu_base->softirq_activated)
624                         expires_next = __hrtimer_get_next_event(cpu_base,
625                                                                 HRTIMER_ACTIVE_HARD);
626                 else
627                         cpu_base->softirq_expires_next = expires_next;
628         }
629 
630         if (skip_equal && expires_next == cpu_base->expires_next)
631                 return;
632 
633         cpu_base->expires_next = expires_next;
634 
635         /*
636          * If hres is not active, hardware does not have to be
637          * reprogrammed yet.
638          *
639          * If a hang was detected in the last timer interrupt then we
640          * leave the hang delay active in the hardware. We want the
641          * system to make progress. That also prevents the following
642          * scenario:
643          * T1 expires 50ms from now
644          * T2 expires 5s from now
645          *
646          * T1 is removed, so this code is called and would reprogram
647          * the hardware to 5s from now. Any hrtimer_start after that
648          * will not reprogram the hardware due to hang_detected being
649          * set. So we'd effectivly block all timers until the T2 event
650          * fires.
651          */
652         if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)
653                 return;
654 
655         tick_program_event(cpu_base->expires_next, 1);
656 }
657 
658 /* High resolution timer related functions */
659 #ifdef CONFIG_HIGH_RES_TIMERS
660 
661 /*
662  * High resolution timer enabled ?
663  */
664 static bool hrtimer_hres_enabled __read_mostly  = true;
665 unsigned int hrtimer_resolution __read_mostly = LOW_RES_NSEC;
666 EXPORT_SYMBOL_GPL(hrtimer_resolution);
667 
668 /*
669  * Enable / Disable high resolution mode
670  */
671 static int __init setup_hrtimer_hres(char *str)
672 {
673         return (kstrtobool(str, &hrtimer_hres_enabled) == 0);
674 }
675 
676 __setup("highres=", setup_hrtimer_hres);
677 
678 /*
679  * hrtimer_high_res_enabled - query, if the highres mode is enabled
680  */
681 static inline int hrtimer_is_hres_enabled(void)
682 {
683         return hrtimer_hres_enabled;
684 }
685 
686 /*
687  * Retrigger next event is called after clock was set
688  *
689  * Called with interrupts disabled via on_each_cpu()
690  */
691 static void retrigger_next_event(void *arg)
692 {
693         struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);
694 
695         if (!__hrtimer_hres_active(base))
696                 return;
697 
698         raw_spin_lock(&base->lock);
699         hrtimer_update_base(base);
700         hrtimer_force_reprogram(base, 0);
701         raw_spin_unlock(&base->lock);
702 }
703 
704 /*
705  * Switch to high resolution mode
706  */
707 static void hrtimer_switch_to_hres(void)
708 {
709         struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);
710 
711         if (tick_init_highres()) {
712                 pr_warn("Could not switch to high resolution mode on CPU %u\n",
713                         base->cpu);
714                 return;
715         }
716         base->hres_active = 1;
717         hrtimer_resolution = HIGH_RES_NSEC;
718 
719         tick_setup_sched_timer();
720         /* "Retrigger" the interrupt to get things going */
721         retrigger_next_event(NULL);
722 }
723 
724 static void clock_was_set_work(struct work_struct *work)
725 {
726         clock_was_set();
727 }
728 
729 static DECLARE_WORK(hrtimer_work, clock_was_set_work);
730 
731 /*
732  * Called from timekeeping and resume code to reprogram the hrtimer
733  * interrupt device on all cpus.
734  */
735 void clock_was_set_delayed(void)
736 {
737         schedule_work(&hrtimer_work);
738 }
739 
740 #else
741 
742 static inline int hrtimer_is_hres_enabled(void) { return 0; }
743 static inline void hrtimer_switch_to_hres(void) { }
744 static inline void retrigger_next_event(void *arg) { }
745 
746 #endif /* CONFIG_HIGH_RES_TIMERS */
747 
748 /*
749  * When a timer is enqueued and expires earlier than the already enqueued
750  * timers, we have to check, whether it expires earlier than the timer for
751  * which the clock event device was armed.
752  *
753  * Called with interrupts disabled and base->cpu_base.lock held
754  */
755 static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)
756 {
757         struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
758         struct hrtimer_clock_base *base = timer->base;
759         ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
760 
761         WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0);
762 
763         /*
764          * CLOCK_REALTIME timer might be requested with an absolute
765          * expiry time which is less than base->offset. Set it to 0.
766          */
767         if (expires < 0)
768                 expires = 0;
769 
770         if (timer->is_soft) {
771                 /*
772                  * soft hrtimer could be started on a remote CPU. In this
773                  * case softirq_expires_next needs to be updated on the
774                  * remote CPU. The soft hrtimer will not expire before the
775                  * first hard hrtimer on the remote CPU -
776                  * hrtimer_check_target() prevents this case.
777                  */
778                 struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base;
779 
780                 if (timer_cpu_base->softirq_activated)
781                         return;
782 
783                 if (!ktime_before(expires, timer_cpu_base->softirq_expires_next))
784                         return;
785 
786                 timer_cpu_base->softirq_next_timer = timer;
787                 timer_cpu_base->softirq_expires_next = expires;
788 
789                 if (!ktime_before(expires, timer_cpu_base->expires_next) ||
790                     !reprogram)
791                         return;
792         }
793 
794         /*
795          * If the timer is not on the current cpu, we cannot reprogram
796          * the other cpus clock event device.
797          */
798         if (base->cpu_base != cpu_base)
799                 return;
800 
801         /*
802          * If the hrtimer interrupt is running, then it will
803          * reevaluate the clock bases and reprogram the clock event
804          * device. The callbacks are always executed in hard interrupt
805          * context so we don't need an extra check for a running
806          * callback.
807          */
808         if (cpu_base->in_hrtirq)
809                 return;
810 
811         if (expires >= cpu_base->expires_next)
812                 return;
813 
814         /* Update the pointer to the next expiring timer */
815         cpu_base->next_timer = timer;
816         cpu_base->expires_next = expires;
817 
818         /*
819          * If hres is not active, hardware does not have to be
820          * programmed yet.
821          *
822          * If a hang was detected in the last timer interrupt then we
823          * do not schedule a timer which is earlier than the expiry
824          * which we enforced in the hang detection. We want the system
825          * to make progress.
826          */
827         if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)
828                 return;
829 
830         /*
831          * Program the timer hardware. We enforce the expiry for
832          * events which are already in the past.
833          */
834         tick_program_event(expires, 1);
835 }
836 
837 /*
838  * Clock realtime was set
839  *
840  * Change the offset of the realtime clock vs. the monotonic
841  * clock.
842  *
843  * We might have to reprogram the high resolution timer interrupt. On
844  * SMP we call the architecture specific code to retrigger _all_ high
845  * resolution timer interrupts. On UP we just disable interrupts and
846  * call the high resolution interrupt code.
847  */
848 void clock_was_set(void)
849 {
850 #ifdef CONFIG_HIGH_RES_TIMERS
851         /* Retrigger the CPU local events everywhere */
852         on_each_cpu(retrigger_next_event, NULL, 1);
853 #endif
854         timerfd_clock_was_set();
855 }
856 
857 /*
858  * During resume we might have to reprogram the high resolution timer
859  * interrupt on all online CPUs.  However, all other CPUs will be
860  * stopped with IRQs interrupts disabled so the clock_was_set() call
861  * must be deferred.
862  */
863 void hrtimers_resume(void)
864 {
865         lockdep_assert_irqs_disabled();
866         /* Retrigger on the local CPU */
867         retrigger_next_event(NULL);
868         /* And schedule a retrigger for all others */
869         clock_was_set_delayed();
870 }
871 
872 /*
873  * Counterpart to lock_hrtimer_base above:
874  */
875 static inline
876 void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
877 {
878         raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);
879 }
880 
881 /**
882  * hrtimer_forward - forward the timer expiry
883  * @timer:      hrtimer to forward
884  * @now:        forward past this time
885  * @interval:   the interval to forward
886  *
887  * Forward the timer expiry so it will expire in the future.
888  * Returns the number of overruns.
889  *
890  * Can be safely called from the callback function of @timer. If
891  * called from other contexts @timer must neither be enqueued nor
892  * running the callback and the caller needs to take care of
893  * serialization.
894  *
895  * Note: This only updates the timer expiry value and does not requeue
896  * the timer.
897  */
898 u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)
899 {
900         u64 orun = 1;
901         ktime_t delta;
902 
903         delta = ktime_sub(now, hrtimer_get_expires(timer));
904 
905         if (delta < 0)
906                 return 0;
907 
908         if (WARN_ON(timer->state & HRTIMER_STATE_ENQUEUED))
909                 return 0;
910 
911         if (interval < hrtimer_resolution)
912                 interval = hrtimer_resolution;
913 
914         if (unlikely(delta >= interval)) {
915                 s64 incr = ktime_to_ns(interval);
916 
917                 orun = ktime_divns(delta, incr);
918                 hrtimer_add_expires_ns(timer, incr * orun);
919                 if (hrtimer_get_expires_tv64(timer) > now)
920                         return orun;
921                 /*
922                  * This (and the ktime_add() below) is the
923                  * correction for exact:
924                  */
925                 orun++;
926         }
927         hrtimer_add_expires(timer, interval);
928 
929         return orun;
930 }
931 EXPORT_SYMBOL_GPL(hrtimer_forward);
932 
933 /*
934  * enqueue_hrtimer - internal function to (re)start a timer
935  *
936  * The timer is inserted in expiry order. Insertion into the
937  * red black tree is O(log(n)). Must hold the base lock.
938  *
939  * Returns 1 when the new timer is the leftmost timer in the tree.
940  */
941 static int enqueue_hrtimer(struct hrtimer *timer,
942                            struct hrtimer_clock_base *base,
943                            enum hrtimer_mode mode)
944 {
945         debug_activate(timer, mode);
946 
947         base->cpu_base->active_bases |= 1 << base->index;
948 
949         timer->state = HRTIMER_STATE_ENQUEUED;
950 
951         return timerqueue_add(&base->active, &timer->node);
952 }
953 
954 /*
955  * __remove_hrtimer - internal function to remove a timer
956  *
957  * Caller must hold the base lock.
958  *
959  * High resolution timer mode reprograms the clock event device when the
960  * timer is the one which expires next. The caller can disable this by setting
961  * reprogram to zero. This is useful, when the context does a reprogramming
962  * anyway (e.g. timer interrupt)
963  */
964 static void __remove_hrtimer(struct hrtimer *timer,
965                              struct hrtimer_clock_base *base,
966                              u8 newstate, int reprogram)
967 {
968         struct hrtimer_cpu_base *cpu_base = base->cpu_base;
969         u8 state = timer->state;
970 
971         timer->state = newstate;
972         if (!(state & HRTIMER_STATE_ENQUEUED))
973                 return;
974 
975         if (!timerqueue_del(&base->active, &timer->node))
976                 cpu_base->active_bases &= ~(1 << base->index);
977 
978         /*
979          * Note: If reprogram is false we do not update
980          * cpu_base->next_timer. This happens when we remove the first
981          * timer on a remote cpu. No harm as we never dereference
982          * cpu_base->next_timer. So the worst thing what can happen is
983          * an superflous call to hrtimer_force_reprogram() on the
984          * remote cpu later on if the same timer gets enqueued again.
985          */
986         if (reprogram && timer == cpu_base->next_timer)
987                 hrtimer_force_reprogram(cpu_base, 1);
988 }
989 
990 /*
991  * remove hrtimer, called with base lock held
992  */
993 static inline int
994 remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, bool restart)
995 {
996         if (hrtimer_is_queued(timer)) {
997                 u8 state = timer->state;
998                 int reprogram;
999 
1000                 /*
1001                  * Remove the timer and force reprogramming when high
1002                  * resolution mode is active and the timer is on the current
1003                  * CPU. If we remove a timer on another CPU, reprogramming is
1004                  * skipped. The interrupt event on this CPU is fired and
1005                  * reprogramming happens in the interrupt handler. This is a
1006                  * rare case and less expensive than a smp call.
1007                  */
1008                 debug_deactivate(timer);
1009                 reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);
1010 
1011                 if (!restart)
1012                         state = HRTIMER_STATE_INACTIVE;
1013 
1014                 __remove_hrtimer(timer, base, state, reprogram);
1015                 return 1;
1016         }
1017         return 0;
1018 }
1019 
1020 static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim,
1021                                             const enum hrtimer_mode mode)
1022 {
1023 #ifdef CONFIG_TIME_LOW_RES
1024         /*
1025          * CONFIG_TIME_LOW_RES indicates that the system has no way to return
1026          * granular time values. For relative timers we add hrtimer_resolution
1027          * (i.e. one jiffie) to prevent short timeouts.
1028          */
1029         timer->is_rel = mode & HRTIMER_MODE_REL;
1030         if (timer->is_rel)
1031                 tim = ktime_add_safe(tim, hrtimer_resolution);
1032 #endif
1033         return tim;
1034 }
1035 
1036 static void
1037 hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram)
1038 {
1039         ktime_t expires;
1040 
1041         /*
1042          * Find the next SOFT expiration.
1043          */
1044         expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
1045 
1046         /*
1047          * reprogramming needs to be triggered, even if the next soft
1048          * hrtimer expires at the same time than the next hard
1049          * hrtimer. cpu_base->softirq_expires_next needs to be updated!
1050          */
1051         if (expires == KTIME_MAX)
1052                 return;
1053 
1054         /*
1055          * cpu_base->*next_timer is recomputed by __hrtimer_get_next_event()
1056          * cpu_base->*expires_next is only set by hrtimer_reprogram()
1057          */
1058         hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram);
1059 }
1060 
1061 static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
1062                                     u64 delta_ns, const enum hrtimer_mode mode,
1063                                     struct hrtimer_clock_base *base)
1064 {
1065         struct hrtimer_clock_base *new_base;
1066 
1067         /* Remove an active timer from the queue: */
1068         remove_hrtimer(timer, base, true);
1069 
1070         if (mode & HRTIMER_MODE_REL)
1071                 tim = ktime_add_safe(tim, base->get_time());
1072 
1073         tim = hrtimer_update_lowres(timer, tim, mode);
1074 
1075         hrtimer_set_expires_range_ns(timer, tim, delta_ns);
1076 
1077         /* Switch the timer base, if necessary: */
1078         new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED);
1079 
1080         return enqueue_hrtimer(timer, new_base, mode);
1081 }
1082 
1083 /**
1084  * hrtimer_start_range_ns - (re)start an hrtimer
1085  * @timer:      the timer to be added
1086  * @tim:        expiry time
1087  * @delta_ns:   "slack" range for the timer
1088  * @mode:       timer mode: absolute (HRTIMER_MODE_ABS) or
1089  *              relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED);
1090  *              softirq based mode is considered for debug purpose only!
1091  */
1092 void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
1093                             u64 delta_ns, const enum hrtimer_mode mode)
1094 {
1095         struct hrtimer_clock_base *base;
1096         unsigned long flags;
1097 
1098         /*
1099          * Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft
1100          * match.
1101          */
1102         WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
1103 
1104         base = lock_hrtimer_base(timer, &flags);
1105 
1106         if (__hrtimer_start_range_ns(timer, tim, delta_ns, mode, base))
1107                 hrtimer_reprogram(timer, true);
1108 
1109         unlock_hrtimer_base(timer, &flags);
1110 }
1111 EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
1112 
1113 /**
1114  * hrtimer_try_to_cancel - try to deactivate a timer
1115  * @timer:      hrtimer to stop
1116  *
1117  * Returns:
1118  *  0 when the timer was not active
1119  *  1 when the timer was active
1120  * -1 when the timer is currently executing the callback function and
1121  *    cannot be stopped
1122  */
1123 int hrtimer_try_to_cancel(struct hrtimer *timer)
1124 {
1125         struct hrtimer_clock_base *base;
1126         unsigned long flags;
1127         int ret = -1;
1128 
1129         /*
1130          * Check lockless first. If the timer is not active (neither
1131          * enqueued nor running the callback, nothing to do here.  The
1132          * base lock does not serialize against a concurrent enqueue,
1133          * so we can avoid taking it.
1134          */
1135         if (!hrtimer_active(timer))
1136                 return 0;
1137 
1138         base = lock_hrtimer_base(timer, &flags);
1139 
1140         if (!hrtimer_callback_running(timer))
1141                 ret = remove_hrtimer(timer, base, false);
1142 
1143         unlock_hrtimer_base(timer, &flags);
1144 
1145         return ret;
1146 
1147 }
1148 EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel);
1149 
1150 /**
1151  * hrtimer_cancel - cancel a timer and wait for the handler to finish.
1152  * @timer:      the timer to be cancelled
1153  *
1154  * Returns:
1155  *  0 when the timer was not active
1156  *  1 when the timer was active
1157  */
1158 int hrtimer_cancel(struct hrtimer *timer)
1159 {
1160         for (;;) {
1161                 int ret = hrtimer_try_to_cancel(timer);
1162 
1163                 if (ret >= 0)
1164                         return ret;
1165                 cpu_relax();
1166         }
1167 }
1168 EXPORT_SYMBOL_GPL(hrtimer_cancel);
1169 
1170 /**
1171  * hrtimer_get_remaining - get remaining time for the timer
1172  * @timer:      the timer to read
1173  * @adjust:     adjust relative timers when CONFIG_TIME_LOW_RES=y
1174  */
1175 ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust)
1176 {
1177         unsigned long flags;
1178         ktime_t rem;
1179 
1180         lock_hrtimer_base(timer, &flags);
1181         if (IS_ENABLED(CONFIG_TIME_LOW_RES) && adjust)
1182                 rem = hrtimer_expires_remaining_adjusted(timer);
1183         else
1184                 rem = hrtimer_expires_remaining(timer);
1185         unlock_hrtimer_base(timer, &flags);
1186 
1187         return rem;
1188 }
1189 EXPORT_SYMBOL_GPL(__hrtimer_get_remaining);
1190 
1191 #ifdef CONFIG_NO_HZ_COMMON
1192 /**
1193  * hrtimer_get_next_event - get the time until next expiry event
1194  *
1195  * Returns the next expiry time or KTIME_MAX if no timer is pending.
1196  */
1197 u64 hrtimer_get_next_event(void)
1198 {
1199         struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1200         u64 expires = KTIME_MAX;
1201         unsigned long flags;
1202 
1203         raw_spin_lock_irqsave(&cpu_base->lock, flags);
1204 
1205         if (!__hrtimer_hres_active(cpu_base))
1206                 expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
1207 
1208         raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1209 
1210         return expires;
1211 }
1212 
1213 /**
1214  * hrtimer_next_event_without - time until next expiry event w/o one timer
1215  * @exclude:    timer to exclude
1216  *
1217  * Returns the next expiry time over all timers except for the @exclude one or
1218  * KTIME_MAX if none of them is pending.
1219  */
1220 u64 hrtimer_next_event_without(const struct hrtimer *exclude)
1221 {
1222         struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1223         u64 expires = KTIME_MAX;
1224         unsigned long flags;
1225 
1226         raw_spin_lock_irqsave(&cpu_base->lock, flags);
1227 
1228         if (__hrtimer_hres_active(cpu_base)) {
1229                 unsigned int active;
1230 
1231                 if (!cpu_base->softirq_activated) {
1232                         active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
1233                         expires = __hrtimer_next_event_base(cpu_base, exclude,
1234                                                             active, KTIME_MAX);
1235                 }
1236                 active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
1237                 expires = __hrtimer_next_event_base(cpu_base, exclude, active,
1238                                                     expires);
1239         }
1240 
1241         raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1242 
1243         return expires;
1244 }
1245 #endif
1246 
1247 static inline int hrtimer_clockid_to_base(clockid_t clock_id)
1248 {
1249         if (likely(clock_id < MAX_CLOCKS)) {
1250                 int base = hrtimer_clock_to_base_table[clock_id];
1251 
1252                 if (likely(base != HRTIMER_MAX_CLOCK_BASES))
1253                         return base;
1254         }
1255         WARN(1, "Invalid clockid %d. Using MONOTONIC\n", clock_id);
1256         return HRTIMER_BASE_MONOTONIC;
1257 }
1258 
1259 static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
1260                            enum hrtimer_mode mode)
1261 {
1262         bool softtimer = !!(mode & HRTIMER_MODE_SOFT);
1263         int base = softtimer ? HRTIMER_MAX_CLOCK_BASES / 2 : 0;
1264         struct hrtimer_cpu_base *cpu_base;
1265 
1266         memset(timer, 0, sizeof(struct hrtimer));
1267 
1268         cpu_base = raw_cpu_ptr(&hrtimer_bases);
1269 
1270         /*
1271          * POSIX magic: Relative CLOCK_REALTIME timers are not affected by
1272          * clock modifications, so they needs to become CLOCK_MONOTONIC to
1273          * ensure POSIX compliance.
1274          */
1275         if (clock_id == CLOCK_REALTIME && mode & HRTIMER_MODE_REL)
1276                 clock_id = CLOCK_MONOTONIC;
1277 
1278         base += hrtimer_clockid_to_base(clock_id);
1279         timer->is_soft = softtimer;
1280         timer->base = &cpu_base->clock_base[base];
1281         timerqueue_init(&timer->node);
1282 }
1283 
1284 /**
1285  * hrtimer_init - initialize a timer to the given clock
1286  * @timer:      the timer to be initialized
1287  * @clock_id:   the clock to be used
1288  * @mode:       The modes which are relevant for intitialization:
1289  *              HRTIMER_MODE_ABS, HRTIMER_MODE_REL, HRTIMER_MODE_ABS_SOFT,
1290  *              HRTIMER_MODE_REL_SOFT
1291  *
1292  *              The PINNED variants of the above can be handed in,
1293  *              but the PINNED bit is ignored as pinning happens
1294  *              when the hrtimer is started
1295  */
1296 void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
1297                   enum hrtimer_mode mode)
1298 {
1299         debug_init(timer, clock_id, mode);
1300         __hrtimer_init(timer, clock_id, mode);
1301 }
1302 EXPORT_SYMBOL_GPL(hrtimer_init);
1303 
1304 /*
1305  * A timer is active, when it is enqueued into the rbtree or the
1306  * callback function is running or it's in the state of being migrated
1307  * to another cpu.
1308  *
1309  * It is important for this function to not return a false negative.
1310  */
1311 bool hrtimer_active(const struct hrtimer *timer)
1312 {
1313         struct hrtimer_clock_base *base;
1314         unsigned int seq;
1315 
1316         do {
1317                 base = READ_ONCE(timer->base);
1318                 seq = raw_read_seqcount_begin(&base->seq);
1319 
1320                 if (timer->state != HRTIMER_STATE_INACTIVE ||
1321                     base->running == timer)
1322                         return true;
1323 
1324         } while (read_seqcount_retry(&base->seq, seq) ||
1325                  base != READ_ONCE(timer->base));
1326 
1327         return false;
1328 }
1329 EXPORT_SYMBOL_GPL(hrtimer_active);
1330 
1331 /*
1332  * The write_seqcount_barrier()s in __run_hrtimer() split the thing into 3
1333  * distinct sections:
1334  *
1335  *  - queued:   the timer is queued
1336  *  - callback: the timer is being ran
1337  *  - post:     the timer is inactive or (re)queued
1338  *
1339  * On the read side we ensure we observe timer->state and cpu_base->running
1340  * from the same section, if anything changed while we looked at it, we retry.
1341  * This includes timer->base changing because sequence numbers alone are
1342  * insufficient for that.
1343  *
1344  * The sequence numbers are required because otherwise we could still observe
1345  * a false negative if the read side got smeared over multiple consequtive
1346  * __run_hrtimer() invocations.
1347  */
1348 
1349 static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
1350                           struct hrtimer_clock_base *base,
1351                           struct hrtimer *timer, ktime_t *now,
1352                           unsigned long flags)
1353 {
1354         enum hrtimer_restart (*fn)(struct hrtimer *);
1355         int restart;
1356 
1357         lockdep_assert_held(&cpu_base->lock);
1358 
1359         debug_deactivate(timer);
1360         base->running = timer;
1361 
1362         /*
1363          * Separate the ->running assignment from the ->state assignment.
1364          *
1365          * As with a regular write barrier, this ensures the read side in
1366          * hrtimer_active() cannot observe base->running == NULL &&
1367          * timer->state == INACTIVE.
1368          */
1369         raw_write_seqcount_barrier(&base->seq);
1370 
1371         __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 0);
1372         fn = timer->function;
1373 
1374         /*
1375          * Clear the 'is relative' flag for the TIME_LOW_RES case. If the
1376          * timer is restarted with a period then it becomes an absolute
1377          * timer. If its not restarted it does not matter.
1378          */
1379         if (IS_ENABLED(CONFIG_TIME_LOW_RES))
1380                 timer->is_rel = false;
1381 
1382         /*
1383          * The timer is marked as running in the CPU base, so it is
1384          * protected against migration to a different CPU even if the lock
1385          * is dropped.
1386          */
1387         raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1388         trace_hrtimer_expire_entry(timer, now);
1389         restart = fn(timer);
1390         trace_hrtimer_expire_exit(timer);
1391         raw_spin_lock_irq(&cpu_base->lock);
1392 
1393         /*
1394          * Note: We clear the running state after enqueue_hrtimer and
1395          * we do not reprogram the event hardware. Happens either in
1396          * hrtimer_start_range_ns() or in hrtimer_interrupt()
1397          *
1398          * Note: Because we dropped the cpu_base->lock above,
1399          * hrtimer_start_range_ns() can have popped in and enqueued the timer
1400          * for us already.
1401          */
1402         if (restart != HRTIMER_NORESTART &&
1403             !(timer->state & HRTIMER_STATE_ENQUEUED))
1404                 enqueue_hrtimer(timer, base, HRTIMER_MODE_ABS);
1405 
1406         /*
1407          * Separate the ->running assignment from the ->state assignment.
1408          *
1409          * As with a regular write barrier, this ensures the read side in
1410          * hrtimer_active() cannot observe base->running.timer == NULL &&
1411          * timer->state == INACTIVE.
1412          */
1413         raw_write_seqcount_barrier(&base->seq);
1414 
1415         WARN_ON_ONCE(base->running != timer);
1416         base->running = NULL;
1417 }
1418 
1419 static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,
1420                                  unsigned long flags, unsigned int active_mask)
1421 {
1422         struct hrtimer_clock_base *base;
1423         unsigned int active = cpu_base->active_bases & active_mask;
1424 
1425         for_each_active_base(base, cpu_base, active) {
1426                 struct timerqueue_node *node;
1427                 ktime_t basenow;
1428 
1429                 basenow = ktime_add(now, base->offset);
1430 
1431                 while ((node = timerqueue_getnext(&base->active))) {
1432                         struct hrtimer *timer;
1433 
1434                         timer = container_of(node, struct hrtimer, node);
1435 
1436                         /*
1437                          * The immediate goal for using the softexpires is
1438                          * minimizing wakeups, not running timers at the
1439                          * earliest interrupt after their soft expiration.
1440                          * This allows us to avoid using a Priority Search
1441                          * Tree, which can answer a stabbing querry for
1442                          * overlapping intervals and instead use the simple
1443                          * BST we already have.
1444                          * We don't add extra wakeups by delaying timers that
1445                          * are right-of a not yet expired timer, because that
1446                          * timer will have to trigger a wakeup anyway.
1447                          */
1448                         if (basenow < hrtimer_get_softexpires_tv64(timer))
1449                                 break;
1450 
1451                         __run_hrtimer(cpu_base, base, timer, &basenow, flags);
1452                 }
1453         }
1454 }
1455 
1456 static __latent_entropy void hrtimer_run_softirq(struct softirq_action *h)
1457 {
1458         struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1459         unsigned long flags;
1460         ktime_t now;
1461 
1462         raw_spin_lock_irqsave(&cpu_base->lock, flags);
1463 
1464         now = hrtimer_update_base(cpu_base);
1465         __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT);
1466 
1467         cpu_base->softirq_activated = 0;
1468         hrtimer_update_softirq_timer(cpu_base, true);
1469 
1470         raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1471 }
1472 
1473 #ifdef CONFIG_HIGH_RES_TIMERS
1474 
1475 /*
1476  * High resolution timer interrupt
1477  * Called with interrupts disabled
1478  */
1479 void hrtimer_interrupt(struct clock_event_device *dev)
1480 {
1481         struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1482         ktime_t expires_next, now, entry_time, delta;
1483         unsigned long flags;
1484         int retries = 0;
1485 
1486         BUG_ON(!cpu_base->hres_active);
1487         cpu_base->nr_events++;
1488         dev->next_event = KTIME_MAX;
1489 
1490         raw_spin_lock_irqsave(&cpu_base->lock, flags);
1491         entry_time = now = hrtimer_update_base(cpu_base);
1492 retry:
1493         cpu_base->in_hrtirq = 1;
1494         /*
1495          * We set expires_next to KTIME_MAX here with cpu_base->lock
1496          * held to prevent that a timer is enqueued in our queue via
1497          * the migration code. This does not affect enqueueing of
1498          * timers which run their callback and need to be requeued on
1499          * this CPU.
1500          */
1501         cpu_base->expires_next = KTIME_MAX;
1502 
1503         if (!ktime_before(now, cpu_base->softirq_expires_next)) {
1504                 cpu_base->softirq_expires_next = KTIME_MAX;
1505                 cpu_base->softirq_activated = 1;
1506                 raise_softirq_irqoff(HRTIMER_SOFTIRQ);
1507         }
1508 
1509         __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
1510 
1511         /* Reevaluate the clock bases for the next expiry */
1512         expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
1513         /*
1514          * Store the new expiry value so the migration code can verify
1515          * against it.
1516          */
1517         cpu_base->expires_next = expires_next;
1518         cpu_base->in_hrtirq = 0;
1519         raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1520 
1521         /* Reprogramming necessary ? */
1522         if (!tick_program_event(expires_next, 0)) {
1523                 cpu_base->hang_detected = 0;
1524                 return;
1525         }
1526 
1527         /*
1528          * The next timer was already expired due to:
1529          * - tracing
1530          * - long lasting callbacks
1531          * - being scheduled away when running in a VM
1532          *
1533          * We need to prevent that we loop forever in the hrtimer
1534          * interrupt routine. We give it 3 attempts to avoid
1535          * overreacting on some spurious event.
1536          *
1537          * Acquire base lock for updating the offsets and retrieving
1538          * the current time.
1539          */
1540         raw_spin_lock_irqsave(&cpu_base->lock, flags);
1541         now = hrtimer_update_base(cpu_base);
1542         cpu_base->nr_retries++;
1543         if (++retries < 3)
1544                 goto retry;
1545         /*
1546          * Give the system a chance to do something else than looping
1547          * here. We stored the entry time, so we know exactly how long
1548          * we spent here. We schedule the next event this amount of
1549          * time away.
1550          */
1551         cpu_base->nr_hangs++;
1552         cpu_base->hang_detected = 1;
1553         raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1554 
1555         delta = ktime_sub(now, entry_time);
1556         if ((unsigned int)delta > cpu_base->max_hang_time)
1557                 cpu_base->max_hang_time = (unsigned int) delta;
1558         /*
1559          * Limit it to a sensible value as we enforce a longer
1560          * delay. Give the CPU at least 100ms to catch up.
1561          */
1562         if (delta > 100 * NSEC_PER_MSEC)
1563                 expires_next = ktime_add_ns(now, 100 * NSEC_PER_MSEC);
1564         else
1565                 expires_next = ktime_add(now, delta);
1566         tick_program_event(expires_next, 1);
1567         pr_warn_once("hrtimer: interrupt took %llu ns\n", ktime_to_ns(delta));
1568 }
1569 
1570 /* called with interrupts disabled */
1571 static inline void __hrtimer_peek_ahead_timers(void)
1572 {
1573         struct tick_device *td;
1574 
1575         if (!hrtimer_hres_active())
1576                 return;
1577 
1578         td = this_cpu_ptr(&tick_cpu_device);
1579         if (td && td->evtdev)
1580                 hrtimer_interrupt(td->evtdev);
1581 }
1582 
1583 #else /* CONFIG_HIGH_RES_TIMERS */
1584 
1585 static inline void __hrtimer_peek_ahead_timers(void) { }
1586 
1587 #endif  /* !CONFIG_HIGH_RES_TIMERS */
1588 
1589 /*
1590  * Called from run_local_timers in hardirq context every jiffy
1591  */
1592 void hrtimer_run_queues(void)
1593 {
1594         struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1595         unsigned long flags;
1596         ktime_t now;
1597 
1598         if (__hrtimer_hres_active(cpu_base))
1599                 return;
1600 
1601         /*
1602          * This _is_ ugly: We have to check periodically, whether we
1603          * can switch to highres and / or nohz mode. The clocksource
1604          * switch happens with xtime_lock held. Notification from
1605          * there only sets the check bit in the tick_oneshot code,
1606          * otherwise we might deadlock vs. xtime_lock.
1607          */
1608         if (tick_check_oneshot_change(!hrtimer_is_hres_enabled())) {
1609                 hrtimer_switch_to_hres();
1610                 return;
1611         }
1612 
1613         raw_spin_lock_irqsave(&cpu_base->lock, flags);
1614         now = hrtimer_update_base(cpu_base);
1615 
1616         if (!ktime_before(now, cpu_base->softirq_expires_next)) {
1617                 cpu_base->softirq_expires_next = KTIME_MAX;
1618                 cpu_base->softirq_activated = 1;
1619                 raise_softirq_irqoff(HRTIMER_SOFTIRQ);
1620         }
1621 
1622         __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
1623         raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1624 }
1625 
1626 /*
1627  * Sleep related functions:
1628  */
1629 static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)
1630 {
1631         struct hrtimer_sleeper *t =
1632                 container_of(timer, struct hrtimer_sleeper, timer);
1633         struct task_struct *task = t->task;
1634 
1635         t->task = NULL;
1636         if (task)
1637                 wake_up_process(task);
1638 
1639         return HRTIMER_NORESTART;
1640 }
1641 
1642 void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task)
1643 {
1644         sl->timer.function = hrtimer_wakeup;
1645         sl->task = task;
1646 }
1647 EXPORT_SYMBOL_GPL(hrtimer_init_sleeper);
1648 
1649 int nanosleep_copyout(struct restart_block *restart, struct timespec64 *ts)
1650 {
1651         switch(restart->nanosleep.type) {
1652 #ifdef CONFIG_COMPAT_32BIT_TIME
1653         case TT_COMPAT:
1654                 if (put_old_timespec32(ts, restart->nanosleep.compat_rmtp))
1655                         return -EFAULT;
1656                 break;
1657 #endif
1658         case TT_NATIVE:
1659                 if (put_timespec64(ts, restart->nanosleep.rmtp))
1660                         return -EFAULT;
1661                 break;
1662         default:
1663                 BUG();
1664         }
1665         return -ERESTART_RESTARTBLOCK;
1666 }
1667 
1668 static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode)
1669 {
1670         struct restart_block *restart;
1671 
1672         hrtimer_init_sleeper(t, current);
1673 
1674         do {
1675                 set_current_state(TASK_INTERRUPTIBLE);
1676                 hrtimer_start_expires(&t->timer, mode);
1677 
1678                 if (likely(t->task))
1679                         freezable_schedule();
1680 
1681                 hrtimer_cancel(&t->timer);
1682                 mode = HRTIMER_MODE_ABS;
1683 
1684         } while (t->task && !signal_pending(current));
1685 
1686         __set_current_state(TASK_RUNNING);
1687 
1688         if (!t->task)
1689                 return 0;
1690 
1691         restart = &current->restart_block;
1692         if (restart->nanosleep.type != TT_NONE) {
1693                 ktime_t rem = hrtimer_expires_remaining(&t->timer);
1694                 struct timespec64 rmt;
1695 
1696                 if (rem <= 0)
1697                         return 0;
1698                 rmt = ktime_to_timespec64(rem);
1699 
1700                 return nanosleep_copyout(restart, &rmt);
1701         }
1702         return -ERESTART_RESTARTBLOCK;
1703 }
1704 
1705 static long __sched hrtimer_nanosleep_restart(struct restart_block *restart)
1706 {
1707         struct hrtimer_sleeper t;
1708         int ret;
1709 
1710         hrtimer_init_on_stack(&t.timer, restart->nanosleep.clockid,
1711                                 HRTIMER_MODE_ABS);
1712         hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);
1713 
1714         ret = do_nanosleep(&t, HRTIMER_MODE_ABS);
1715         destroy_hrtimer_on_stack(&t.timer);
1716         return ret;
1717 }
1718 
1719 long hrtimer_nanosleep(const struct timespec64 *rqtp,
1720                        const enum hrtimer_mode mode, const clockid_t clockid)
1721 {
1722         struct restart_block *restart;
1723         struct hrtimer_sleeper t;
1724         int ret = 0;
1725         u64 slack;
1726 
1727         slack = current->timer_slack_ns;
1728         if (dl_task(current) || rt_task(current))
1729                 slack = 0;
1730 
1731         hrtimer_init_on_stack(&t.timer, clockid, mode);
1732         hrtimer_set_expires_range_ns(&t.timer, timespec64_to_ktime(*rqtp), slack);
1733         ret = do_nanosleep(&t, mode);
1734         if (ret != -ERESTART_RESTARTBLOCK)
1735                 goto out;
1736 
1737         /* Absolute timers do not update the rmtp value and restart: */
1738         if (mode == HRTIMER_MODE_ABS) {
1739                 ret = -ERESTARTNOHAND;
1740                 goto out;
1741         }
1742 
1743         restart = &current->restart_block;
1744         restart->fn = hrtimer_nanosleep_restart;
1745         restart->nanosleep.clockid = t.timer.base->clockid;
1746         restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer);
1747 out:
1748         destroy_hrtimer_on_stack(&t.timer);
1749         return ret;
1750 }
1751 
1752 #if !defined(CONFIG_64BIT_TIME) || defined(CONFIG_64BIT)
1753 
1754 SYSCALL_DEFINE2(nanosleep, struct __kernel_timespec __user *, rqtp,
1755                 struct __kernel_timespec __user *, rmtp)
1756 {
1757         struct timespec64 tu;
1758 
1759         if (get_timespec64(&tu, rqtp))
1760                 return -EFAULT;
1761 
1762         if (!timespec64_valid(&tu))
1763                 return -EINVAL;
1764 
1765         current->restart_block.nanosleep.type = rmtp ? TT_NATIVE : TT_NONE;
1766         current->restart_block.nanosleep.rmtp = rmtp;
1767         return hrtimer_nanosleep(&tu, HRTIMER_MODE_REL, CLOCK_MONOTONIC);
1768 }
1769 
1770 #endif
1771 
1772 #ifdef CONFIG_COMPAT_32BIT_TIME
1773 
1774 COMPAT_SYSCALL_DEFINE2(nanosleep, struct old_timespec32 __user *, rqtp,
1775                        struct old_timespec32 __user *, rmtp)
1776 {
1777         struct timespec64 tu;
1778 
1779         if (get_old_timespec32(&tu, rqtp))
1780                 return -EFAULT;
1781 
1782         if (!timespec64_valid(&tu))
1783                 return -EINVAL;
1784 
1785         current->restart_block.nanosleep.type = rmtp ? TT_COMPAT : TT_NONE;
1786         current->restart_block.nanosleep.compat_rmtp = rmtp;
1787         return hrtimer_nanosleep(&tu, HRTIMER_MODE_REL, CLOCK_MONOTONIC);
1788 }
1789 #endif
1790 
1791 /*
1792  * Functions related to boot-time initialization:
1793  */
1794 int hrtimers_prepare_cpu(unsigned int cpu)
1795 {
1796         struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);
1797         int i;
1798 
1799         for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
1800                 cpu_base->clock_base[i].cpu_base = cpu_base;
1801                 timerqueue_init_head(&cpu_base->clock_base[i].active);
1802         }
1803 
1804         cpu_base->cpu = cpu;
1805         cpu_base->active_bases = 0;
1806         cpu_base->hres_active = 0;
1807         cpu_base->hang_detected = 0;
1808         cpu_base->next_timer = NULL;
1809         cpu_base->softirq_next_timer = NULL;
1810         cpu_base->expires_next = KTIME_MAX;
1811         cpu_base->softirq_expires_next = KTIME_MAX;
1812         return 0;
1813 }
1814 
1815 #ifdef CONFIG_HOTPLUG_CPU
1816 
1817 static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
1818                                 struct hrtimer_clock_base *new_base)
1819 {
1820         struct hrtimer *timer;
1821         struct timerqueue_node *node;
1822 
1823         while ((node = timerqueue_getnext(&old_base->active))) {
1824                 timer = container_of(node, struct hrtimer, node);
1825                 BUG_ON(hrtimer_callback_running(timer));
1826                 debug_deactivate(timer);
1827 
1828                 /*
1829                  * Mark it as ENQUEUED not INACTIVE otherwise the
1830                  * timer could be seen as !active and just vanish away
1831                  * under us on another CPU
1832                  */
1833                 __remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, 0);
1834                 timer->base = new_base;
1835                 /*
1836                  * Enqueue the timers on the new cpu. This does not
1837                  * reprogram the event device in case the timer
1838                  * expires before the earliest on this CPU, but we run
1839                  * hrtimer_interrupt after we migrated everything to
1840                  * sort out already expired timers and reprogram the
1841                  * event device.
1842                  */
1843                 enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS);
1844         }
1845 }
1846 
1847 int hrtimers_dead_cpu(unsigned int scpu)
1848 {
1849         struct hrtimer_cpu_base *old_base, *new_base;
1850         int i;
1851 
1852         BUG_ON(cpu_online(scpu));
1853         tick_cancel_sched_timer(scpu);
1854 
1855         /*
1856          * this BH disable ensures that raise_softirq_irqoff() does
1857          * not wakeup ksoftirqd (and acquire the pi-lock) while
1858          * holding the cpu_base lock
1859          */
1860         local_bh_disable();
1861         local_irq_disable();
1862         old_base = &per_cpu(hrtimer_bases, scpu);
1863         new_base = this_cpu_ptr(&hrtimer_bases);
1864         /*
1865          * The caller is globally serialized and nobody else
1866          * takes two locks at once, deadlock is not possible.
1867          */
1868         raw_spin_lock(&new_base->lock);
1869         raw_spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
1870 
1871         for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
1872                 migrate_hrtimer_list(&old_base->clock_base[i],
1873                                      &new_base->clock_base[i]);
1874         }
1875 
1876         /*
1877          * The migration might have changed the first expiring softirq
1878          * timer on this CPU. Update it.
1879          */
1880         hrtimer_update_softirq_timer(new_base, false);
1881 
1882         raw_spin_unlock(&old_base->lock);
1883         raw_spin_unlock(&new_base->lock);
1884 
1885         /* Check, if we got expired work to do */
1886         __hrtimer_peek_ahead_timers();
1887         local_irq_enable();
1888         local_bh_enable();
1889         return 0;
1890 }
1891 
1892 #endif /* CONFIG_HOTPLUG_CPU */
1893 
1894 void __init hrtimers_init(void)
1895 {
1896         hrtimers_prepare_cpu(smp_processor_id());
1897         open_softirq(HRTIMER_SOFTIRQ, hrtimer_run_softirq);
1898 }
1899 
1900 /**
1901  * schedule_hrtimeout_range_clock - sleep until timeout
1902  * @expires:    timeout value (ktime_t)
1903  * @delta:      slack in expires timeout (ktime_t)
1904  * @mode:       timer mode
1905  * @clock_id:   timer clock to be used
1906  */
1907 int __sched
1908 schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
1909                                const enum hrtimer_mode mode, clockid_t clock_id)
1910 {
1911         struct hrtimer_sleeper t;
1912 
1913         /*
1914          * Optimize when a zero timeout value is given. It does not
1915          * matter whether this is an absolute or a relative time.
1916          */
1917         if (expires && *expires == 0) {
1918                 __set_current_state(TASK_RUNNING);
1919                 return 0;
1920         }
1921 
1922         /*
1923          * A NULL parameter means "infinite"
1924          */
1925         if (!expires) {
1926                 schedule();
1927                 return -EINTR;
1928         }
1929 
1930         hrtimer_init_on_stack(&t.timer, clock_id, mode);
1931         hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
1932 
1933         hrtimer_init_sleeper(&t, current);
1934 
1935         hrtimer_start_expires(&t.timer, mode);
1936 
1937         if (likely(t.task))
1938                 schedule();
1939 
1940         hrtimer_cancel(&t.timer);
1941         destroy_hrtimer_on_stack(&t.timer);
1942 
1943         __set_current_state(TASK_RUNNING);
1944 
1945         return !t.task ? 0 : -EINTR;
1946 }
1947 
1948 /**
1949  * schedule_hrtimeout_range - sleep until timeout
1950  * @expires:    timeout value (ktime_t)
1951  * @delta:      slack in expires timeout (ktime_t)
1952  * @mode:       timer mode
1953  *
1954  * Make the current task sleep until the given expiry time has
1955  * elapsed. The routine will return immediately unless
1956  * the current task state has been set (see set_current_state()).
1957  *
1958  * The @delta argument gives the kernel the freedom to schedule the
1959  * actual wakeup to a time that is both power and performance friendly.
1960  * The kernel give the normal best effort behavior for "@expires+@delta",
1961  * but may decide to fire the timer earlier, but no earlier than @expires.
1962  *
1963  * You can set the task state as follows -
1964  *
1965  * %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
1966  * pass before the routine returns unless the current task is explicitly
1967  * woken up, (e.g. by wake_up_process()).
1968  *
1969  * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
1970  * delivered to the current task or the current task is explicitly woken
1971  * up.
1972  *
1973  * The current task state is guaranteed to be TASK_RUNNING when this
1974  * routine returns.
1975  *
1976  * Returns 0 when the timer has expired. If the task was woken before the
1977  * timer expired by a signal (only possible in state TASK_INTERRUPTIBLE) or
1978  * by an explicit wakeup, it returns -EINTR.
1979  */
1980 int __sched schedule_hrtimeout_range(ktime_t *expires, u64 delta,
1981                                      const enum hrtimer_mode mode)
1982 {
1983         return schedule_hrtimeout_range_clock(expires, delta, mode,
1984                                               CLOCK_MONOTONIC);
1985 }
1986 EXPORT_SYMBOL_GPL(schedule_hrtimeout_range);
1987 
1988 /**
1989  * schedule_hrtimeout - sleep until timeout
1990  * @expires:    timeout value (ktime_t)
1991  * @mode:       timer mode
1992  *
1993  * Make the current task sleep until the given expiry time has
1994  * elapsed. The routine will return immediately unless
1995  * the current task state has been set (see set_current_state()).
1996  *
1997  * You can set the task state as follows -
1998  *
1999  * %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
2000  * pass before the routine returns unless the current task is explicitly
2001  * woken up, (e.g. by wake_up_process()).
2002  *
2003  * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
2004  * delivered to the current task or the current task is explicitly woken
2005  * up.
2006  *
2007  * The current task state is guaranteed to be TASK_RUNNING when this
2008  * routine returns.
2009  *
2010  * Returns 0 when the timer has expired. If the task was woken before the
2011  * timer expired by a signal (only possible in state TASK_INTERRUPTIBLE) or
2012  * by an explicit wakeup, it returns -EINTR.
2013  */
2014 int __sched schedule_hrtimeout(ktime_t *expires,
2015                                const enum hrtimer_mode mode)
2016 {
2017         return schedule_hrtimeout_range(expires, 0, mode);
2018 }
2019 EXPORT_SYMBOL_GPL(schedule_hrtimeout);
2020 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp