~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/time/timekeeping.c

Version: ~ [ linux-5.4-rc7 ] ~ [ linux-5.3.10 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.83 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.153 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.200 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.200 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.76 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *  linux/kernel/time/timekeeping.c
  3  *
  4  *  Kernel timekeeping code and accessor functions
  5  *
  6  *  This code was moved from linux/kernel/timer.c.
  7  *  Please see that file for copyright and history logs.
  8  *
  9  */
 10 
 11 #include <linux/timekeeper_internal.h>
 12 #include <linux/module.h>
 13 #include <linux/interrupt.h>
 14 #include <linux/percpu.h>
 15 #include <linux/init.h>
 16 #include <linux/mm.h>
 17 #include <linux/sched.h>
 18 #include <linux/syscore_ops.h>
 19 #include <linux/clocksource.h>
 20 #include <linux/jiffies.h>
 21 #include <linux/time.h>
 22 #include <linux/tick.h>
 23 #include <linux/stop_machine.h>
 24 #include <linux/pvclock_gtod.h>
 25 #include <linux/compiler.h>
 26 
 27 #include "tick-internal.h"
 28 #include "ntp_internal.h"
 29 #include "timekeeping_internal.h"
 30 
 31 #define TK_CLEAR_NTP            (1 << 0)
 32 #define TK_MIRROR               (1 << 1)
 33 #define TK_CLOCK_WAS_SET        (1 << 2)
 34 
 35 /*
 36  * The most important data for readout fits into a single 64 byte
 37  * cache line.
 38  */
 39 static struct {
 40         seqcount_t              seq;
 41         struct timekeeper       timekeeper;
 42 } tk_core ____cacheline_aligned;
 43 
 44 static DEFINE_RAW_SPINLOCK(timekeeper_lock);
 45 static struct timekeeper shadow_timekeeper;
 46 
 47 /**
 48  * struct tk_fast - NMI safe timekeeper
 49  * @seq:        Sequence counter for protecting updates. The lowest bit
 50  *              is the index for the tk_read_base array
 51  * @base:       tk_read_base array. Access is indexed by the lowest bit of
 52  *              @seq.
 53  *
 54  * See @update_fast_timekeeper() below.
 55  */
 56 struct tk_fast {
 57         seqcount_t              seq;
 58         struct tk_read_base     base[2];
 59 };
 60 
 61 static struct tk_fast tk_fast_mono ____cacheline_aligned;
 62 static struct tk_fast tk_fast_raw  ____cacheline_aligned;
 63 
 64 /* flag for if timekeeping is suspended */
 65 int __read_mostly timekeeping_suspended;
 66 
 67 static inline void tk_normalize_xtime(struct timekeeper *tk)
 68 {
 69         while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) {
 70                 tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
 71                 tk->xtime_sec++;
 72         }
 73 }
 74 
 75 static inline struct timespec64 tk_xtime(struct timekeeper *tk)
 76 {
 77         struct timespec64 ts;
 78 
 79         ts.tv_sec = tk->xtime_sec;
 80         ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
 81         return ts;
 82 }
 83 
 84 static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts)
 85 {
 86         tk->xtime_sec = ts->tv_sec;
 87         tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift;
 88 }
 89 
 90 static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts)
 91 {
 92         tk->xtime_sec += ts->tv_sec;
 93         tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift;
 94         tk_normalize_xtime(tk);
 95 }
 96 
 97 static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm)
 98 {
 99         struct timespec64 tmp;
100 
101         /*
102          * Verify consistency of: offset_real = -wall_to_monotonic
103          * before modifying anything
104          */
105         set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec,
106                                         -tk->wall_to_monotonic.tv_nsec);
107         WARN_ON_ONCE(tk->offs_real.tv64 != timespec64_to_ktime(tmp).tv64);
108         tk->wall_to_monotonic = wtm;
109         set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec);
110         tk->offs_real = timespec64_to_ktime(tmp);
111         tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0));
112 }
113 
114 static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)
115 {
116         tk->offs_boot = ktime_add(tk->offs_boot, delta);
117 }
118 
119 /*
120  * tk_clock_read - atomic clocksource read() helper
121  *
122  * This helper is necessary to use in the read paths because, while the
123  * seqlock ensures we don't return a bad value while structures are updated,
124  * it doesn't protect from potential crashes. There is the possibility that
125  * the tkr's clocksource may change between the read reference, and the
126  * clock reference passed to the read function.  This can cause crashes if
127  * the wrong clocksource is passed to the wrong read function.
128  * This isn't necessary to use when holding the timekeeper_lock or doing
129  * a read of the fast-timekeeper tkrs (which is protected by its own locking
130  * and update logic).
131  */
132 static inline u64 tk_clock_read(struct tk_read_base *tkr)
133 {
134         struct clocksource *clock = READ_ONCE(tkr->clock);
135 
136         return clock->read(clock);
137 }
138 
139 #ifdef CONFIG_DEBUG_TIMEKEEPING
140 #define WARNING_FREQ (HZ*300) /* 5 minute rate-limiting */
141 /*
142  * These simple flag variables are managed
143  * without locks, which is racy, but ok since
144  * we don't really care about being super
145  * precise about how many events were seen,
146  * just that a problem was observed.
147  */
148 static int timekeeping_underflow_seen;
149 static int timekeeping_overflow_seen;
150 
151 /* last_warning is only modified under the timekeeping lock */
152 static long timekeeping_last_warning;
153 
154 static void timekeeping_check_update(struct timekeeper *tk, cycle_t offset)
155 {
156 
157         cycle_t max_cycles = tk->tkr_mono.clock->max_cycles;
158         const char *name = tk->tkr_mono.clock->name;
159 
160         if (offset > max_cycles) {
161                 printk_deferred("WARNING: timekeeping: Cycle offset (%lld) is larger than allowed by the '%s' clock's max_cycles value (%lld): time overflow danger\n",
162                                 offset, name, max_cycles);
163                 printk_deferred("         timekeeping: Your kernel is sick, but tries to cope by capping time updates\n");
164         } else {
165                 if (offset > (max_cycles >> 1)) {
166                         printk_deferred("INFO: timekeeping: Cycle offset (%lld) is larger than the the '%s' clock's 50%% safety margin (%lld)\n",
167                                         offset, name, max_cycles >> 1);
168                         printk_deferred("      timekeeping: Your kernel is still fine, but is feeling a bit nervous\n");
169                 }
170         }
171 
172         if (timekeeping_underflow_seen) {
173                 if (jiffies - timekeeping_last_warning > WARNING_FREQ) {
174                         printk_deferred("WARNING: Underflow in clocksource '%s' observed, time update ignored.\n", name);
175                         printk_deferred("         Please report this, consider using a different clocksource, if possible.\n");
176                         printk_deferred("         Your kernel is probably still fine.\n");
177                         timekeeping_last_warning = jiffies;
178                 }
179                 timekeeping_underflow_seen = 0;
180         }
181 
182         if (timekeeping_overflow_seen) {
183                 if (jiffies - timekeeping_last_warning > WARNING_FREQ) {
184                         printk_deferred("WARNING: Overflow in clocksource '%s' observed, time update capped.\n", name);
185                         printk_deferred("         Please report this, consider using a different clocksource, if possible.\n");
186                         printk_deferred("         Your kernel is probably still fine.\n");
187                         timekeeping_last_warning = jiffies;
188                 }
189                 timekeeping_overflow_seen = 0;
190         }
191 }
192 
193 static inline cycle_t timekeeping_get_delta(struct tk_read_base *tkr)
194 {
195         cycle_t now, last, mask, max, delta;
196         unsigned int seq;
197 
198         /*
199          * Since we're called holding a seqlock, the data may shift
200          * under us while we're doing the calculation. This can cause
201          * false positives, since we'd note a problem but throw the
202          * results away. So nest another seqlock here to atomically
203          * grab the points we are checking with.
204          */
205         do {
206                 seq = read_seqcount_begin(&tk_core.seq);
207                 now = tk_clock_read(tkr);
208                 last = tkr->cycle_last;
209                 mask = tkr->mask;
210                 max = tkr->clock->max_cycles;
211         } while (read_seqcount_retry(&tk_core.seq, seq));
212 
213         delta = clocksource_delta(now, last, mask);
214 
215         /*
216          * Try to catch underflows by checking if we are seeing small
217          * mask-relative negative values.
218          */
219         if (unlikely((~delta & mask) < (mask >> 3))) {
220                 timekeeping_underflow_seen = 1;
221                 delta = 0;
222         }
223 
224         /* Cap delta value to the max_cycles values to avoid mult overflows */
225         if (unlikely(delta > max)) {
226                 timekeeping_overflow_seen = 1;
227                 delta = tkr->clock->max_cycles;
228         }
229 
230         return delta;
231 }
232 #else
233 static inline void timekeeping_check_update(struct timekeeper *tk, cycle_t offset)
234 {
235 }
236 static inline cycle_t timekeeping_get_delta(struct tk_read_base *tkr)
237 {
238         cycle_t cycle_now, delta;
239 
240         /* read clocksource */
241         cycle_now = tk_clock_read(tkr);
242 
243         /* calculate the delta since the last update_wall_time */
244         delta = clocksource_delta(cycle_now, tkr->cycle_last, tkr->mask);
245 
246         return delta;
247 }
248 #endif
249 
250 /**
251  * tk_setup_internals - Set up internals to use clocksource clock.
252  *
253  * @tk:         The target timekeeper to setup.
254  * @clock:              Pointer to clocksource.
255  *
256  * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment
257  * pair and interval request.
258  *
259  * Unless you're the timekeeping code, you should not be using this!
260  */
261 static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
262 {
263         cycle_t interval;
264         u64 tmp, ntpinterval;
265         struct clocksource *old_clock;
266 
267         old_clock = tk->tkr_mono.clock;
268         tk->tkr_mono.clock = clock;
269         tk->tkr_mono.mask = clock->mask;
270         tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono);
271 
272         tk->tkr_raw.clock = clock;
273         tk->tkr_raw.mask = clock->mask;
274         tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last;
275 
276         /* Do the ns -> cycle conversion first, using original mult */
277         tmp = NTP_INTERVAL_LENGTH;
278         tmp <<= clock->shift;
279         ntpinterval = tmp;
280         tmp += clock->mult/2;
281         do_div(tmp, clock->mult);
282         if (tmp == 0)
283                 tmp = 1;
284 
285         interval = (cycle_t) tmp;
286         tk->cycle_interval = interval;
287 
288         /* Go back from cycles -> shifted ns */
289         tk->xtime_interval = (u64) interval * clock->mult;
290         tk->xtime_remainder = ntpinterval - tk->xtime_interval;
291         tk->raw_interval =
292                 ((u64) interval * clock->mult) >> clock->shift;
293 
294          /* if changing clocks, convert xtime_nsec shift units */
295         if (old_clock) {
296                 int shift_change = clock->shift - old_clock->shift;
297                 if (shift_change < 0)
298                         tk->tkr_mono.xtime_nsec >>= -shift_change;
299                 else
300                         tk->tkr_mono.xtime_nsec <<= shift_change;
301         }
302         tk->tkr_raw.xtime_nsec = 0;
303 
304         tk->tkr_mono.shift = clock->shift;
305         tk->tkr_raw.shift = clock->shift;
306 
307         tk->ntp_error = 0;
308         tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
309         tk->ntp_tick = ntpinterval << tk->ntp_error_shift;
310 
311         /*
312          * The timekeeper keeps its own mult values for the currently
313          * active clocksource. These value will be adjusted via NTP
314          * to counteract clock drifting.
315          */
316         tk->tkr_mono.mult = clock->mult;
317         tk->tkr_raw.mult = clock->mult;
318         tk->ntp_err_mult = 0;
319 }
320 
321 /* Timekeeper helper functions. */
322 
323 #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
324 static u32 default_arch_gettimeoffset(void) { return 0; }
325 u32 (*arch_gettimeoffset)(void) = default_arch_gettimeoffset;
326 #else
327 static inline u32 arch_gettimeoffset(void) { return 0; }
328 #endif
329 
330 static inline u64 timekeeping_delta_to_ns(struct tk_read_base *tkr,
331                                           cycle_t delta)
332 {
333         u64 nsec;
334 
335         nsec = delta * tkr->mult + tkr->xtime_nsec;
336         nsec >>= tkr->shift;
337 
338         /* If arch requires, add in get_arch_timeoffset() */
339         return nsec + arch_gettimeoffset();
340 }
341 
342 static inline s64 timekeeping_get_ns(struct tk_read_base *tkr)
343 {
344         cycle_t delta;
345 
346         delta = timekeeping_get_delta(tkr);
347         return timekeeping_delta_to_ns(tkr, delta);
348 }
349 
350 static inline s64 timekeeping_cycles_to_ns(struct tk_read_base *tkr,
351                                             cycle_t cycles)
352 {
353         cycle_t delta;
354 
355         /* calculate the delta since the last update_wall_time */
356         delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask);
357         return timekeeping_delta_to_ns(tkr, delta);
358 }
359 
360 /**
361  * update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper.
362  * @tkr: Timekeeping readout base from which we take the update
363  *
364  * We want to use this from any context including NMI and tracing /
365  * instrumenting the timekeeping code itself.
366  *
367  * So we handle this differently than the other timekeeping accessor
368  * functions which retry when the sequence count has changed. The
369  * update side does:
370  *
371  * smp_wmb();   <- Ensure that the last base[1] update is visible
372  * tkf->seq++;
373  * smp_wmb();   <- Ensure that the seqcount update is visible
374  * update(tkf->base[0], tkr);
375  * smp_wmb();   <- Ensure that the base[0] update is visible
376  * tkf->seq++;
377  * smp_wmb();   <- Ensure that the seqcount update is visible
378  * update(tkf->base[1], tkr);
379  *
380  * The reader side does:
381  *
382  * do {
383  *      seq = tkf->seq;
384  *      smp_rmb();
385  *      idx = seq & 0x01;
386  *      now = now(tkf->base[idx]);
387  *      smp_rmb();
388  * } while (seq != tkf->seq)
389  *
390  * As long as we update base[0] readers are forced off to
391  * base[1]. Once base[0] is updated readers are redirected to base[0]
392  * and the base[1] update takes place.
393  *
394  * So if a NMI hits the update of base[0] then it will use base[1]
395  * which is still consistent. In the worst case this can result is a
396  * slightly wrong timestamp (a few nanoseconds). See
397  * @ktime_get_mono_fast_ns.
398  */
399 static void update_fast_timekeeper(struct tk_read_base *tkr, struct tk_fast *tkf)
400 {
401         struct tk_read_base *base = tkf->base;
402 
403         /* Force readers off to base[1] */
404         raw_write_seqcount_latch(&tkf->seq);
405 
406         /* Update base[0] */
407         memcpy(base, tkr, sizeof(*base));
408 
409         /* Force readers back to base[0] */
410         raw_write_seqcount_latch(&tkf->seq);
411 
412         /* Update base[1] */
413         memcpy(base + 1, base, sizeof(*base));
414 }
415 
416 /**
417  * ktime_get_mono_fast_ns - Fast NMI safe access to clock monotonic
418  *
419  * This timestamp is not guaranteed to be monotonic across an update.
420  * The timestamp is calculated by:
421  *
422  *      now = base_mono + clock_delta * slope
423  *
424  * So if the update lowers the slope, readers who are forced to the
425  * not yet updated second array are still using the old steeper slope.
426  *
427  * tmono
428  * ^
429  * |    o  n
430  * |   o n
431  * |  u
432  * | o
433  * |o
434  * |12345678---> reader order
435  *
436  * o = old slope
437  * u = update
438  * n = new slope
439  *
440  * So reader 6 will observe time going backwards versus reader 5.
441  *
442  * While other CPUs are likely to be able observe that, the only way
443  * for a CPU local observation is when an NMI hits in the middle of
444  * the update. Timestamps taken from that NMI context might be ahead
445  * of the following timestamps. Callers need to be aware of that and
446  * deal with it.
447  */
448 static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf)
449 {
450         struct tk_read_base *tkr;
451         unsigned int seq;
452         u64 now;
453 
454         do {
455                 seq = raw_read_seqcount(&tkf->seq);
456                 tkr = tkf->base + (seq & 0x01);
457                 now = ktime_to_ns(tkr->base);
458 
459                 now += timekeeping_delta_to_ns(tkr,
460                                 clocksource_delta(
461                                         tk_clock_read(tkr),
462                                         tkr->cycle_last,
463                                         tkr->mask));
464         } while (read_seqcount_retry(&tkf->seq, seq));
465 
466         return now;
467 }
468 
469 u64 ktime_get_mono_fast_ns(void)
470 {
471         return __ktime_get_fast_ns(&tk_fast_mono);
472 }
473 EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns);
474 
475 u64 ktime_get_raw_fast_ns(void)
476 {
477         return __ktime_get_fast_ns(&tk_fast_raw);
478 }
479 EXPORT_SYMBOL_GPL(ktime_get_raw_fast_ns);
480 
481 /* Suspend-time cycles value for halted fast timekeeper. */
482 static cycle_t cycles_at_suspend;
483 
484 static cycle_t dummy_clock_read(struct clocksource *cs)
485 {
486         return cycles_at_suspend;
487 }
488 
489 static struct clocksource dummy_clock = {
490         .read = dummy_clock_read,
491 };
492 
493 /**
494  * halt_fast_timekeeper - Prevent fast timekeeper from accessing clocksource.
495  * @tk: Timekeeper to snapshot.
496  *
497  * It generally is unsafe to access the clocksource after timekeeping has been
498  * suspended, so take a snapshot of the readout base of @tk and use it as the
499  * fast timekeeper's readout base while suspended.  It will return the same
500  * number of cycles every time until timekeeping is resumed at which time the
501  * proper readout base for the fast timekeeper will be restored automatically.
502  */
503 static void halt_fast_timekeeper(struct timekeeper *tk)
504 {
505         static struct tk_read_base tkr_dummy;
506         struct tk_read_base *tkr = &tk->tkr_mono;
507 
508         memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
509         cycles_at_suspend = tk_clock_read(tkr);
510         tkr_dummy.clock = &dummy_clock;
511         update_fast_timekeeper(&tkr_dummy, &tk_fast_mono);
512 
513         tkr = &tk->tkr_raw;
514         memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
515         tkr_dummy.clock = &dummy_clock;
516         update_fast_timekeeper(&tkr_dummy, &tk_fast_raw);
517 }
518 
519 #ifdef CONFIG_GENERIC_TIME_VSYSCALL_OLD
520 
521 static inline void update_vsyscall(struct timekeeper *tk)
522 {
523         struct timespec xt, wm;
524 
525         xt = timespec64_to_timespec(tk_xtime(tk));
526         wm = timespec64_to_timespec(tk->wall_to_monotonic);
527         update_vsyscall_old(&xt, &wm, tk->tkr_mono.clock, tk->tkr_mono.mult,
528                             tk->tkr_mono.cycle_last);
529 }
530 
531 static inline void old_vsyscall_fixup(struct timekeeper *tk)
532 {
533         s64 remainder;
534 
535         /*
536         * Store only full nanoseconds into xtime_nsec after rounding
537         * it up and add the remainder to the error difference.
538         * XXX - This is necessary to avoid small 1ns inconsistnecies caused
539         * by truncating the remainder in vsyscalls. However, it causes
540         * additional work to be done in timekeeping_adjust(). Once
541         * the vsyscall implementations are converted to use xtime_nsec
542         * (shifted nanoseconds), and CONFIG_GENERIC_TIME_VSYSCALL_OLD
543         * users are removed, this can be killed.
544         */
545         remainder = tk->tkr_mono.xtime_nsec & ((1ULL << tk->tkr_mono.shift) - 1);
546         tk->tkr_mono.xtime_nsec -= remainder;
547         tk->tkr_mono.xtime_nsec += 1ULL << tk->tkr_mono.shift;
548         tk->ntp_error += remainder << tk->ntp_error_shift;
549         tk->ntp_error -= (1ULL << tk->tkr_mono.shift) << tk->ntp_error_shift;
550 }
551 #else
552 #define old_vsyscall_fixup(tk)
553 #endif
554 
555 static RAW_NOTIFIER_HEAD(pvclock_gtod_chain);
556 
557 static void update_pvclock_gtod(struct timekeeper *tk, bool was_set)
558 {
559         raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk);
560 }
561 
562 /**
563  * pvclock_gtod_register_notifier - register a pvclock timedata update listener
564  */
565 int pvclock_gtod_register_notifier(struct notifier_block *nb)
566 {
567         struct timekeeper *tk = &tk_core.timekeeper;
568         unsigned long flags;
569         int ret;
570 
571         raw_spin_lock_irqsave(&timekeeper_lock, flags);
572         ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb);
573         update_pvclock_gtod(tk, true);
574         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
575 
576         return ret;
577 }
578 EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier);
579 
580 /**
581  * pvclock_gtod_unregister_notifier - unregister a pvclock
582  * timedata update listener
583  */
584 int pvclock_gtod_unregister_notifier(struct notifier_block *nb)
585 {
586         unsigned long flags;
587         int ret;
588 
589         raw_spin_lock_irqsave(&timekeeper_lock, flags);
590         ret = raw_notifier_chain_unregister(&pvclock_gtod_chain, nb);
591         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
592 
593         return ret;
594 }
595 EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier);
596 
597 /*
598  * Update the ktime_t based scalar nsec members of the timekeeper
599  */
600 static inline void tk_update_ktime_data(struct timekeeper *tk)
601 {
602         u64 seconds;
603         u32 nsec;
604 
605         /*
606          * The xtime based monotonic readout is:
607          *      nsec = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec + now();
608          * The ktime based monotonic readout is:
609          *      nsec = base_mono + now();
610          * ==> base_mono = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec
611          */
612         seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec);
613         nsec = (u32) tk->wall_to_monotonic.tv_nsec;
614         tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec);
615 
616         /* Update the monotonic raw base */
617         tk->tkr_raw.base = timespec64_to_ktime(tk->raw_time);
618 
619         /*
620          * The sum of the nanoseconds portions of xtime and
621          * wall_to_monotonic can be greater/equal one second. Take
622          * this into account before updating tk->ktime_sec.
623          */
624         nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
625         if (nsec >= NSEC_PER_SEC)
626                 seconds++;
627         tk->ktime_sec = seconds;
628 }
629 
630 /* must hold timekeeper_lock */
631 static void timekeeping_update(struct timekeeper *tk, unsigned int action)
632 {
633         if (action & TK_CLEAR_NTP) {
634                 tk->ntp_error = 0;
635                 ntp_clear();
636         }
637 
638         tk_update_ktime_data(tk);
639 
640         update_vsyscall(tk);
641         update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET);
642 
643         if (action & TK_MIRROR)
644                 memcpy(&shadow_timekeeper, &tk_core.timekeeper,
645                        sizeof(tk_core.timekeeper));
646 
647         update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono);
648         update_fast_timekeeper(&tk->tkr_raw,  &tk_fast_raw);
649 }
650 
651 /**
652  * timekeeping_forward_now - update clock to the current time
653  *
654  * Forward the current clock to update its state since the last call to
655  * update_wall_time(). This is useful before significant clock changes,
656  * as it avoids having to deal with this time offset explicitly.
657  */
658 static void timekeeping_forward_now(struct timekeeper *tk)
659 {
660         cycle_t cycle_now, delta;
661         s64 nsec;
662 
663         cycle_now = tk_clock_read(&tk->tkr_mono);
664         delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
665         tk->tkr_mono.cycle_last = cycle_now;
666         tk->tkr_raw.cycle_last  = cycle_now;
667 
668         tk->tkr_mono.xtime_nsec += delta * tk->tkr_mono.mult;
669 
670         /* If arch requires, add in get_arch_timeoffset() */
671         tk->tkr_mono.xtime_nsec += (u64)arch_gettimeoffset() << tk->tkr_mono.shift;
672 
673         tk_normalize_xtime(tk);
674 
675         nsec = clocksource_cyc2ns(delta, tk->tkr_raw.mult, tk->tkr_raw.shift);
676         timespec64_add_ns(&tk->raw_time, nsec);
677 }
678 
679 /**
680  * __getnstimeofday64 - Returns the time of day in a timespec64.
681  * @ts:         pointer to the timespec to be set
682  *
683  * Updates the time of day in the timespec.
684  * Returns 0 on success, or -ve when suspended (timespec will be undefined).
685  */
686 int __getnstimeofday64(struct timespec64 *ts)
687 {
688         struct timekeeper *tk = &tk_core.timekeeper;
689         unsigned long seq;
690         s64 nsecs = 0;
691 
692         do {
693                 seq = read_seqcount_begin(&tk_core.seq);
694 
695                 ts->tv_sec = tk->xtime_sec;
696                 nsecs = timekeeping_get_ns(&tk->tkr_mono);
697 
698         } while (read_seqcount_retry(&tk_core.seq, seq));
699 
700         ts->tv_nsec = 0;
701         timespec64_add_ns(ts, nsecs);
702 
703         /*
704          * Do not bail out early, in case there were callers still using
705          * the value, even in the face of the WARN_ON.
706          */
707         if (unlikely(timekeeping_suspended))
708                 return -EAGAIN;
709         return 0;
710 }
711 EXPORT_SYMBOL(__getnstimeofday64);
712 
713 /**
714  * getnstimeofday64 - Returns the time of day in a timespec64.
715  * @ts:         pointer to the timespec64 to be set
716  *
717  * Returns the time of day in a timespec64 (WARN if suspended).
718  */
719 void getnstimeofday64(struct timespec64 *ts)
720 {
721         WARN_ON(__getnstimeofday64(ts));
722 }
723 EXPORT_SYMBOL(getnstimeofday64);
724 
725 ktime_t ktime_get(void)
726 {
727         struct timekeeper *tk = &tk_core.timekeeper;
728         unsigned int seq;
729         ktime_t base;
730         s64 nsecs;
731 
732         WARN_ON(timekeeping_suspended);
733 
734         do {
735                 seq = read_seqcount_begin(&tk_core.seq);
736                 base = tk->tkr_mono.base;
737                 nsecs = timekeeping_get_ns(&tk->tkr_mono);
738 
739         } while (read_seqcount_retry(&tk_core.seq, seq));
740 
741         return ktime_add_ns(base, nsecs);
742 }
743 EXPORT_SYMBOL_GPL(ktime_get);
744 
745 static ktime_t *offsets[TK_OFFS_MAX] = {
746         [TK_OFFS_REAL]  = &tk_core.timekeeper.offs_real,
747         [TK_OFFS_BOOT]  = &tk_core.timekeeper.offs_boot,
748         [TK_OFFS_TAI]   = &tk_core.timekeeper.offs_tai,
749 };
750 
751 ktime_t ktime_get_with_offset(enum tk_offsets offs)
752 {
753         struct timekeeper *tk = &tk_core.timekeeper;
754         unsigned int seq;
755         ktime_t base, *offset = offsets[offs];
756         s64 nsecs;
757 
758         WARN_ON(timekeeping_suspended);
759 
760         do {
761                 seq = read_seqcount_begin(&tk_core.seq);
762                 base = ktime_add(tk->tkr_mono.base, *offset);
763                 nsecs = timekeeping_get_ns(&tk->tkr_mono);
764 
765         } while (read_seqcount_retry(&tk_core.seq, seq));
766 
767         return ktime_add_ns(base, nsecs);
768 
769 }
770 EXPORT_SYMBOL_GPL(ktime_get_with_offset);
771 
772 /**
773  * ktime_mono_to_any() - convert mononotic time to any other time
774  * @tmono:      time to convert.
775  * @offs:       which offset to use
776  */
777 ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs)
778 {
779         ktime_t *offset = offsets[offs];
780         unsigned long seq;
781         ktime_t tconv;
782 
783         do {
784                 seq = read_seqcount_begin(&tk_core.seq);
785                 tconv = ktime_add(tmono, *offset);
786         } while (read_seqcount_retry(&tk_core.seq, seq));
787 
788         return tconv;
789 }
790 EXPORT_SYMBOL_GPL(ktime_mono_to_any);
791 
792 /**
793  * ktime_get_raw - Returns the raw monotonic time in ktime_t format
794  */
795 ktime_t ktime_get_raw(void)
796 {
797         struct timekeeper *tk = &tk_core.timekeeper;
798         unsigned int seq;
799         ktime_t base;
800         s64 nsecs;
801 
802         do {
803                 seq = read_seqcount_begin(&tk_core.seq);
804                 base = tk->tkr_raw.base;
805                 nsecs = timekeeping_get_ns(&tk->tkr_raw);
806 
807         } while (read_seqcount_retry(&tk_core.seq, seq));
808 
809         return ktime_add_ns(base, nsecs);
810 }
811 EXPORT_SYMBOL_GPL(ktime_get_raw);
812 
813 /**
814  * ktime_get_ts64 - get the monotonic clock in timespec64 format
815  * @ts:         pointer to timespec variable
816  *
817  * The function calculates the monotonic clock from the realtime
818  * clock and the wall_to_monotonic offset and stores the result
819  * in normalized timespec64 format in the variable pointed to by @ts.
820  */
821 void ktime_get_ts64(struct timespec64 *ts)
822 {
823         struct timekeeper *tk = &tk_core.timekeeper;
824         struct timespec64 tomono;
825         s64 nsec;
826         unsigned int seq;
827 
828         WARN_ON(timekeeping_suspended);
829 
830         do {
831                 seq = read_seqcount_begin(&tk_core.seq);
832                 ts->tv_sec = tk->xtime_sec;
833                 nsec = timekeeping_get_ns(&tk->tkr_mono);
834                 tomono = tk->wall_to_monotonic;
835 
836         } while (read_seqcount_retry(&tk_core.seq, seq));
837 
838         ts->tv_sec += tomono.tv_sec;
839         ts->tv_nsec = 0;
840         timespec64_add_ns(ts, nsec + tomono.tv_nsec);
841 }
842 EXPORT_SYMBOL_GPL(ktime_get_ts64);
843 
844 /**
845  * ktime_get_seconds - Get the seconds portion of CLOCK_MONOTONIC
846  *
847  * Returns the seconds portion of CLOCK_MONOTONIC with a single non
848  * serialized read. tk->ktime_sec is of type 'unsigned long' so this
849  * works on both 32 and 64 bit systems. On 32 bit systems the readout
850  * covers ~136 years of uptime which should be enough to prevent
851  * premature wrap arounds.
852  */
853 time64_t ktime_get_seconds(void)
854 {
855         struct timekeeper *tk = &tk_core.timekeeper;
856 
857         WARN_ON(timekeeping_suspended);
858         return tk->ktime_sec;
859 }
860 EXPORT_SYMBOL_GPL(ktime_get_seconds);
861 
862 /**
863  * ktime_get_real_seconds - Get the seconds portion of CLOCK_REALTIME
864  *
865  * Returns the wall clock seconds since 1970. This replaces the
866  * get_seconds() interface which is not y2038 safe on 32bit systems.
867  *
868  * For 64bit systems the fast access to tk->xtime_sec is preserved. On
869  * 32bit systems the access must be protected with the sequence
870  * counter to provide "atomic" access to the 64bit tk->xtime_sec
871  * value.
872  */
873 time64_t ktime_get_real_seconds(void)
874 {
875         struct timekeeper *tk = &tk_core.timekeeper;
876         time64_t seconds;
877         unsigned int seq;
878 
879         if (IS_ENABLED(CONFIG_64BIT))
880                 return tk->xtime_sec;
881 
882         do {
883                 seq = read_seqcount_begin(&tk_core.seq);
884                 seconds = tk->xtime_sec;
885 
886         } while (read_seqcount_retry(&tk_core.seq, seq));
887 
888         return seconds;
889 }
890 EXPORT_SYMBOL_GPL(ktime_get_real_seconds);
891 
892 #ifdef CONFIG_NTP_PPS
893 
894 /**
895  * getnstime_raw_and_real - get day and raw monotonic time in timespec format
896  * @ts_raw:     pointer to the timespec to be set to raw monotonic time
897  * @ts_real:    pointer to the timespec to be set to the time of day
898  *
899  * This function reads both the time of day and raw monotonic time at the
900  * same time atomically and stores the resulting timestamps in timespec
901  * format.
902  */
903 void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real)
904 {
905         struct timekeeper *tk = &tk_core.timekeeper;
906         unsigned long seq;
907         s64 nsecs_raw, nsecs_real;
908 
909         WARN_ON_ONCE(timekeeping_suspended);
910 
911         do {
912                 seq = read_seqcount_begin(&tk_core.seq);
913 
914                 *ts_raw = timespec64_to_timespec(tk->raw_time);
915                 ts_real->tv_sec = tk->xtime_sec;
916                 ts_real->tv_nsec = 0;
917 
918                 nsecs_raw  = timekeeping_get_ns(&tk->tkr_raw);
919                 nsecs_real = timekeeping_get_ns(&tk->tkr_mono);
920 
921         } while (read_seqcount_retry(&tk_core.seq, seq));
922 
923         timespec_add_ns(ts_raw, nsecs_raw);
924         timespec_add_ns(ts_real, nsecs_real);
925 }
926 EXPORT_SYMBOL(getnstime_raw_and_real);
927 
928 #endif /* CONFIG_NTP_PPS */
929 
930 /**
931  * do_gettimeofday - Returns the time of day in a timeval
932  * @tv:         pointer to the timeval to be set
933  *
934  * NOTE: Users should be converted to using getnstimeofday()
935  */
936 void do_gettimeofday(struct timeval *tv)
937 {
938         struct timespec64 now;
939 
940         getnstimeofday64(&now);
941         tv->tv_sec = now.tv_sec;
942         tv->tv_usec = now.tv_nsec/1000;
943 }
944 EXPORT_SYMBOL(do_gettimeofday);
945 
946 /**
947  * do_settimeofday64 - Sets the time of day.
948  * @ts:     pointer to the timespec64 variable containing the new time
949  *
950  * Sets the time of day to the new time and update NTP and notify hrtimers
951  */
952 int do_settimeofday64(const struct timespec64 *ts)
953 {
954         struct timekeeper *tk = &tk_core.timekeeper;
955         struct timespec64 ts_delta, xt;
956         unsigned long flags;
957 
958         if (!timespec64_valid_strict(ts))
959                 return -EINVAL;
960 
961         raw_spin_lock_irqsave(&timekeeper_lock, flags);
962         write_seqcount_begin(&tk_core.seq);
963 
964         timekeeping_forward_now(tk);
965 
966         xt = tk_xtime(tk);
967         ts_delta.tv_sec = ts->tv_sec - xt.tv_sec;
968         ts_delta.tv_nsec = ts->tv_nsec - xt.tv_nsec;
969 
970         tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts_delta));
971 
972         tk_set_xtime(tk, ts);
973 
974         timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
975 
976         write_seqcount_end(&tk_core.seq);
977         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
978 
979         /* signal hrtimers about time change */
980         clock_was_set();
981 
982         return 0;
983 }
984 EXPORT_SYMBOL(do_settimeofday64);
985 
986 /**
987  * timekeeping_inject_offset - Adds or subtracts from the current time.
988  * @tv:         pointer to the timespec variable containing the offset
989  *
990  * Adds or subtracts an offset value from the current time.
991  */
992 int timekeeping_inject_offset(struct timespec *ts)
993 {
994         struct timekeeper *tk = &tk_core.timekeeper;
995         unsigned long flags;
996         struct timespec64 ts64, tmp;
997         int ret = 0;
998 
999         if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC)
1000                 return -EINVAL;
1001 
1002         ts64 = timespec_to_timespec64(*ts);
1003 
1004         raw_spin_lock_irqsave(&timekeeper_lock, flags);
1005         write_seqcount_begin(&tk_core.seq);
1006 
1007         timekeeping_forward_now(tk);
1008 
1009         /* Make sure the proposed value is valid */
1010         tmp = timespec64_add(tk_xtime(tk),  ts64);
1011         if (!timespec64_valid_strict(&tmp)) {
1012                 ret = -EINVAL;
1013                 goto error;
1014         }
1015 
1016         tk_xtime_add(tk, &ts64);
1017         tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts64));
1018 
1019 error: /* even if we error out, we forwarded the time, so call update */
1020         timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1021 
1022         write_seqcount_end(&tk_core.seq);
1023         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1024 
1025         /* signal hrtimers about time change */
1026         clock_was_set();
1027 
1028         return ret;
1029 }
1030 EXPORT_SYMBOL(timekeeping_inject_offset);
1031 
1032 
1033 /**
1034  * timekeeping_get_tai_offset - Returns current TAI offset from UTC
1035  *
1036  */
1037 s32 timekeeping_get_tai_offset(void)
1038 {
1039         struct timekeeper *tk = &tk_core.timekeeper;
1040         unsigned int seq;
1041         s32 ret;
1042 
1043         do {
1044                 seq = read_seqcount_begin(&tk_core.seq);
1045                 ret = tk->tai_offset;
1046         } while (read_seqcount_retry(&tk_core.seq, seq));
1047 
1048         return ret;
1049 }
1050 
1051 /**
1052  * __timekeeping_set_tai_offset - Lock free worker function
1053  *
1054  */
1055 static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset)
1056 {
1057         tk->tai_offset = tai_offset;
1058         tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0));
1059 }
1060 
1061 /**
1062  * timekeeping_set_tai_offset - Sets the current TAI offset from UTC
1063  *
1064  */
1065 void timekeeping_set_tai_offset(s32 tai_offset)
1066 {
1067         struct timekeeper *tk = &tk_core.timekeeper;
1068         unsigned long flags;
1069 
1070         raw_spin_lock_irqsave(&timekeeper_lock, flags);
1071         write_seqcount_begin(&tk_core.seq);
1072         __timekeeping_set_tai_offset(tk, tai_offset);
1073         timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
1074         write_seqcount_end(&tk_core.seq);
1075         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1076         clock_was_set();
1077 }
1078 
1079 /**
1080  * change_clocksource - Swaps clocksources if a new one is available
1081  *
1082  * Accumulates current time interval and initializes new clocksource
1083  */
1084 static int change_clocksource(void *data)
1085 {
1086         struct timekeeper *tk = &tk_core.timekeeper;
1087         struct clocksource *new, *old;
1088         unsigned long flags;
1089 
1090         new = (struct clocksource *) data;
1091 
1092         raw_spin_lock_irqsave(&timekeeper_lock, flags);
1093         write_seqcount_begin(&tk_core.seq);
1094 
1095         timekeeping_forward_now(tk);
1096         /*
1097          * If the cs is in module, get a module reference. Succeeds
1098          * for built-in code (owner == NULL) as well.
1099          */
1100         if (try_module_get(new->owner)) {
1101                 if (!new->enable || new->enable(new) == 0) {
1102                         old = tk->tkr_mono.clock;
1103                         tk_setup_internals(tk, new);
1104                         if (old->disable)
1105                                 old->disable(old);
1106                         module_put(old->owner);
1107                 } else {
1108                         module_put(new->owner);
1109                 }
1110         }
1111         timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1112 
1113         write_seqcount_end(&tk_core.seq);
1114         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1115 
1116         return 0;
1117 }
1118 
1119 /**
1120  * timekeeping_notify - Install a new clock source
1121  * @clock:              pointer to the clock source
1122  *
1123  * This function is called from clocksource.c after a new, better clock
1124  * source has been registered. The caller holds the clocksource_mutex.
1125  */
1126 int timekeeping_notify(struct clocksource *clock)
1127 {
1128         struct timekeeper *tk = &tk_core.timekeeper;
1129 
1130         if (tk->tkr_mono.clock == clock)
1131                 return 0;
1132         stop_machine(change_clocksource, clock, NULL);
1133         tick_clock_notify();
1134         return tk->tkr_mono.clock == clock ? 0 : -1;
1135 }
1136 
1137 /**
1138  * getrawmonotonic64 - Returns the raw monotonic time in a timespec
1139  * @ts:         pointer to the timespec64 to be set
1140  *
1141  * Returns the raw monotonic time (completely un-modified by ntp)
1142  */
1143 void getrawmonotonic64(struct timespec64 *ts)
1144 {
1145         struct timekeeper *tk = &tk_core.timekeeper;
1146         struct timespec64 ts64;
1147         unsigned long seq;
1148         s64 nsecs;
1149 
1150         do {
1151                 seq = read_seqcount_begin(&tk_core.seq);
1152                 nsecs = timekeeping_get_ns(&tk->tkr_raw);
1153                 ts64 = tk->raw_time;
1154 
1155         } while (read_seqcount_retry(&tk_core.seq, seq));
1156 
1157         timespec64_add_ns(&ts64, nsecs);
1158         *ts = ts64;
1159 }
1160 EXPORT_SYMBOL(getrawmonotonic64);
1161 
1162 
1163 /**
1164  * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
1165  */
1166 int timekeeping_valid_for_hres(void)
1167 {
1168         struct timekeeper *tk = &tk_core.timekeeper;
1169         unsigned long seq;
1170         int ret;
1171 
1172         do {
1173                 seq = read_seqcount_begin(&tk_core.seq);
1174 
1175                 ret = tk->tkr_mono.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
1176 
1177         } while (read_seqcount_retry(&tk_core.seq, seq));
1178 
1179         return ret;
1180 }
1181 
1182 /**
1183  * timekeeping_max_deferment - Returns max time the clocksource can be deferred
1184  */
1185 u64 timekeeping_max_deferment(void)
1186 {
1187         struct timekeeper *tk = &tk_core.timekeeper;
1188         unsigned long seq;
1189         u64 ret;
1190 
1191         do {
1192                 seq = read_seqcount_begin(&tk_core.seq);
1193 
1194                 ret = tk->tkr_mono.clock->max_idle_ns;
1195 
1196         } while (read_seqcount_retry(&tk_core.seq, seq));
1197 
1198         return ret;
1199 }
1200 
1201 /**
1202  * read_persistent_clock -  Return time from the persistent clock.
1203  *
1204  * Weak dummy function for arches that do not yet support it.
1205  * Reads the time from the battery backed persistent clock.
1206  * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported.
1207  *
1208  *  XXX - Do be sure to remove it once all arches implement it.
1209  */
1210 void __weak read_persistent_clock(struct timespec *ts)
1211 {
1212         ts->tv_sec = 0;
1213         ts->tv_nsec = 0;
1214 }
1215 
1216 void __weak read_persistent_clock64(struct timespec64 *ts64)
1217 {
1218         struct timespec ts;
1219 
1220         read_persistent_clock(&ts);
1221         *ts64 = timespec_to_timespec64(ts);
1222 }
1223 
1224 /**
1225  * read_boot_clock -  Return time of the system start.
1226  *
1227  * Weak dummy function for arches that do not yet support it.
1228  * Function to read the exact time the system has been started.
1229  * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported.
1230  *
1231  *  XXX - Do be sure to remove it once all arches implement it.
1232  */
1233 void __weak read_boot_clock(struct timespec *ts)
1234 {
1235         ts->tv_sec = 0;
1236         ts->tv_nsec = 0;
1237 }
1238 
1239 void __weak read_boot_clock64(struct timespec64 *ts64)
1240 {
1241         struct timespec ts;
1242 
1243         read_boot_clock(&ts);
1244         *ts64 = timespec_to_timespec64(ts);
1245 }
1246 
1247 /* Flag for if timekeeping_resume() has injected sleeptime */
1248 static bool sleeptime_injected;
1249 
1250 /* Flag for if there is a persistent clock on this platform */
1251 static bool persistent_clock_exists;
1252 
1253 /*
1254  * timekeeping_init - Initializes the clocksource and common timekeeping values
1255  */
1256 void __init timekeeping_init(void)
1257 {
1258         struct timekeeper *tk = &tk_core.timekeeper;
1259         struct clocksource *clock;
1260         unsigned long flags;
1261         struct timespec64 now, boot, tmp;
1262 
1263         read_persistent_clock64(&now);
1264         if (!timespec64_valid_strict(&now)) {
1265                 pr_warn("WARNING: Persistent clock returned invalid value!\n"
1266                         "         Check your CMOS/BIOS settings.\n");
1267                 now.tv_sec = 0;
1268                 now.tv_nsec = 0;
1269         } else if (now.tv_sec || now.tv_nsec)
1270                 persistent_clock_exists = true;
1271 
1272         read_boot_clock64(&boot);
1273         if (!timespec64_valid_strict(&boot)) {
1274                 pr_warn("WARNING: Boot clock returned invalid value!\n"
1275                         "         Check your CMOS/BIOS settings.\n");
1276                 boot.tv_sec = 0;
1277                 boot.tv_nsec = 0;
1278         }
1279 
1280         raw_spin_lock_irqsave(&timekeeper_lock, flags);
1281         write_seqcount_begin(&tk_core.seq);
1282         ntp_init();
1283 
1284         clock = clocksource_default_clock();
1285         if (clock->enable)
1286                 clock->enable(clock);
1287         tk_setup_internals(tk, clock);
1288 
1289         tk_set_xtime(tk, &now);
1290         tk->raw_time.tv_sec = 0;
1291         tk->raw_time.tv_nsec = 0;
1292         if (boot.tv_sec == 0 && boot.tv_nsec == 0)
1293                 boot = tk_xtime(tk);
1294 
1295         set_normalized_timespec64(&tmp, -boot.tv_sec, -boot.tv_nsec);
1296         tk_set_wall_to_mono(tk, tmp);
1297 
1298         timekeeping_update(tk, TK_MIRROR);
1299 
1300         write_seqcount_end(&tk_core.seq);
1301         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1302 }
1303 
1304 /* time in seconds when suspend began for persistent clock */
1305 static struct timespec64 timekeeping_suspend_time;
1306 
1307 /**
1308  * __timekeeping_inject_sleeptime - Internal function to add sleep interval
1309  * @delta: pointer to a timespec delta value
1310  *
1311  * Takes a timespec offset measuring a suspend interval and properly
1312  * adds the sleep offset to the timekeeping variables.
1313  */
1314 static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
1315                                            struct timespec64 *delta)
1316 {
1317         if (!timespec64_valid_strict(delta)) {
1318                 printk_deferred(KERN_WARNING
1319                                 "__timekeeping_inject_sleeptime: Invalid "
1320                                 "sleep delta value!\n");
1321                 return;
1322         }
1323         tk_xtime_add(tk, delta);
1324         tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta));
1325         tk_update_sleep_time(tk, timespec64_to_ktime(*delta));
1326         tk_debug_account_sleep_time(delta);
1327 }
1328 
1329 #if defined(CONFIG_PM_SLEEP) && defined(CONFIG_RTC_HCTOSYS_DEVICE)
1330 /**
1331  * We have three kinds of time sources to use for sleep time
1332  * injection, the preference order is:
1333  * 1) non-stop clocksource
1334  * 2) persistent clock (ie: RTC accessible when irqs are off)
1335  * 3) RTC
1336  *
1337  * 1) and 2) are used by timekeeping, 3) by RTC subsystem.
1338  * If system has neither 1) nor 2), 3) will be used finally.
1339  *
1340  *
1341  * If timekeeping has injected sleeptime via either 1) or 2),
1342  * 3) becomes needless, so in this case we don't need to call
1343  * rtc_resume(), and this is what timekeeping_rtc_skipresume()
1344  * means.
1345  */
1346 bool timekeeping_rtc_skipresume(void)
1347 {
1348         return sleeptime_injected;
1349 }
1350 
1351 /**
1352  * 1) can be determined whether to use or not only when doing
1353  * timekeeping_resume() which is invoked after rtc_suspend(),
1354  * so we can't skip rtc_suspend() surely if system has 1).
1355  *
1356  * But if system has 2), 2) will definitely be used, so in this
1357  * case we don't need to call rtc_suspend(), and this is what
1358  * timekeeping_rtc_skipsuspend() means.
1359  */
1360 bool timekeeping_rtc_skipsuspend(void)
1361 {
1362         return persistent_clock_exists;
1363 }
1364 
1365 /**
1366  * timekeeping_inject_sleeptime64 - Adds suspend interval to timeekeeping values
1367  * @delta: pointer to a timespec64 delta value
1368  *
1369  * This hook is for architectures that cannot support read_persistent_clock64
1370  * because their RTC/persistent clock is only accessible when irqs are enabled.
1371  * and also don't have an effective nonstop clocksource.
1372  *
1373  * This function should only be called by rtc_resume(), and allows
1374  * a suspend offset to be injected into the timekeeping values.
1375  */
1376 void timekeeping_inject_sleeptime64(struct timespec64 *delta)
1377 {
1378         struct timekeeper *tk = &tk_core.timekeeper;
1379         unsigned long flags;
1380 
1381         raw_spin_lock_irqsave(&timekeeper_lock, flags);
1382         write_seqcount_begin(&tk_core.seq);
1383 
1384         timekeeping_forward_now(tk);
1385 
1386         __timekeeping_inject_sleeptime(tk, delta);
1387 
1388         timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1389 
1390         write_seqcount_end(&tk_core.seq);
1391         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1392 
1393         /* signal hrtimers about time change */
1394         clock_was_set();
1395 }
1396 #endif
1397 
1398 /**
1399  * timekeeping_resume - Resumes the generic timekeeping subsystem.
1400  */
1401 void timekeeping_resume(void)
1402 {
1403         struct timekeeper *tk = &tk_core.timekeeper;
1404         struct clocksource *clock = tk->tkr_mono.clock;
1405         unsigned long flags;
1406         struct timespec64 ts_new, ts_delta;
1407         cycle_t cycle_now, cycle_delta;
1408 
1409         sleeptime_injected = false;
1410         read_persistent_clock64(&ts_new);
1411 
1412         clockevents_resume();
1413         clocksource_resume();
1414 
1415         raw_spin_lock_irqsave(&timekeeper_lock, flags);
1416         write_seqcount_begin(&tk_core.seq);
1417 
1418         /*
1419          * After system resumes, we need to calculate the suspended time and
1420          * compensate it for the OS time. There are 3 sources that could be
1421          * used: Nonstop clocksource during suspend, persistent clock and rtc
1422          * device.
1423          *
1424          * One specific platform may have 1 or 2 or all of them, and the
1425          * preference will be:
1426          *      suspend-nonstop clocksource -> persistent clock -> rtc
1427          * The less preferred source will only be tried if there is no better
1428          * usable source. The rtc part is handled separately in rtc core code.
1429          */
1430         cycle_now = tk_clock_read(&tk->tkr_mono);
1431         if ((clock->flags & CLOCK_SOURCE_SUSPEND_NONSTOP) &&
1432                 cycle_now > tk->tkr_mono.cycle_last) {
1433                 u64 num, max = ULLONG_MAX;
1434                 u32 mult = clock->mult;
1435                 u32 shift = clock->shift;
1436                 s64 nsec = 0;
1437 
1438                 cycle_delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last,
1439                                                 tk->tkr_mono.mask);
1440 
1441                 /*
1442                  * "cycle_delta * mutl" may cause 64 bits overflow, if the
1443                  * suspended time is too long. In that case we need do the
1444                  * 64 bits math carefully
1445                  */
1446                 do_div(max, mult);
1447                 if (cycle_delta > max) {
1448                         num = div64_u64(cycle_delta, max);
1449                         nsec = (((u64) max * mult) >> shift) * num;
1450                         cycle_delta -= num * max;
1451                 }
1452                 nsec += ((u64) cycle_delta * mult) >> shift;
1453 
1454                 ts_delta = ns_to_timespec64(nsec);
1455                 sleeptime_injected = true;
1456         } else if (timespec64_compare(&ts_new, &timekeeping_suspend_time) > 0) {
1457                 ts_delta = timespec64_sub(ts_new, timekeeping_suspend_time);
1458                 sleeptime_injected = true;
1459         }
1460 
1461         if (sleeptime_injected)
1462                 __timekeeping_inject_sleeptime(tk, &ts_delta);
1463 
1464         /* Re-base the last cycle value */
1465         tk->tkr_mono.cycle_last = cycle_now;
1466         tk->tkr_raw.cycle_last  = cycle_now;
1467 
1468         tk->ntp_error = 0;
1469         timekeeping_suspended = 0;
1470         timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
1471         write_seqcount_end(&tk_core.seq);
1472         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1473 
1474         touch_softlockup_watchdog();
1475 
1476         tick_resume();
1477         hrtimers_resume();
1478 }
1479 
1480 int timekeeping_suspend(void)
1481 {
1482         struct timekeeper *tk = &tk_core.timekeeper;
1483         unsigned long flags;
1484         struct timespec64               delta, delta_delta;
1485         static struct timespec64        old_delta;
1486 
1487         read_persistent_clock64(&timekeeping_suspend_time);
1488 
1489         /*
1490          * On some systems the persistent_clock can not be detected at
1491          * timekeeping_init by its return value, so if we see a valid
1492          * value returned, update the persistent_clock_exists flag.
1493          */
1494         if (timekeeping_suspend_time.tv_sec || timekeeping_suspend_time.tv_nsec)
1495                 persistent_clock_exists = true;
1496 
1497         raw_spin_lock_irqsave(&timekeeper_lock, flags);
1498         write_seqcount_begin(&tk_core.seq);
1499         timekeeping_forward_now(tk);
1500         timekeeping_suspended = 1;
1501 
1502         if (persistent_clock_exists) {
1503                 /*
1504                  * To avoid drift caused by repeated suspend/resumes,
1505                  * which each can add ~1 second drift error,
1506                  * try to compensate so the difference in system time
1507                  * and persistent_clock time stays close to constant.
1508                  */
1509                 delta = timespec64_sub(tk_xtime(tk), timekeeping_suspend_time);
1510                 delta_delta = timespec64_sub(delta, old_delta);
1511                 if (abs(delta_delta.tv_sec) >= 2) {
1512                         /*
1513                          * if delta_delta is too large, assume time correction
1514                          * has occurred and set old_delta to the current delta.
1515                          */
1516                         old_delta = delta;
1517                 } else {
1518                         /* Otherwise try to adjust old_system to compensate */
1519                         timekeeping_suspend_time =
1520                                 timespec64_add(timekeeping_suspend_time, delta_delta);
1521                 }
1522         }
1523 
1524         timekeeping_update(tk, TK_MIRROR);
1525         halt_fast_timekeeper(tk);
1526         write_seqcount_end(&tk_core.seq);
1527         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1528 
1529         tick_suspend();
1530         clocksource_suspend();
1531         clockevents_suspend();
1532 
1533         return 0;
1534 }
1535 
1536 /* sysfs resume/suspend bits for timekeeping */
1537 static struct syscore_ops timekeeping_syscore_ops = {
1538         .resume         = timekeeping_resume,
1539         .suspend        = timekeeping_suspend,
1540 };
1541 
1542 static int __init timekeeping_init_ops(void)
1543 {
1544         register_syscore_ops(&timekeeping_syscore_ops);
1545         return 0;
1546 }
1547 device_initcall(timekeeping_init_ops);
1548 
1549 /*
1550  * Apply a multiplier adjustment to the timekeeper
1551  */
1552 static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
1553                                                          s64 offset,
1554                                                          bool negative,
1555                                                          int adj_scale)
1556 {
1557         s64 interval = tk->cycle_interval;
1558         s32 mult_adj = 1;
1559 
1560         if (negative) {
1561                 mult_adj = -mult_adj;
1562                 interval = -interval;
1563                 offset  = -offset;
1564         }
1565         mult_adj <<= adj_scale;
1566         interval <<= adj_scale;
1567         offset <<= adj_scale;
1568 
1569         /*
1570          * So the following can be confusing.
1571          *
1572          * To keep things simple, lets assume mult_adj == 1 for now.
1573          *
1574          * When mult_adj != 1, remember that the interval and offset values
1575          * have been appropriately scaled so the math is the same.
1576          *
1577          * The basic idea here is that we're increasing the multiplier
1578          * by one, this causes the xtime_interval to be incremented by
1579          * one cycle_interval. This is because:
1580          *      xtime_interval = cycle_interval * mult
1581          * So if mult is being incremented by one:
1582          *      xtime_interval = cycle_interval * (mult + 1)
1583          * Its the same as:
1584          *      xtime_interval = (cycle_interval * mult) + cycle_interval
1585          * Which can be shortened to:
1586          *      xtime_interval += cycle_interval
1587          *
1588          * So offset stores the non-accumulated cycles. Thus the current
1589          * time (in shifted nanoseconds) is:
1590          *      now = (offset * adj) + xtime_nsec
1591          * Now, even though we're adjusting the clock frequency, we have
1592          * to keep time consistent. In other words, we can't jump back
1593          * in time, and we also want to avoid jumping forward in time.
1594          *
1595          * So given the same offset value, we need the time to be the same
1596          * both before and after the freq adjustment.
1597          *      now = (offset * adj_1) + xtime_nsec_1
1598          *      now = (offset * adj_2) + xtime_nsec_2
1599          * So:
1600          *      (offset * adj_1) + xtime_nsec_1 =
1601          *              (offset * adj_2) + xtime_nsec_2
1602          * And we know:
1603          *      adj_2 = adj_1 + 1
1604          * So:
1605          *      (offset * adj_1) + xtime_nsec_1 =
1606          *              (offset * (adj_1+1)) + xtime_nsec_2
1607          *      (offset * adj_1) + xtime_nsec_1 =
1608          *              (offset * adj_1) + offset + xtime_nsec_2
1609          * Canceling the sides:
1610          *      xtime_nsec_1 = offset + xtime_nsec_2
1611          * Which gives us:
1612          *      xtime_nsec_2 = xtime_nsec_1 - offset
1613          * Which simplfies to:
1614          *      xtime_nsec -= offset
1615          *
1616          * XXX - TODO: Doc ntp_error calculation.
1617          */
1618         if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) {
1619                 /* NTP adjustment caused clocksource mult overflow */
1620                 WARN_ON_ONCE(1);
1621                 return;
1622         }
1623 
1624         tk->tkr_mono.mult += mult_adj;
1625         tk->xtime_interval += interval;
1626         tk->tkr_mono.xtime_nsec -= offset;
1627         tk->ntp_error -= (interval - offset) << tk->ntp_error_shift;
1628 }
1629 
1630 /*
1631  * Calculate the multiplier adjustment needed to match the frequency
1632  * specified by NTP
1633  */
1634 static __always_inline void timekeeping_freqadjust(struct timekeeper *tk,
1635                                                         s64 offset)
1636 {
1637         s64 interval = tk->cycle_interval;
1638         s64 xinterval = tk->xtime_interval;
1639         s64 tick_error;
1640         bool negative;
1641         u32 adj;
1642 
1643         /* Remove any current error adj from freq calculation */
1644         if (tk->ntp_err_mult)
1645                 xinterval -= tk->cycle_interval;
1646 
1647         tk->ntp_tick = ntp_tick_length();
1648 
1649         /* Calculate current error per tick */
1650         tick_error = ntp_tick_length() >> tk->ntp_error_shift;
1651         tick_error -= (xinterval + tk->xtime_remainder);
1652 
1653         /* Don't worry about correcting it if its small */
1654         if (likely((tick_error >= 0) && (tick_error <= interval)))
1655                 return;
1656 
1657         /* preserve the direction of correction */
1658         negative = (tick_error < 0);
1659 
1660         /* Sort out the magnitude of the correction */
1661         tick_error = abs64(tick_error);
1662         for (adj = 0; tick_error > interval; adj++)
1663                 tick_error >>= 1;
1664 
1665         /* scale the corrections */
1666         timekeeping_apply_adjustment(tk, offset, negative, adj);
1667 }
1668 
1669 /*
1670  * Adjust the timekeeper's multiplier to the correct frequency
1671  * and also to reduce the accumulated error value.
1672  */
1673 static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
1674 {
1675         /* Correct for the current frequency error */
1676         timekeeping_freqadjust(tk, offset);
1677 
1678         /* Next make a small adjustment to fix any cumulative error */
1679         if (!tk->ntp_err_mult && (tk->ntp_error > 0)) {
1680                 tk->ntp_err_mult = 1;
1681                 timekeeping_apply_adjustment(tk, offset, 0, 0);
1682         } else if (tk->ntp_err_mult && (tk->ntp_error <= 0)) {
1683                 /* Undo any existing error adjustment */
1684                 timekeeping_apply_adjustment(tk, offset, 1, 0);
1685                 tk->ntp_err_mult = 0;
1686         }
1687 
1688         if (unlikely(tk->tkr_mono.clock->maxadj &&
1689                 (abs(tk->tkr_mono.mult - tk->tkr_mono.clock->mult)
1690                         > tk->tkr_mono.clock->maxadj))) {
1691                 printk_once(KERN_WARNING
1692                         "Adjusting %s more than 11%% (%ld vs %ld)\n",
1693                         tk->tkr_mono.clock->name, (long)tk->tkr_mono.mult,
1694                         (long)tk->tkr_mono.clock->mult + tk->tkr_mono.clock->maxadj);
1695         }
1696 
1697         /*
1698          * It may be possible that when we entered this function, xtime_nsec
1699          * was very small.  Further, if we're slightly speeding the clocksource
1700          * in the code above, its possible the required corrective factor to
1701          * xtime_nsec could cause it to underflow.
1702          *
1703          * Now, since we already accumulated the second, cannot simply roll
1704          * the accumulated second back, since the NTP subsystem has been
1705          * notified via second_overflow. So instead we push xtime_nsec forward
1706          * by the amount we underflowed, and add that amount into the error.
1707          *
1708          * We'll correct this error next time through this function, when
1709          * xtime_nsec is not as small.
1710          */
1711         if (unlikely((s64)tk->tkr_mono.xtime_nsec < 0)) {
1712                 s64 neg = -(s64)tk->tkr_mono.xtime_nsec;
1713                 tk->tkr_mono.xtime_nsec = 0;
1714                 tk->ntp_error += neg << tk->ntp_error_shift;
1715         }
1716 }
1717 
1718 /**
1719  * accumulate_nsecs_to_secs - Accumulates nsecs into secs
1720  *
1721  * Helper function that accumulates a the nsecs greater then a second
1722  * from the xtime_nsec field to the xtime_secs field.
1723  * It also calls into the NTP code to handle leapsecond processing.
1724  *
1725  */
1726 static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
1727 {
1728         u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
1729         unsigned int clock_set = 0;
1730 
1731         while (tk->tkr_mono.xtime_nsec >= nsecps) {
1732                 int leap;
1733 
1734                 tk->tkr_mono.xtime_nsec -= nsecps;
1735                 tk->xtime_sec++;
1736 
1737                 /* Figure out if its a leap sec and apply if needed */
1738                 leap = second_overflow(tk->xtime_sec);
1739                 if (unlikely(leap)) {
1740                         struct timespec64 ts;
1741 
1742                         tk->xtime_sec += leap;
1743 
1744                         ts.tv_sec = leap;
1745                         ts.tv_nsec = 0;
1746                         tk_set_wall_to_mono(tk,
1747                                 timespec64_sub(tk->wall_to_monotonic, ts));
1748 
1749                         __timekeeping_set_tai_offset(tk, tk->tai_offset - leap);
1750 
1751                         clock_set = TK_CLOCK_WAS_SET;
1752                 }
1753         }
1754         return clock_set;
1755 }
1756 
1757 /**
1758  * logarithmic_accumulation - shifted accumulation of cycles
1759  *
1760  * This functions accumulates a shifted interval of cycles into
1761  * into a shifted interval nanoseconds. Allows for O(log) accumulation
1762  * loop.
1763  *
1764  * Returns the unconsumed cycles.
1765  */
1766 static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset,
1767                                                 u32 shift,
1768                                                 unsigned int *clock_set)
1769 {
1770         cycle_t interval = tk->cycle_interval << shift;
1771         u64 raw_nsecs;
1772 
1773         /* If the offset is smaller then a shifted interval, do nothing */
1774         if (offset < interval)
1775                 return offset;
1776 
1777         /* Accumulate one shifted interval */
1778         offset -= interval;
1779         tk->tkr_mono.cycle_last += interval;
1780         tk->tkr_raw.cycle_last  += interval;
1781 
1782         tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift;
1783         *clock_set |= accumulate_nsecs_to_secs(tk);
1784 
1785         /* Accumulate raw time */
1786         raw_nsecs = (u64)tk->raw_interval << shift;
1787         raw_nsecs += tk->raw_time.tv_nsec;
1788         if (raw_nsecs >= NSEC_PER_SEC) {
1789                 u64 raw_secs = raw_nsecs;
1790                 raw_nsecs = do_div(raw_secs, NSEC_PER_SEC);
1791                 tk->raw_time.tv_sec += raw_secs;
1792         }
1793         tk->raw_time.tv_nsec = raw_nsecs;
1794 
1795         /* Accumulate error between NTP and clock interval */
1796         tk->ntp_error += tk->ntp_tick << shift;
1797         tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
1798                                                 (tk->ntp_error_shift + shift);
1799 
1800         return offset;
1801 }
1802 
1803 /**
1804  * update_wall_time - Uses the current clocksource to increment the wall time
1805  *
1806  */
1807 void update_wall_time(void)
1808 {
1809         struct timekeeper *real_tk = &tk_core.timekeeper;
1810         struct timekeeper *tk = &shadow_timekeeper;
1811         cycle_t offset;
1812         int shift = 0, maxshift;
1813         unsigned int clock_set = 0;
1814         unsigned long flags;
1815 
1816         raw_spin_lock_irqsave(&timekeeper_lock, flags);
1817 
1818         /* Make sure we're fully resumed: */
1819         if (unlikely(timekeeping_suspended))
1820                 goto out;
1821 
1822 #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
1823         offset = real_tk->cycle_interval;
1824 #else
1825         offset = clocksource_delta(tk_clock_read(&tk->tkr_mono),
1826                                    tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
1827 #endif
1828 
1829         /* Check if there's really nothing to do */
1830         if (offset < real_tk->cycle_interval)
1831                 goto out;
1832 
1833         /* Do some additional sanity checking */
1834         timekeeping_check_update(real_tk, offset);
1835 
1836         /*
1837          * With NO_HZ we may have to accumulate many cycle_intervals
1838          * (think "ticks") worth of time at once. To do this efficiently,
1839          * we calculate the largest doubling multiple of cycle_intervals
1840          * that is smaller than the offset.  We then accumulate that
1841          * chunk in one go, and then try to consume the next smaller
1842          * doubled multiple.
1843          */
1844         shift = ilog2(offset) - ilog2(tk->cycle_interval);
1845         shift = max(0, shift);
1846         /* Bound shift to one less than what overflows tick_length */
1847         maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;
1848         shift = min(shift, maxshift);
1849         while (offset >= tk->cycle_interval) {
1850                 offset = logarithmic_accumulation(tk, offset, shift,
1851                                                         &clock_set);
1852                 if (offset < tk->cycle_interval<<shift)
1853                         shift--;
1854         }
1855 
1856         /* correct the clock when NTP error is too big */
1857         timekeeping_adjust(tk, offset);
1858 
1859         /*
1860          * XXX This can be killed once everyone converts
1861          * to the new update_vsyscall.
1862          */
1863         old_vsyscall_fixup(tk);
1864 
1865         /*
1866          * Finally, make sure that after the rounding
1867          * xtime_nsec isn't larger than NSEC_PER_SEC
1868          */
1869         clock_set |= accumulate_nsecs_to_secs(tk);
1870 
1871         write_seqcount_begin(&tk_core.seq);
1872         /*
1873          * Update the real timekeeper.
1874          *
1875          * We could avoid this memcpy by switching pointers, but that
1876          * requires changes to all other timekeeper usage sites as
1877          * well, i.e. move the timekeeper pointer getter into the
1878          * spinlocked/seqcount protected sections. And we trade this
1879          * memcpy under the tk_core.seq against one before we start
1880          * updating.
1881          */
1882         memcpy(real_tk, tk, sizeof(*tk));
1883         timekeeping_update(real_tk, clock_set);
1884         write_seqcount_end(&tk_core.seq);
1885 out:
1886         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1887         if (clock_set)
1888                 /* Have to call _delayed version, since in irq context*/
1889                 clock_was_set_delayed();
1890 }
1891 
1892 /**
1893  * getboottime64 - Return the real time of system boot.
1894  * @ts:         pointer to the timespec64 to be set
1895  *
1896  * Returns the wall-time of boot in a timespec64.
1897  *
1898  * This is based on the wall_to_monotonic offset and the total suspend
1899  * time. Calls to settimeofday will affect the value returned (which
1900  * basically means that however wrong your real time clock is at boot time,
1901  * you get the right time here).
1902  */
1903 void getboottime64(struct timespec64 *ts)
1904 {
1905         struct timekeeper *tk = &tk_core.timekeeper;
1906         ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot);
1907 
1908         *ts = ktime_to_timespec64(t);
1909 }
1910 EXPORT_SYMBOL_GPL(getboottime64);
1911 
1912 unsigned long get_seconds(void)
1913 {
1914         struct timekeeper *tk = &tk_core.timekeeper;
1915 
1916         return tk->xtime_sec;
1917 }
1918 EXPORT_SYMBOL(get_seconds);
1919 
1920 struct timespec __current_kernel_time(void)
1921 {
1922         struct timekeeper *tk = &tk_core.timekeeper;
1923 
1924         return timespec64_to_timespec(tk_xtime(tk));
1925 }
1926 
1927 struct timespec current_kernel_time(void)
1928 {
1929         struct timekeeper *tk = &tk_core.timekeeper;
1930         struct timespec64 now;
1931         unsigned long seq;
1932 
1933         do {
1934                 seq = read_seqcount_begin(&tk_core.seq);
1935 
1936                 now = tk_xtime(tk);
1937         } while (read_seqcount_retry(&tk_core.seq, seq));
1938 
1939         return timespec64_to_timespec(now);
1940 }
1941 EXPORT_SYMBOL(current_kernel_time);
1942 
1943 struct timespec64 get_monotonic_coarse64(void)
1944 {
1945         struct timekeeper *tk = &tk_core.timekeeper;
1946         struct timespec64 now, mono;
1947         unsigned long seq;
1948 
1949         do {
1950                 seq = read_seqcount_begin(&tk_core.seq);
1951 
1952                 now = tk_xtime(tk);
1953                 mono = tk->wall_to_monotonic;
1954         } while (read_seqcount_retry(&tk_core.seq, seq));
1955 
1956         set_normalized_timespec64(&now, now.tv_sec + mono.tv_sec,
1957                                 now.tv_nsec + mono.tv_nsec);
1958 
1959         return now;
1960 }
1961 
1962 /*
1963  * Must hold jiffies_lock
1964  */
1965 void do_timer(unsigned long ticks)
1966 {
1967         jiffies_64 += ticks;
1968         calc_global_load(ticks);
1969 }
1970 
1971 /**
1972  * ktime_get_update_offsets_tick - hrtimer helper
1973  * @offs_real:  pointer to storage for monotonic -> realtime offset
1974  * @offs_boot:  pointer to storage for monotonic -> boottime offset
1975  * @offs_tai:   pointer to storage for monotonic -> clock tai offset
1976  *
1977  * Returns monotonic time at last tick and various offsets
1978  */
1979 ktime_t ktime_get_update_offsets_tick(ktime_t *offs_real, ktime_t *offs_boot,
1980                                                         ktime_t *offs_tai)
1981 {
1982         struct timekeeper *tk = &tk_core.timekeeper;
1983         unsigned int seq;
1984         ktime_t base;
1985         u64 nsecs;
1986 
1987         do {
1988                 seq = read_seqcount_begin(&tk_core.seq);
1989 
1990                 base = tk->tkr_mono.base;
1991                 nsecs = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;
1992 
1993                 *offs_real = tk->offs_real;
1994                 *offs_boot = tk->offs_boot;
1995                 *offs_tai = tk->offs_tai;
1996         } while (read_seqcount_retry(&tk_core.seq, seq));
1997 
1998         return ktime_add_ns(base, nsecs);
1999 }
2000 
2001 #ifdef CONFIG_HIGH_RES_TIMERS
2002 /**
2003  * ktime_get_update_offsets_now - hrtimer helper
2004  * @offs_real:  pointer to storage for monotonic -> realtime offset
2005  * @offs_boot:  pointer to storage for monotonic -> boottime offset
2006  * @offs_tai:   pointer to storage for monotonic -> clock tai offset
2007  *
2008  * Returns current monotonic time and updates the offsets
2009  * Called from hrtimer_interrupt() or retrigger_next_event()
2010  */
2011 ktime_t ktime_get_update_offsets_now(ktime_t *offs_real, ktime_t *offs_boot,
2012                                                         ktime_t *offs_tai)
2013 {
2014         struct timekeeper *tk = &tk_core.timekeeper;
2015         unsigned int seq;
2016         ktime_t base;
2017         u64 nsecs;
2018 
2019         do {
2020                 seq = read_seqcount_begin(&tk_core.seq);
2021 
2022                 base = tk->tkr_mono.base;
2023                 nsecs = timekeeping_get_ns(&tk->tkr_mono);
2024 
2025                 *offs_real = tk->offs_real;
2026                 *offs_boot = tk->offs_boot;
2027                 *offs_tai = tk->offs_tai;
2028         } while (read_seqcount_retry(&tk_core.seq, seq));
2029 
2030         return ktime_add_ns(base, nsecs);
2031 }
2032 #endif
2033 
2034 /**
2035  * do_adjtimex() - Accessor function to NTP __do_adjtimex function
2036  */
2037 int do_adjtimex(struct timex *txc)
2038 {
2039         struct timekeeper *tk = &tk_core.timekeeper;
2040         unsigned long flags;
2041         struct timespec64 ts;
2042         s32 orig_tai, tai;
2043         int ret;
2044 
2045         /* Validate the data before disabling interrupts */
2046         ret = ntp_validate_timex(txc);
2047         if (ret)
2048                 return ret;
2049 
2050         if (txc->modes & ADJ_SETOFFSET) {
2051                 struct timespec delta;
2052                 delta.tv_sec  = txc->time.tv_sec;
2053                 delta.tv_nsec = txc->time.tv_usec;
2054                 if (!(txc->modes & ADJ_NANO))
2055                         delta.tv_nsec *= 1000;
2056                 ret = timekeeping_inject_offset(&delta);
2057                 if (ret)
2058                         return ret;
2059         }
2060 
2061         getnstimeofday64(&ts);
2062 
2063         raw_spin_lock_irqsave(&timekeeper_lock, flags);
2064         write_seqcount_begin(&tk_core.seq);
2065 
2066         orig_tai = tai = tk->tai_offset;
2067         ret = __do_adjtimex(txc, &ts, &tai);
2068 
2069         if (tai != orig_tai) {
2070                 __timekeeping_set_tai_offset(tk, tai);
2071                 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
2072         }
2073         write_seqcount_end(&tk_core.seq);
2074         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
2075 
2076         if (tai != orig_tai)
2077                 clock_was_set();
2078 
2079         ntp_notify_cmos_timer();
2080 
2081         return ret;
2082 }
2083 
2084 #ifdef CONFIG_NTP_PPS
2085 /**
2086  * hardpps() - Accessor function to NTP __hardpps function
2087  */
2088 void hardpps(const struct timespec *phase_ts, const struct timespec *raw_ts)
2089 {
2090         unsigned long flags;
2091 
2092         raw_spin_lock_irqsave(&timekeeper_lock, flags);
2093         write_seqcount_begin(&tk_core.seq);
2094 
2095         __hardpps(phase_ts, raw_ts);
2096 
2097         write_seqcount_end(&tk_core.seq);
2098         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
2099 }
2100 EXPORT_SYMBOL(hardpps);
2101 #endif
2102 
2103 /**
2104  * xtime_update() - advances the timekeeping infrastructure
2105  * @ticks:      number of ticks, that have elapsed since the last call.
2106  *
2107  * Must be called with interrupts disabled.
2108  */
2109 void xtime_update(unsigned long ticks)
2110 {
2111         write_seqlock(&jiffies_lock);
2112         do_timer(ticks);
2113         write_sequnlock(&jiffies_lock);
2114         update_wall_time();
2115 }
2116 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp