~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/time/timekeeping.c

Version: ~ [ linux-5.0-rc6 ] ~ [ linux-4.20.10 ] ~ [ linux-4.19.23 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.101 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.158 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.174 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.134 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.63 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  *  Kernel timekeeping code and accessor functions. Based on code from
  4  *  timer.c, moved in commit 8524070b7982.
  5  */
  6 #include <linux/timekeeper_internal.h>
  7 #include <linux/module.h>
  8 #include <linux/interrupt.h>
  9 #include <linux/percpu.h>
 10 #include <linux/init.h>
 11 #include <linux/mm.h>
 12 #include <linux/nmi.h>
 13 #include <linux/sched.h>
 14 #include <linux/sched/loadavg.h>
 15 #include <linux/sched/clock.h>
 16 #include <linux/syscore_ops.h>
 17 #include <linux/clocksource.h>
 18 #include <linux/jiffies.h>
 19 #include <linux/time.h>
 20 #include <linux/tick.h>
 21 #include <linux/stop_machine.h>
 22 #include <linux/pvclock_gtod.h>
 23 #include <linux/compiler.h>
 24 #include <linux/ccsecurity.h>
 25 
 26 #include "tick-internal.h"
 27 #include "ntp_internal.h"
 28 #include "timekeeping_internal.h"
 29 
 30 #define TK_CLEAR_NTP            (1 << 0)
 31 #define TK_MIRROR               (1 << 1)
 32 #define TK_CLOCK_WAS_SET        (1 << 2)
 33 
 34 enum timekeeping_adv_mode {
 35         /* Update timekeeper when a tick has passed */
 36         TK_ADV_TICK,
 37 
 38         /* Update timekeeper on a direct frequency change */
 39         TK_ADV_FREQ
 40 };
 41 
 42 /*
 43  * The most important data for readout fits into a single 64 byte
 44  * cache line.
 45  */
 46 static struct {
 47         seqcount_t              seq;
 48         struct timekeeper       timekeeper;
 49 } tk_core ____cacheline_aligned = {
 50         .seq = SEQCNT_ZERO(tk_core.seq),
 51 };
 52 
 53 static DEFINE_RAW_SPINLOCK(timekeeper_lock);
 54 static struct timekeeper shadow_timekeeper;
 55 
 56 /**
 57  * struct tk_fast - NMI safe timekeeper
 58  * @seq:        Sequence counter for protecting updates. The lowest bit
 59  *              is the index for the tk_read_base array
 60  * @base:       tk_read_base array. Access is indexed by the lowest bit of
 61  *              @seq.
 62  *
 63  * See @update_fast_timekeeper() below.
 64  */
 65 struct tk_fast {
 66         seqcount_t              seq;
 67         struct tk_read_base     base[2];
 68 };
 69 
 70 /* Suspend-time cycles value for halted fast timekeeper. */
 71 static u64 cycles_at_suspend;
 72 
 73 static u64 dummy_clock_read(struct clocksource *cs)
 74 {
 75         return cycles_at_suspend;
 76 }
 77 
 78 static struct clocksource dummy_clock = {
 79         .read = dummy_clock_read,
 80 };
 81 
 82 static struct tk_fast tk_fast_mono ____cacheline_aligned = {
 83         .base[0] = { .clock = &dummy_clock, },
 84         .base[1] = { .clock = &dummy_clock, },
 85 };
 86 
 87 static struct tk_fast tk_fast_raw  ____cacheline_aligned = {
 88         .base[0] = { .clock = &dummy_clock, },
 89         .base[1] = { .clock = &dummy_clock, },
 90 };
 91 
 92 /* flag for if timekeeping is suspended */
 93 int __read_mostly timekeeping_suspended;
 94 
 95 static inline void tk_normalize_xtime(struct timekeeper *tk)
 96 {
 97         while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) {
 98                 tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
 99                 tk->xtime_sec++;
100         }
101         while (tk->tkr_raw.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_raw.shift)) {
102                 tk->tkr_raw.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_raw.shift;
103                 tk->raw_sec++;
104         }
105 }
106 
107 static inline struct timespec64 tk_xtime(const struct timekeeper *tk)
108 {
109         struct timespec64 ts;
110 
111         ts.tv_sec = tk->xtime_sec;
112         ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
113         return ts;
114 }
115 
116 static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts)
117 {
118         tk->xtime_sec = ts->tv_sec;
119         tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift;
120 }
121 
122 static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts)
123 {
124         tk->xtime_sec += ts->tv_sec;
125         tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift;
126         tk_normalize_xtime(tk);
127 }
128 
129 static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm)
130 {
131         struct timespec64 tmp;
132 
133         /*
134          * Verify consistency of: offset_real = -wall_to_monotonic
135          * before modifying anything
136          */
137         set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec,
138                                         -tk->wall_to_monotonic.tv_nsec);
139         WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp));
140         tk->wall_to_monotonic = wtm;
141         set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec);
142         tk->offs_real = timespec64_to_ktime(tmp);
143         tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0));
144 }
145 
146 static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)
147 {
148         tk->offs_boot = ktime_add(tk->offs_boot, delta);
149 }
150 
151 /*
152  * tk_clock_read - atomic clocksource read() helper
153  *
154  * This helper is necessary to use in the read paths because, while the
155  * seqlock ensures we don't return a bad value while structures are updated,
156  * it doesn't protect from potential crashes. There is the possibility that
157  * the tkr's clocksource may change between the read reference, and the
158  * clock reference passed to the read function.  This can cause crashes if
159  * the wrong clocksource is passed to the wrong read function.
160  * This isn't necessary to use when holding the timekeeper_lock or doing
161  * a read of the fast-timekeeper tkrs (which is protected by its own locking
162  * and update logic).
163  */
164 static inline u64 tk_clock_read(const struct tk_read_base *tkr)
165 {
166         struct clocksource *clock = READ_ONCE(tkr->clock);
167 
168         return clock->read(clock);
169 }
170 
171 #ifdef CONFIG_DEBUG_TIMEKEEPING
172 #define WARNING_FREQ (HZ*300) /* 5 minute rate-limiting */
173 
174 static void timekeeping_check_update(struct timekeeper *tk, u64 offset)
175 {
176 
177         u64 max_cycles = tk->tkr_mono.clock->max_cycles;
178         const char *name = tk->tkr_mono.clock->name;
179 
180         if (offset > max_cycles) {
181                 printk_deferred("WARNING: timekeeping: Cycle offset (%lld) is larger than allowed by the '%s' clock's max_cycles value (%lld): time overflow danger\n",
182                                 offset, name, max_cycles);
183                 printk_deferred("         timekeeping: Your kernel is sick, but tries to cope by capping time updates\n");
184         } else {
185                 if (offset > (max_cycles >> 1)) {
186                         printk_deferred("INFO: timekeeping: Cycle offset (%lld) is larger than the '%s' clock's 50%% safety margin (%lld)\n",
187                                         offset, name, max_cycles >> 1);
188                         printk_deferred("      timekeeping: Your kernel is still fine, but is feeling a bit nervous\n");
189                 }
190         }
191 
192         if (tk->underflow_seen) {
193                 if (jiffies - tk->last_warning > WARNING_FREQ) {
194                         printk_deferred("WARNING: Underflow in clocksource '%s' observed, time update ignored.\n", name);
195                         printk_deferred("         Please report this, consider using a different clocksource, if possible.\n");
196                         printk_deferred("         Your kernel is probably still fine.\n");
197                         tk->last_warning = jiffies;
198                 }
199                 tk->underflow_seen = 0;
200         }
201 
202         if (tk->overflow_seen) {
203                 if (jiffies - tk->last_warning > WARNING_FREQ) {
204                         printk_deferred("WARNING: Overflow in clocksource '%s' observed, time update capped.\n", name);
205                         printk_deferred("         Please report this, consider using a different clocksource, if possible.\n");
206                         printk_deferred("         Your kernel is probably still fine.\n");
207                         tk->last_warning = jiffies;
208                 }
209                 tk->overflow_seen = 0;
210         }
211 }
212 
213 static inline u64 timekeeping_get_delta(const struct tk_read_base *tkr)
214 {
215         struct timekeeper *tk = &tk_core.timekeeper;
216         u64 now, last, mask, max, delta;
217         unsigned int seq;
218 
219         /*
220          * Since we're called holding a seqlock, the data may shift
221          * under us while we're doing the calculation. This can cause
222          * false positives, since we'd note a problem but throw the
223          * results away. So nest another seqlock here to atomically
224          * grab the points we are checking with.
225          */
226         do {
227                 seq = read_seqcount_begin(&tk_core.seq);
228                 now = tk_clock_read(tkr);
229                 last = tkr->cycle_last;
230                 mask = tkr->mask;
231                 max = tkr->clock->max_cycles;
232         } while (read_seqcount_retry(&tk_core.seq, seq));
233 
234         delta = clocksource_delta(now, last, mask);
235 
236         /*
237          * Try to catch underflows by checking if we are seeing small
238          * mask-relative negative values.
239          */
240         if (unlikely((~delta & mask) < (mask >> 3))) {
241                 tk->underflow_seen = 1;
242                 delta = 0;
243         }
244 
245         /* Cap delta value to the max_cycles values to avoid mult overflows */
246         if (unlikely(delta > max)) {
247                 tk->overflow_seen = 1;
248                 delta = tkr->clock->max_cycles;
249         }
250 
251         return delta;
252 }
253 #else
254 static inline void timekeeping_check_update(struct timekeeper *tk, u64 offset)
255 {
256 }
257 static inline u64 timekeeping_get_delta(const struct tk_read_base *tkr)
258 {
259         u64 cycle_now, delta;
260 
261         /* read clocksource */
262         cycle_now = tk_clock_read(tkr);
263 
264         /* calculate the delta since the last update_wall_time */
265         delta = clocksource_delta(cycle_now, tkr->cycle_last, tkr->mask);
266 
267         return delta;
268 }
269 #endif
270 
271 /**
272  * tk_setup_internals - Set up internals to use clocksource clock.
273  *
274  * @tk:         The target timekeeper to setup.
275  * @clock:              Pointer to clocksource.
276  *
277  * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment
278  * pair and interval request.
279  *
280  * Unless you're the timekeeping code, you should not be using this!
281  */
282 static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
283 {
284         u64 interval;
285         u64 tmp, ntpinterval;
286         struct clocksource *old_clock;
287 
288         ++tk->cs_was_changed_seq;
289         old_clock = tk->tkr_mono.clock;
290         tk->tkr_mono.clock = clock;
291         tk->tkr_mono.mask = clock->mask;
292         tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono);
293 
294         tk->tkr_raw.clock = clock;
295         tk->tkr_raw.mask = clock->mask;
296         tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last;
297 
298         /* Do the ns -> cycle conversion first, using original mult */
299         tmp = NTP_INTERVAL_LENGTH;
300         tmp <<= clock->shift;
301         ntpinterval = tmp;
302         tmp += clock->mult/2;
303         do_div(tmp, clock->mult);
304         if (tmp == 0)
305                 tmp = 1;
306 
307         interval = (u64) tmp;
308         tk->cycle_interval = interval;
309 
310         /* Go back from cycles -> shifted ns */
311         tk->xtime_interval = interval * clock->mult;
312         tk->xtime_remainder = ntpinterval - tk->xtime_interval;
313         tk->raw_interval = interval * clock->mult;
314 
315          /* if changing clocks, convert xtime_nsec shift units */
316         if (old_clock) {
317                 int shift_change = clock->shift - old_clock->shift;
318                 if (shift_change < 0) {
319                         tk->tkr_mono.xtime_nsec >>= -shift_change;
320                         tk->tkr_raw.xtime_nsec >>= -shift_change;
321                 } else {
322                         tk->tkr_mono.xtime_nsec <<= shift_change;
323                         tk->tkr_raw.xtime_nsec <<= shift_change;
324                 }
325         }
326 
327         tk->tkr_mono.shift = clock->shift;
328         tk->tkr_raw.shift = clock->shift;
329 
330         tk->ntp_error = 0;
331         tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
332         tk->ntp_tick = ntpinterval << tk->ntp_error_shift;
333 
334         /*
335          * The timekeeper keeps its own mult values for the currently
336          * active clocksource. These value will be adjusted via NTP
337          * to counteract clock drifting.
338          */
339         tk->tkr_mono.mult = clock->mult;
340         tk->tkr_raw.mult = clock->mult;
341         tk->ntp_err_mult = 0;
342         tk->skip_second_overflow = 0;
343 }
344 
345 /* Timekeeper helper functions. */
346 
347 #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
348 static u32 default_arch_gettimeoffset(void) { return 0; }
349 u32 (*arch_gettimeoffset)(void) = default_arch_gettimeoffset;
350 #else
351 static inline u32 arch_gettimeoffset(void) { return 0; }
352 #endif
353 
354 static inline u64 timekeeping_delta_to_ns(const struct tk_read_base *tkr, u64 delta)
355 {
356         u64 nsec;
357 
358         nsec = delta * tkr->mult + tkr->xtime_nsec;
359         nsec >>= tkr->shift;
360 
361         /* If arch requires, add in get_arch_timeoffset() */
362         return nsec + arch_gettimeoffset();
363 }
364 
365 static inline u64 timekeeping_get_ns(const struct tk_read_base *tkr)
366 {
367         u64 delta;
368 
369         delta = timekeeping_get_delta(tkr);
370         return timekeeping_delta_to_ns(tkr, delta);
371 }
372 
373 static inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles)
374 {
375         u64 delta;
376 
377         /* calculate the delta since the last update_wall_time */
378         delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask);
379         return timekeeping_delta_to_ns(tkr, delta);
380 }
381 
382 /**
383  * update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper.
384  * @tkr: Timekeeping readout base from which we take the update
385  *
386  * We want to use this from any context including NMI and tracing /
387  * instrumenting the timekeeping code itself.
388  *
389  * Employ the latch technique; see @raw_write_seqcount_latch.
390  *
391  * So if a NMI hits the update of base[0] then it will use base[1]
392  * which is still consistent. In the worst case this can result is a
393  * slightly wrong timestamp (a few nanoseconds). See
394  * @ktime_get_mono_fast_ns.
395  */
396 static void update_fast_timekeeper(const struct tk_read_base *tkr,
397                                    struct tk_fast *tkf)
398 {
399         struct tk_read_base *base = tkf->base;
400 
401         /* Force readers off to base[1] */
402         raw_write_seqcount_latch(&tkf->seq);
403 
404         /* Update base[0] */
405         memcpy(base, tkr, sizeof(*base));
406 
407         /* Force readers back to base[0] */
408         raw_write_seqcount_latch(&tkf->seq);
409 
410         /* Update base[1] */
411         memcpy(base + 1, base, sizeof(*base));
412 }
413 
414 /**
415  * ktime_get_mono_fast_ns - Fast NMI safe access to clock monotonic
416  *
417  * This timestamp is not guaranteed to be monotonic across an update.
418  * The timestamp is calculated by:
419  *
420  *      now = base_mono + clock_delta * slope
421  *
422  * So if the update lowers the slope, readers who are forced to the
423  * not yet updated second array are still using the old steeper slope.
424  *
425  * tmono
426  * ^
427  * |    o  n
428  * |   o n
429  * |  u
430  * | o
431  * |o
432  * |12345678---> reader order
433  *
434  * o = old slope
435  * u = update
436  * n = new slope
437  *
438  * So reader 6 will observe time going backwards versus reader 5.
439  *
440  * While other CPUs are likely to be able observe that, the only way
441  * for a CPU local observation is when an NMI hits in the middle of
442  * the update. Timestamps taken from that NMI context might be ahead
443  * of the following timestamps. Callers need to be aware of that and
444  * deal with it.
445  */
446 static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf)
447 {
448         struct tk_read_base *tkr;
449         unsigned int seq;
450         u64 now;
451 
452         do {
453                 seq = raw_read_seqcount_latch(&tkf->seq);
454                 tkr = tkf->base + (seq & 0x01);
455                 now = ktime_to_ns(tkr->base);
456 
457                 now += timekeeping_delta_to_ns(tkr,
458                                 clocksource_delta(
459                                         tk_clock_read(tkr),
460                                         tkr->cycle_last,
461                                         tkr->mask));
462         } while (read_seqcount_retry(&tkf->seq, seq));
463 
464         return now;
465 }
466 
467 u64 ktime_get_mono_fast_ns(void)
468 {
469         return __ktime_get_fast_ns(&tk_fast_mono);
470 }
471 EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns);
472 
473 u64 ktime_get_raw_fast_ns(void)
474 {
475         return __ktime_get_fast_ns(&tk_fast_raw);
476 }
477 EXPORT_SYMBOL_GPL(ktime_get_raw_fast_ns);
478 
479 /**
480  * ktime_get_boot_fast_ns - NMI safe and fast access to boot clock.
481  *
482  * To keep it NMI safe since we're accessing from tracing, we're not using a
483  * separate timekeeper with updates to monotonic clock and boot offset
484  * protected with seqlocks. This has the following minor side effects:
485  *
486  * (1) Its possible that a timestamp be taken after the boot offset is updated
487  * but before the timekeeper is updated. If this happens, the new boot offset
488  * is added to the old timekeeping making the clock appear to update slightly
489  * earlier:
490  *    CPU 0                                        CPU 1
491  *    timekeeping_inject_sleeptime64()
492  *    __timekeeping_inject_sleeptime(tk, delta);
493  *                                                 timestamp();
494  *    timekeeping_update(tk, TK_CLEAR_NTP...);
495  *
496  * (2) On 32-bit systems, the 64-bit boot offset (tk->offs_boot) may be
497  * partially updated.  Since the tk->offs_boot update is a rare event, this
498  * should be a rare occurrence which postprocessing should be able to handle.
499  */
500 u64 notrace ktime_get_boot_fast_ns(void)
501 {
502         struct timekeeper *tk = &tk_core.timekeeper;
503 
504         return (ktime_get_mono_fast_ns() + ktime_to_ns(tk->offs_boot));
505 }
506 EXPORT_SYMBOL_GPL(ktime_get_boot_fast_ns);
507 
508 
509 /*
510  * See comment for __ktime_get_fast_ns() vs. timestamp ordering
511  */
512 static __always_inline u64 __ktime_get_real_fast_ns(struct tk_fast *tkf)
513 {
514         struct tk_read_base *tkr;
515         unsigned int seq;
516         u64 now;
517 
518         do {
519                 seq = raw_read_seqcount_latch(&tkf->seq);
520                 tkr = tkf->base + (seq & 0x01);
521                 now = ktime_to_ns(tkr->base_real);
522 
523                 now += timekeeping_delta_to_ns(tkr,
524                                 clocksource_delta(
525                                         tk_clock_read(tkr),
526                                         tkr->cycle_last,
527                                         tkr->mask));
528         } while (read_seqcount_retry(&tkf->seq, seq));
529 
530         return now;
531 }
532 
533 /**
534  * ktime_get_real_fast_ns: - NMI safe and fast access to clock realtime.
535  */
536 u64 ktime_get_real_fast_ns(void)
537 {
538         return __ktime_get_real_fast_ns(&tk_fast_mono);
539 }
540 EXPORT_SYMBOL_GPL(ktime_get_real_fast_ns);
541 
542 /**
543  * halt_fast_timekeeper - Prevent fast timekeeper from accessing clocksource.
544  * @tk: Timekeeper to snapshot.
545  *
546  * It generally is unsafe to access the clocksource after timekeeping has been
547  * suspended, so take a snapshot of the readout base of @tk and use it as the
548  * fast timekeeper's readout base while suspended.  It will return the same
549  * number of cycles every time until timekeeping is resumed at which time the
550  * proper readout base for the fast timekeeper will be restored automatically.
551  */
552 static void halt_fast_timekeeper(const struct timekeeper *tk)
553 {
554         static struct tk_read_base tkr_dummy;
555         const struct tk_read_base *tkr = &tk->tkr_mono;
556 
557         memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
558         cycles_at_suspend = tk_clock_read(tkr);
559         tkr_dummy.clock = &dummy_clock;
560         tkr_dummy.base_real = tkr->base + tk->offs_real;
561         update_fast_timekeeper(&tkr_dummy, &tk_fast_mono);
562 
563         tkr = &tk->tkr_raw;
564         memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
565         tkr_dummy.clock = &dummy_clock;
566         update_fast_timekeeper(&tkr_dummy, &tk_fast_raw);
567 }
568 
569 static RAW_NOTIFIER_HEAD(pvclock_gtod_chain);
570 
571 static void update_pvclock_gtod(struct timekeeper *tk, bool was_set)
572 {
573         raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk);
574 }
575 
576 /**
577  * pvclock_gtod_register_notifier - register a pvclock timedata update listener
578  */
579 int pvclock_gtod_register_notifier(struct notifier_block *nb)
580 {
581         struct timekeeper *tk = &tk_core.timekeeper;
582         unsigned long flags;
583         int ret;
584 
585         raw_spin_lock_irqsave(&timekeeper_lock, flags);
586         ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb);
587         update_pvclock_gtod(tk, true);
588         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
589 
590         return ret;
591 }
592 EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier);
593 
594 /**
595  * pvclock_gtod_unregister_notifier - unregister a pvclock
596  * timedata update listener
597  */
598 int pvclock_gtod_unregister_notifier(struct notifier_block *nb)
599 {
600         unsigned long flags;
601         int ret;
602 
603         raw_spin_lock_irqsave(&timekeeper_lock, flags);
604         ret = raw_notifier_chain_unregister(&pvclock_gtod_chain, nb);
605         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
606 
607         return ret;
608 }
609 EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier);
610 
611 /*
612  * tk_update_leap_state - helper to update the next_leap_ktime
613  */
614 static inline void tk_update_leap_state(struct timekeeper *tk)
615 {
616         tk->next_leap_ktime = ntp_get_next_leap();
617         if (tk->next_leap_ktime != KTIME_MAX)
618                 /* Convert to monotonic time */
619                 tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real);
620 }
621 
622 /*
623  * Update the ktime_t based scalar nsec members of the timekeeper
624  */
625 static inline void tk_update_ktime_data(struct timekeeper *tk)
626 {
627         u64 seconds;
628         u32 nsec;
629 
630         /*
631          * The xtime based monotonic readout is:
632          *      nsec = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec + now();
633          * The ktime based monotonic readout is:
634          *      nsec = base_mono + now();
635          * ==> base_mono = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec
636          */
637         seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec);
638         nsec = (u32) tk->wall_to_monotonic.tv_nsec;
639         tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec);
640 
641         /*
642          * The sum of the nanoseconds portions of xtime and
643          * wall_to_monotonic can be greater/equal one second. Take
644          * this into account before updating tk->ktime_sec.
645          */
646         nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
647         if (nsec >= NSEC_PER_SEC)
648                 seconds++;
649         tk->ktime_sec = seconds;
650 
651         /* Update the monotonic raw base */
652         tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC);
653 }
654 
655 /* must hold timekeeper_lock */
656 static void timekeeping_update(struct timekeeper *tk, unsigned int action)
657 {
658         if (action & TK_CLEAR_NTP) {
659                 tk->ntp_error = 0;
660                 ntp_clear();
661         }
662 
663         tk_update_leap_state(tk);
664         tk_update_ktime_data(tk);
665 
666         update_vsyscall(tk);
667         update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET);
668 
669         tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real;
670         update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono);
671         update_fast_timekeeper(&tk->tkr_raw,  &tk_fast_raw);
672 
673         if (action & TK_CLOCK_WAS_SET)
674                 tk->clock_was_set_seq++;
675         /*
676          * The mirroring of the data to the shadow-timekeeper needs
677          * to happen last here to ensure we don't over-write the
678          * timekeeper structure on the next update with stale data
679          */
680         if (action & TK_MIRROR)
681                 memcpy(&shadow_timekeeper, &tk_core.timekeeper,
682                        sizeof(tk_core.timekeeper));
683 }
684 
685 /**
686  * timekeeping_forward_now - update clock to the current time
687  *
688  * Forward the current clock to update its state since the last call to
689  * update_wall_time(). This is useful before significant clock changes,
690  * as it avoids having to deal with this time offset explicitly.
691  */
692 static void timekeeping_forward_now(struct timekeeper *tk)
693 {
694         u64 cycle_now, delta;
695 
696         cycle_now = tk_clock_read(&tk->tkr_mono);
697         delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
698         tk->tkr_mono.cycle_last = cycle_now;
699         tk->tkr_raw.cycle_last  = cycle_now;
700 
701         tk->tkr_mono.xtime_nsec += delta * tk->tkr_mono.mult;
702 
703         /* If arch requires, add in get_arch_timeoffset() */
704         tk->tkr_mono.xtime_nsec += (u64)arch_gettimeoffset() << tk->tkr_mono.shift;
705 
706 
707         tk->tkr_raw.xtime_nsec += delta * tk->tkr_raw.mult;
708 
709         /* If arch requires, add in get_arch_timeoffset() */
710         tk->tkr_raw.xtime_nsec += (u64)arch_gettimeoffset() << tk->tkr_raw.shift;
711 
712         tk_normalize_xtime(tk);
713 }
714 
715 /**
716  * ktime_get_real_ts64 - Returns the time of day in a timespec64.
717  * @ts:         pointer to the timespec to be set
718  *
719  * Returns the time of day in a timespec64 (WARN if suspended).
720  */
721 void ktime_get_real_ts64(struct timespec64 *ts)
722 {
723         struct timekeeper *tk = &tk_core.timekeeper;
724         unsigned long seq;
725         u64 nsecs;
726 
727         WARN_ON(timekeeping_suspended);
728 
729         do {
730                 seq = read_seqcount_begin(&tk_core.seq);
731 
732                 ts->tv_sec = tk->xtime_sec;
733                 nsecs = timekeeping_get_ns(&tk->tkr_mono);
734 
735         } while (read_seqcount_retry(&tk_core.seq, seq));
736 
737         ts->tv_nsec = 0;
738         timespec64_add_ns(ts, nsecs);
739 }
740 EXPORT_SYMBOL(ktime_get_real_ts64);
741 
742 ktime_t ktime_get(void)
743 {
744         struct timekeeper *tk = &tk_core.timekeeper;
745         unsigned int seq;
746         ktime_t base;
747         u64 nsecs;
748 
749         WARN_ON(timekeeping_suspended);
750 
751         do {
752                 seq = read_seqcount_begin(&tk_core.seq);
753                 base = tk->tkr_mono.base;
754                 nsecs = timekeeping_get_ns(&tk->tkr_mono);
755 
756         } while (read_seqcount_retry(&tk_core.seq, seq));
757 
758         return ktime_add_ns(base, nsecs);
759 }
760 EXPORT_SYMBOL_GPL(ktime_get);
761 
762 u32 ktime_get_resolution_ns(void)
763 {
764         struct timekeeper *tk = &tk_core.timekeeper;
765         unsigned int seq;
766         u32 nsecs;
767 
768         WARN_ON(timekeeping_suspended);
769 
770         do {
771                 seq = read_seqcount_begin(&tk_core.seq);
772                 nsecs = tk->tkr_mono.mult >> tk->tkr_mono.shift;
773         } while (read_seqcount_retry(&tk_core.seq, seq));
774 
775         return nsecs;
776 }
777 EXPORT_SYMBOL_GPL(ktime_get_resolution_ns);
778 
779 static ktime_t *offsets[TK_OFFS_MAX] = {
780         [TK_OFFS_REAL]  = &tk_core.timekeeper.offs_real,
781         [TK_OFFS_BOOT]  = &tk_core.timekeeper.offs_boot,
782         [TK_OFFS_TAI]   = &tk_core.timekeeper.offs_tai,
783 };
784 
785 ktime_t ktime_get_with_offset(enum tk_offsets offs)
786 {
787         struct timekeeper *tk = &tk_core.timekeeper;
788         unsigned int seq;
789         ktime_t base, *offset = offsets[offs];
790         u64 nsecs;
791 
792         WARN_ON(timekeeping_suspended);
793 
794         do {
795                 seq = read_seqcount_begin(&tk_core.seq);
796                 base = ktime_add(tk->tkr_mono.base, *offset);
797                 nsecs = timekeeping_get_ns(&tk->tkr_mono);
798 
799         } while (read_seqcount_retry(&tk_core.seq, seq));
800 
801         return ktime_add_ns(base, nsecs);
802 
803 }
804 EXPORT_SYMBOL_GPL(ktime_get_with_offset);
805 
806 ktime_t ktime_get_coarse_with_offset(enum tk_offsets offs)
807 {
808         struct timekeeper *tk = &tk_core.timekeeper;
809         unsigned int seq;
810         ktime_t base, *offset = offsets[offs];
811 
812         WARN_ON(timekeeping_suspended);
813 
814         do {
815                 seq = read_seqcount_begin(&tk_core.seq);
816                 base = ktime_add(tk->tkr_mono.base, *offset);
817 
818         } while (read_seqcount_retry(&tk_core.seq, seq));
819 
820         return base;
821 
822 }
823 EXPORT_SYMBOL_GPL(ktime_get_coarse_with_offset);
824 
825 /**
826  * ktime_mono_to_any() - convert mononotic time to any other time
827  * @tmono:      time to convert.
828  * @offs:       which offset to use
829  */
830 ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs)
831 {
832         ktime_t *offset = offsets[offs];
833         unsigned long seq;
834         ktime_t tconv;
835 
836         do {
837                 seq = read_seqcount_begin(&tk_core.seq);
838                 tconv = ktime_add(tmono, *offset);
839         } while (read_seqcount_retry(&tk_core.seq, seq));
840 
841         return tconv;
842 }
843 EXPORT_SYMBOL_GPL(ktime_mono_to_any);
844 
845 /**
846  * ktime_get_raw - Returns the raw monotonic time in ktime_t format
847  */
848 ktime_t ktime_get_raw(void)
849 {
850         struct timekeeper *tk = &tk_core.timekeeper;
851         unsigned int seq;
852         ktime_t base;
853         u64 nsecs;
854 
855         do {
856                 seq = read_seqcount_begin(&tk_core.seq);
857                 base = tk->tkr_raw.base;
858                 nsecs = timekeeping_get_ns(&tk->tkr_raw);
859 
860         } while (read_seqcount_retry(&tk_core.seq, seq));
861 
862         return ktime_add_ns(base, nsecs);
863 }
864 EXPORT_SYMBOL_GPL(ktime_get_raw);
865 
866 /**
867  * ktime_get_ts64 - get the monotonic clock in timespec64 format
868  * @ts:         pointer to timespec variable
869  *
870  * The function calculates the monotonic clock from the realtime
871  * clock and the wall_to_monotonic offset and stores the result
872  * in normalized timespec64 format in the variable pointed to by @ts.
873  */
874 void ktime_get_ts64(struct timespec64 *ts)
875 {
876         struct timekeeper *tk = &tk_core.timekeeper;
877         struct timespec64 tomono;
878         unsigned int seq;
879         u64 nsec;
880 
881         WARN_ON(timekeeping_suspended);
882 
883         do {
884                 seq = read_seqcount_begin(&tk_core.seq);
885                 ts->tv_sec = tk->xtime_sec;
886                 nsec = timekeeping_get_ns(&tk->tkr_mono);
887                 tomono = tk->wall_to_monotonic;
888 
889         } while (read_seqcount_retry(&tk_core.seq, seq));
890 
891         ts->tv_sec += tomono.tv_sec;
892         ts->tv_nsec = 0;
893         timespec64_add_ns(ts, nsec + tomono.tv_nsec);
894 }
895 EXPORT_SYMBOL_GPL(ktime_get_ts64);
896 
897 /**
898  * ktime_get_seconds - Get the seconds portion of CLOCK_MONOTONIC
899  *
900  * Returns the seconds portion of CLOCK_MONOTONIC with a single non
901  * serialized read. tk->ktime_sec is of type 'unsigned long' so this
902  * works on both 32 and 64 bit systems. On 32 bit systems the readout
903  * covers ~136 years of uptime which should be enough to prevent
904  * premature wrap arounds.
905  */
906 time64_t ktime_get_seconds(void)
907 {
908         struct timekeeper *tk = &tk_core.timekeeper;
909 
910         WARN_ON(timekeeping_suspended);
911         return tk->ktime_sec;
912 }
913 EXPORT_SYMBOL_GPL(ktime_get_seconds);
914 
915 /**
916  * ktime_get_real_seconds - Get the seconds portion of CLOCK_REALTIME
917  *
918  * Returns the wall clock seconds since 1970. This replaces the
919  * get_seconds() interface which is not y2038 safe on 32bit systems.
920  *
921  * For 64bit systems the fast access to tk->xtime_sec is preserved. On
922  * 32bit systems the access must be protected with the sequence
923  * counter to provide "atomic" access to the 64bit tk->xtime_sec
924  * value.
925  */
926 time64_t ktime_get_real_seconds(void)
927 {
928         struct timekeeper *tk = &tk_core.timekeeper;
929         time64_t seconds;
930         unsigned int seq;
931 
932         if (IS_ENABLED(CONFIG_64BIT))
933                 return tk->xtime_sec;
934 
935         do {
936                 seq = read_seqcount_begin(&tk_core.seq);
937                 seconds = tk->xtime_sec;
938 
939         } while (read_seqcount_retry(&tk_core.seq, seq));
940 
941         return seconds;
942 }
943 EXPORT_SYMBOL_GPL(ktime_get_real_seconds);
944 
945 /**
946  * __ktime_get_real_seconds - The same as ktime_get_real_seconds
947  * but without the sequence counter protect. This internal function
948  * is called just when timekeeping lock is already held.
949  */
950 time64_t __ktime_get_real_seconds(void)
951 {
952         struct timekeeper *tk = &tk_core.timekeeper;
953 
954         return tk->xtime_sec;
955 }
956 
957 /**
958  * ktime_get_snapshot - snapshots the realtime/monotonic raw clocks with counter
959  * @systime_snapshot:   pointer to struct receiving the system time snapshot
960  */
961 void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot)
962 {
963         struct timekeeper *tk = &tk_core.timekeeper;
964         unsigned long seq;
965         ktime_t base_raw;
966         ktime_t base_real;
967         u64 nsec_raw;
968         u64 nsec_real;
969         u64 now;
970 
971         WARN_ON_ONCE(timekeeping_suspended);
972 
973         do {
974                 seq = read_seqcount_begin(&tk_core.seq);
975                 now = tk_clock_read(&tk->tkr_mono);
976                 systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq;
977                 systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq;
978                 base_real = ktime_add(tk->tkr_mono.base,
979                                       tk_core.timekeeper.offs_real);
980                 base_raw = tk->tkr_raw.base;
981                 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now);
982                 nsec_raw  = timekeeping_cycles_to_ns(&tk->tkr_raw, now);
983         } while (read_seqcount_retry(&tk_core.seq, seq));
984 
985         systime_snapshot->cycles = now;
986         systime_snapshot->real = ktime_add_ns(base_real, nsec_real);
987         systime_snapshot->raw = ktime_add_ns(base_raw, nsec_raw);
988 }
989 EXPORT_SYMBOL_GPL(ktime_get_snapshot);
990 
991 /* Scale base by mult/div checking for overflow */
992 static int scale64_check_overflow(u64 mult, u64 div, u64 *base)
993 {
994         u64 tmp, rem;
995 
996         tmp = div64_u64_rem(*base, div, &rem);
997 
998         if (((int)sizeof(u64)*8 - fls64(mult) < fls64(tmp)) ||
999             ((int)sizeof(u64)*8 - fls64(mult) < fls64(rem)))
1000                 return -EOVERFLOW;
1001         tmp *= mult;
1002         rem *= mult;
1003 
1004         do_div(rem, div);
1005         *base = tmp + rem;
1006         return 0;
1007 }
1008 
1009 /**
1010  * adjust_historical_crosststamp - adjust crosstimestamp previous to current interval
1011  * @history:                    Snapshot representing start of history
1012  * @partial_history_cycles:     Cycle offset into history (fractional part)
1013  * @total_history_cycles:       Total history length in cycles
1014  * @discontinuity:              True indicates clock was set on history period
1015  * @ts:                         Cross timestamp that should be adjusted using
1016  *      partial/total ratio
1017  *
1018  * Helper function used by get_device_system_crosststamp() to correct the
1019  * crosstimestamp corresponding to the start of the current interval to the
1020  * system counter value (timestamp point) provided by the driver. The
1021  * total_history_* quantities are the total history starting at the provided
1022  * reference point and ending at the start of the current interval. The cycle
1023  * count between the driver timestamp point and the start of the current
1024  * interval is partial_history_cycles.
1025  */
1026 static int adjust_historical_crosststamp(struct system_time_snapshot *history,
1027                                          u64 partial_history_cycles,
1028                                          u64 total_history_cycles,
1029                                          bool discontinuity,
1030                                          struct system_device_crosststamp *ts)
1031 {
1032         struct timekeeper *tk = &tk_core.timekeeper;
1033         u64 corr_raw, corr_real;
1034         bool interp_forward;
1035         int ret;
1036 
1037         if (total_history_cycles == 0 || partial_history_cycles == 0)
1038                 return 0;
1039 
1040         /* Interpolate shortest distance from beginning or end of history */
1041         interp_forward = partial_history_cycles > total_history_cycles / 2;
1042         partial_history_cycles = interp_forward ?
1043                 total_history_cycles - partial_history_cycles :
1044                 partial_history_cycles;
1045 
1046         /*
1047          * Scale the monotonic raw time delta by:
1048          *      partial_history_cycles / total_history_cycles
1049          */
1050         corr_raw = (u64)ktime_to_ns(
1051                 ktime_sub(ts->sys_monoraw, history->raw));
1052         ret = scale64_check_overflow(partial_history_cycles,
1053                                      total_history_cycles, &corr_raw);
1054         if (ret)
1055                 return ret;
1056 
1057         /*
1058          * If there is a discontinuity in the history, scale monotonic raw
1059          *      correction by:
1060          *      mult(real)/mult(raw) yielding the realtime correction
1061          * Otherwise, calculate the realtime correction similar to monotonic
1062          *      raw calculation
1063          */
1064         if (discontinuity) {
1065                 corr_real = mul_u64_u32_div
1066                         (corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult);
1067         } else {
1068                 corr_real = (u64)ktime_to_ns(
1069                         ktime_sub(ts->sys_realtime, history->real));
1070                 ret = scale64_check_overflow(partial_history_cycles,
1071                                              total_history_cycles, &corr_real);
1072                 if (ret)
1073                         return ret;
1074         }
1075 
1076         /* Fixup monotonic raw and real time time values */
1077         if (interp_forward) {
1078                 ts->sys_monoraw = ktime_add_ns(history->raw, corr_raw);
1079                 ts->sys_realtime = ktime_add_ns(history->real, corr_real);
1080         } else {
1081                 ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw);
1082                 ts->sys_realtime = ktime_sub_ns(ts->sys_realtime, corr_real);
1083         }
1084 
1085         return 0;
1086 }
1087 
1088 /*
1089  * cycle_between - true if test occurs chronologically between before and after
1090  */
1091 static bool cycle_between(u64 before, u64 test, u64 after)
1092 {
1093         if (test > before && test < after)
1094                 return true;
1095         if (test < before && before > after)
1096                 return true;
1097         return false;
1098 }
1099 
1100 /**
1101  * get_device_system_crosststamp - Synchronously capture system/device timestamp
1102  * @get_time_fn:        Callback to get simultaneous device time and
1103  *      system counter from the device driver
1104  * @ctx:                Context passed to get_time_fn()
1105  * @history_begin:      Historical reference point used to interpolate system
1106  *      time when counter provided by the driver is before the current interval
1107  * @xtstamp:            Receives simultaneously captured system and device time
1108  *
1109  * Reads a timestamp from a device and correlates it to system time
1110  */
1111 int get_device_system_crosststamp(int (*get_time_fn)
1112                                   (ktime_t *device_time,
1113                                    struct system_counterval_t *sys_counterval,
1114                                    void *ctx),
1115                                   void *ctx,
1116                                   struct system_time_snapshot *history_begin,
1117                                   struct system_device_crosststamp *xtstamp)
1118 {
1119         struct system_counterval_t system_counterval;
1120         struct timekeeper *tk = &tk_core.timekeeper;
1121         u64 cycles, now, interval_start;
1122         unsigned int clock_was_set_seq = 0;
1123         ktime_t base_real, base_raw;
1124         u64 nsec_real, nsec_raw;
1125         u8 cs_was_changed_seq;
1126         unsigned long seq;
1127         bool do_interp;
1128         int ret;
1129 
1130         do {
1131                 seq = read_seqcount_begin(&tk_core.seq);
1132                 /*
1133                  * Try to synchronously capture device time and a system
1134                  * counter value calling back into the device driver
1135                  */
1136                 ret = get_time_fn(&xtstamp->device, &system_counterval, ctx);
1137                 if (ret)
1138                         return ret;
1139 
1140                 /*
1141                  * Verify that the clocksource associated with the captured
1142                  * system counter value is the same as the currently installed
1143                  * timekeeper clocksource
1144                  */
1145                 if (tk->tkr_mono.clock != system_counterval.cs)
1146                         return -ENODEV;
1147                 cycles = system_counterval.cycles;
1148 
1149                 /*
1150                  * Check whether the system counter value provided by the
1151                  * device driver is on the current timekeeping interval.
1152                  */
1153                 now = tk_clock_read(&tk->tkr_mono);
1154                 interval_start = tk->tkr_mono.cycle_last;
1155                 if (!cycle_between(interval_start, cycles, now)) {
1156                         clock_was_set_seq = tk->clock_was_set_seq;
1157                         cs_was_changed_seq = tk->cs_was_changed_seq;
1158                         cycles = interval_start;
1159                         do_interp = true;
1160                 } else {
1161                         do_interp = false;
1162                 }
1163 
1164                 base_real = ktime_add(tk->tkr_mono.base,
1165                                       tk_core.timekeeper.offs_real);
1166                 base_raw = tk->tkr_raw.base;
1167 
1168                 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono,
1169                                                      system_counterval.cycles);
1170                 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw,
1171                                                     system_counterval.cycles);
1172         } while (read_seqcount_retry(&tk_core.seq, seq));
1173 
1174         xtstamp->sys_realtime = ktime_add_ns(base_real, nsec_real);
1175         xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw);
1176 
1177         /*
1178          * Interpolate if necessary, adjusting back from the start of the
1179          * current interval
1180          */
1181         if (do_interp) {
1182                 u64 partial_history_cycles, total_history_cycles;
1183                 bool discontinuity;
1184 
1185                 /*
1186                  * Check that the counter value occurs after the provided
1187                  * history reference and that the history doesn't cross a
1188                  * clocksource change
1189                  */
1190                 if (!history_begin ||
1191                     !cycle_between(history_begin->cycles,
1192                                    system_counterval.cycles, cycles) ||
1193                     history_begin->cs_was_changed_seq != cs_was_changed_seq)
1194                         return -EINVAL;
1195                 partial_history_cycles = cycles - system_counterval.cycles;
1196                 total_history_cycles = cycles - history_begin->cycles;
1197                 discontinuity =
1198                         history_begin->clock_was_set_seq != clock_was_set_seq;
1199 
1200                 ret = adjust_historical_crosststamp(history_begin,
1201                                                     partial_history_cycles,
1202                                                     total_history_cycles,
1203                                                     discontinuity, xtstamp);
1204                 if (ret)
1205                         return ret;
1206         }
1207 
1208         return 0;
1209 }
1210 EXPORT_SYMBOL_GPL(get_device_system_crosststamp);
1211 
1212 /**
1213  * do_settimeofday64 - Sets the time of day.
1214  * @ts:     pointer to the timespec64 variable containing the new time
1215  *
1216  * Sets the time of day to the new time and update NTP and notify hrtimers
1217  */
1218 int do_settimeofday64(const struct timespec64 *ts)
1219 {
1220         struct timekeeper *tk = &tk_core.timekeeper;
1221         struct timespec64 ts_delta, xt;
1222         unsigned long flags;
1223         int ret = 0;
1224 
1225         if (!timespec64_valid_strict(ts))
1226                 return -EINVAL;
1227 
1228         raw_spin_lock_irqsave(&timekeeper_lock, flags);
1229         write_seqcount_begin(&tk_core.seq);
1230 
1231         timekeeping_forward_now(tk);
1232 
1233         xt = tk_xtime(tk);
1234         ts_delta.tv_sec = ts->tv_sec - xt.tv_sec;
1235         ts_delta.tv_nsec = ts->tv_nsec - xt.tv_nsec;
1236 
1237         if (timespec64_compare(&tk->wall_to_monotonic, &ts_delta) > 0) {
1238                 ret = -EINVAL;
1239                 goto out;
1240         }
1241 
1242         tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts_delta));
1243 
1244         tk_set_xtime(tk, ts);
1245 out:
1246         timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1247 
1248         write_seqcount_end(&tk_core.seq);
1249         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1250 
1251         /* signal hrtimers about time change */
1252         clock_was_set();
1253 
1254         return ret;
1255 }
1256 EXPORT_SYMBOL(do_settimeofday64);
1257 
1258 /**
1259  * timekeeping_inject_offset - Adds or subtracts from the current time.
1260  * @tv:         pointer to the timespec variable containing the offset
1261  *
1262  * Adds or subtracts an offset value from the current time.
1263  */
1264 static int timekeeping_inject_offset(const struct timespec64 *ts)
1265 {
1266         struct timekeeper *tk = &tk_core.timekeeper;
1267         unsigned long flags;
1268         struct timespec64 tmp;
1269         int ret = 0;
1270 
1271         if (ts->tv_nsec < 0 || ts->tv_nsec >= NSEC_PER_SEC)
1272                 return -EINVAL;
1273 
1274         raw_spin_lock_irqsave(&timekeeper_lock, flags);
1275         write_seqcount_begin(&tk_core.seq);
1276 
1277         timekeeping_forward_now(tk);
1278 
1279         /* Make sure the proposed value is valid */
1280         tmp = timespec64_add(tk_xtime(tk), *ts);
1281         if (timespec64_compare(&tk->wall_to_monotonic, ts) > 0 ||
1282             !timespec64_valid_strict(&tmp)) {
1283                 ret = -EINVAL;
1284                 goto error;
1285         }
1286 
1287         tk_xtime_add(tk, ts);
1288         tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *ts));
1289 
1290 error: /* even if we error out, we forwarded the time, so call update */
1291         timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1292 
1293         write_seqcount_end(&tk_core.seq);
1294         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1295 
1296         /* signal hrtimers about time change */
1297         clock_was_set();
1298 
1299         return ret;
1300 }
1301 
1302 /*
1303  * Indicates if there is an offset between the system clock and the hardware
1304  * clock/persistent clock/rtc.
1305  */
1306 int persistent_clock_is_local;
1307 
1308 /*
1309  * Adjust the time obtained from the CMOS to be UTC time instead of
1310  * local time.
1311  *
1312  * This is ugly, but preferable to the alternatives.  Otherwise we
1313  * would either need to write a program to do it in /etc/rc (and risk
1314  * confusion if the program gets run more than once; it would also be
1315  * hard to make the program warp the clock precisely n hours)  or
1316  * compile in the timezone information into the kernel.  Bad, bad....
1317  *
1318  *                                              - TYT, 1992-01-01
1319  *
1320  * The best thing to do is to keep the CMOS clock in universal time (UTC)
1321  * as real UNIX machines always do it. This avoids all headaches about
1322  * daylight saving times and warping kernel clocks.
1323  */
1324 void timekeeping_warp_clock(void)
1325 {
1326         if (sys_tz.tz_minuteswest != 0) {
1327                 struct timespec64 adjust;
1328 
1329                 persistent_clock_is_local = 1;
1330                 adjust.tv_sec = sys_tz.tz_minuteswest * 60;
1331                 adjust.tv_nsec = 0;
1332                 timekeeping_inject_offset(&adjust);
1333         }
1334 }
1335 
1336 /**
1337  * __timekeeping_set_tai_offset - Sets the TAI offset from UTC and monotonic
1338  *
1339  */
1340 static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset)
1341 {
1342         tk->tai_offset = tai_offset;
1343         tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0));
1344 }
1345 
1346 /**
1347  * change_clocksource - Swaps clocksources if a new one is available
1348  *
1349  * Accumulates current time interval and initializes new clocksource
1350  */
1351 static int change_clocksource(void *data)
1352 {
1353         struct timekeeper *tk = &tk_core.timekeeper;
1354         struct clocksource *new, *old;
1355         unsigned long flags;
1356 
1357         new = (struct clocksource *) data;
1358 
1359         raw_spin_lock_irqsave(&timekeeper_lock, flags);
1360         write_seqcount_begin(&tk_core.seq);
1361 
1362         timekeeping_forward_now(tk);
1363         /*
1364          * If the cs is in module, get a module reference. Succeeds
1365          * for built-in code (owner == NULL) as well.
1366          */
1367         if (try_module_get(new->owner)) {
1368                 if (!new->enable || new->enable(new) == 0) {
1369                         old = tk->tkr_mono.clock;
1370                         tk_setup_internals(tk, new);
1371                         if (old->disable)
1372                                 old->disable(old);
1373                         module_put(old->owner);
1374                 } else {
1375                         module_put(new->owner);
1376                 }
1377         }
1378         timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1379 
1380         write_seqcount_end(&tk_core.seq);
1381         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1382 
1383         return 0;
1384 }
1385 
1386 /**
1387  * timekeeping_notify - Install a new clock source
1388  * @clock:              pointer to the clock source
1389  *
1390  * This function is called from clocksource.c after a new, better clock
1391  * source has been registered. The caller holds the clocksource_mutex.
1392  */
1393 int timekeeping_notify(struct clocksource *clock)
1394 {
1395         struct timekeeper *tk = &tk_core.timekeeper;
1396 
1397         if (tk->tkr_mono.clock == clock)
1398                 return 0;
1399         stop_machine(change_clocksource, clock, NULL);
1400         tick_clock_notify();
1401         return tk->tkr_mono.clock == clock ? 0 : -1;
1402 }
1403 
1404 /**
1405  * ktime_get_raw_ts64 - Returns the raw monotonic time in a timespec
1406  * @ts:         pointer to the timespec64 to be set
1407  *
1408  * Returns the raw monotonic time (completely un-modified by ntp)
1409  */
1410 void ktime_get_raw_ts64(struct timespec64 *ts)
1411 {
1412         struct timekeeper *tk = &tk_core.timekeeper;
1413         unsigned long seq;
1414         u64 nsecs;
1415 
1416         do {
1417                 seq = read_seqcount_begin(&tk_core.seq);
1418                 ts->tv_sec = tk->raw_sec;
1419                 nsecs = timekeeping_get_ns(&tk->tkr_raw);
1420 
1421         } while (read_seqcount_retry(&tk_core.seq, seq));
1422 
1423         ts->tv_nsec = 0;
1424         timespec64_add_ns(ts, nsecs);
1425 }
1426 EXPORT_SYMBOL(ktime_get_raw_ts64);
1427 
1428 
1429 /**
1430  * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
1431  */
1432 int timekeeping_valid_for_hres(void)
1433 {
1434         struct timekeeper *tk = &tk_core.timekeeper;
1435         unsigned long seq;
1436         int ret;
1437 
1438         do {
1439                 seq = read_seqcount_begin(&tk_core.seq);
1440 
1441                 ret = tk->tkr_mono.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
1442 
1443         } while (read_seqcount_retry(&tk_core.seq, seq));
1444 
1445         return ret;
1446 }
1447 
1448 /**
1449  * timekeeping_max_deferment - Returns max time the clocksource can be deferred
1450  */
1451 u64 timekeeping_max_deferment(void)
1452 {
1453         struct timekeeper *tk = &tk_core.timekeeper;
1454         unsigned long seq;
1455         u64 ret;
1456 
1457         do {
1458                 seq = read_seqcount_begin(&tk_core.seq);
1459 
1460                 ret = tk->tkr_mono.clock->max_idle_ns;
1461 
1462         } while (read_seqcount_retry(&tk_core.seq, seq));
1463 
1464         return ret;
1465 }
1466 
1467 /**
1468  * read_persistent_clock64 -  Return time from the persistent clock.
1469  *
1470  * Weak dummy function for arches that do not yet support it.
1471  * Reads the time from the battery backed persistent clock.
1472  * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported.
1473  *
1474  *  XXX - Do be sure to remove it once all arches implement it.
1475  */
1476 void __weak read_persistent_clock64(struct timespec64 *ts)
1477 {
1478         ts->tv_sec = 0;
1479         ts->tv_nsec = 0;
1480 }
1481 
1482 /**
1483  * read_persistent_wall_and_boot_offset - Read persistent clock, and also offset
1484  *                                        from the boot.
1485  *
1486  * Weak dummy function for arches that do not yet support it.
1487  * wall_time    - current time as returned by persistent clock
1488  * boot_offset  - offset that is defined as wall_time - boot_time
1489  * The default function calculates offset based on the current value of
1490  * local_clock(). This way architectures that support sched_clock() but don't
1491  * support dedicated boot time clock will provide the best estimate of the
1492  * boot time.
1493  */
1494 void __weak __init
1495 read_persistent_wall_and_boot_offset(struct timespec64 *wall_time,
1496                                      struct timespec64 *boot_offset)
1497 {
1498         read_persistent_clock64(wall_time);
1499         *boot_offset = ns_to_timespec64(local_clock());
1500 }
1501 
1502 /*
1503  * Flag reflecting whether timekeeping_resume() has injected sleeptime.
1504  *
1505  * The flag starts of false and is only set when a suspend reaches
1506  * timekeeping_suspend(), timekeeping_resume() sets it to false when the
1507  * timekeeper clocksource is not stopping across suspend and has been
1508  * used to update sleep time. If the timekeeper clocksource has stopped
1509  * then the flag stays true and is used by the RTC resume code to decide
1510  * whether sleeptime must be injected and if so the flag gets false then.
1511  *
1512  * If a suspend fails before reaching timekeeping_resume() then the flag
1513  * stays false and prevents erroneous sleeptime injection.
1514  */
1515 static bool suspend_timing_needed;
1516 
1517 /* Flag for if there is a persistent clock on this platform */
1518 static bool persistent_clock_exists;
1519 
1520 /*
1521  * timekeeping_init - Initializes the clocksource and common timekeeping values
1522  */
1523 void __init timekeeping_init(void)
1524 {
1525         struct timespec64 wall_time, boot_offset, wall_to_mono;
1526         struct timekeeper *tk = &tk_core.timekeeper;
1527         struct clocksource *clock;
1528         unsigned long flags;
1529 
1530         read_persistent_wall_and_boot_offset(&wall_time, &boot_offset);
1531         if (timespec64_valid_strict(&wall_time) &&
1532             timespec64_to_ns(&wall_time) > 0) {
1533                 persistent_clock_exists = true;
1534         } else if (timespec64_to_ns(&wall_time) != 0) {
1535                 pr_warn("Persistent clock returned invalid value");
1536                 wall_time = (struct timespec64){0};
1537         }
1538 
1539         if (timespec64_compare(&wall_time, &boot_offset) < 0)
1540                 boot_offset = (struct timespec64){0};
1541 
1542         /*
1543          * We want set wall_to_mono, so the following is true:
1544          * wall time + wall_to_mono = boot time
1545          */
1546         wall_to_mono = timespec64_sub(boot_offset, wall_time);
1547 
1548         raw_spin_lock_irqsave(&timekeeper_lock, flags);
1549         write_seqcount_begin(&tk_core.seq);
1550         ntp_init();
1551 
1552         clock = clocksource_default_clock();
1553         if (clock->enable)
1554                 clock->enable(clock);
1555         tk_setup_internals(tk, clock);
1556 
1557         tk_set_xtime(tk, &wall_time);
1558         tk->raw_sec = 0;
1559 
1560         tk_set_wall_to_mono(tk, wall_to_mono);
1561 
1562         timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
1563 
1564         write_seqcount_end(&tk_core.seq);
1565         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1566 }
1567 
1568 /* time in seconds when suspend began for persistent clock */
1569 static struct timespec64 timekeeping_suspend_time;
1570 
1571 /**
1572  * __timekeeping_inject_sleeptime - Internal function to add sleep interval
1573  * @delta: pointer to a timespec delta value
1574  *
1575  * Takes a timespec offset measuring a suspend interval and properly
1576  * adds the sleep offset to the timekeeping variables.
1577  */
1578 static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
1579                                            const struct timespec64 *delta)
1580 {
1581         if (!timespec64_valid_strict(delta)) {
1582                 printk_deferred(KERN_WARNING
1583                                 "__timekeeping_inject_sleeptime: Invalid "
1584                                 "sleep delta value!\n");
1585                 return;
1586         }
1587         tk_xtime_add(tk, delta);
1588         tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta));
1589         tk_update_sleep_time(tk, timespec64_to_ktime(*delta));
1590         tk_debug_account_sleep_time(delta);
1591 }
1592 
1593 #if defined(CONFIG_PM_SLEEP) && defined(CONFIG_RTC_HCTOSYS_DEVICE)
1594 /**
1595  * We have three kinds of time sources to use for sleep time
1596  * injection, the preference order is:
1597  * 1) non-stop clocksource
1598  * 2) persistent clock (ie: RTC accessible when irqs are off)
1599  * 3) RTC
1600  *
1601  * 1) and 2) are used by timekeeping, 3) by RTC subsystem.
1602  * If system has neither 1) nor 2), 3) will be used finally.
1603  *
1604  *
1605  * If timekeeping has injected sleeptime via either 1) or 2),
1606  * 3) becomes needless, so in this case we don't need to call
1607  * rtc_resume(), and this is what timekeeping_rtc_skipresume()
1608  * means.
1609  */
1610 bool timekeeping_rtc_skipresume(void)
1611 {
1612         return !suspend_timing_needed;
1613 }
1614 
1615 /**
1616  * 1) can be determined whether to use or not only when doing
1617  * timekeeping_resume() which is invoked after rtc_suspend(),
1618  * so we can't skip rtc_suspend() surely if system has 1).
1619  *
1620  * But if system has 2), 2) will definitely be used, so in this
1621  * case we don't need to call rtc_suspend(), and this is what
1622  * timekeeping_rtc_skipsuspend() means.
1623  */
1624 bool timekeeping_rtc_skipsuspend(void)
1625 {
1626         return persistent_clock_exists;
1627 }
1628 
1629 /**
1630  * timekeeping_inject_sleeptime64 - Adds suspend interval to timeekeeping values
1631  * @delta: pointer to a timespec64 delta value
1632  *
1633  * This hook is for architectures that cannot support read_persistent_clock64
1634  * because their RTC/persistent clock is only accessible when irqs are enabled.
1635  * and also don't have an effective nonstop clocksource.
1636  *
1637  * This function should only be called by rtc_resume(), and allows
1638  * a suspend offset to be injected into the timekeeping values.
1639  */
1640 void timekeeping_inject_sleeptime64(const struct timespec64 *delta)
1641 {
1642         struct timekeeper *tk = &tk_core.timekeeper;
1643         unsigned long flags;
1644 
1645         raw_spin_lock_irqsave(&timekeeper_lock, flags);
1646         write_seqcount_begin(&tk_core.seq);
1647 
1648         suspend_timing_needed = false;
1649 
1650         timekeeping_forward_now(tk);
1651 
1652         __timekeeping_inject_sleeptime(tk, delta);
1653 
1654         timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1655 
1656         write_seqcount_end(&tk_core.seq);
1657         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1658 
1659         /* signal hrtimers about time change */
1660         clock_was_set();
1661 }
1662 #endif
1663 
1664 /**
1665  * timekeeping_resume - Resumes the generic timekeeping subsystem.
1666  */
1667 void timekeeping_resume(void)
1668 {
1669         struct timekeeper *tk = &tk_core.timekeeper;
1670         struct clocksource *clock = tk->tkr_mono.clock;
1671         unsigned long flags;
1672         struct timespec64 ts_new, ts_delta;
1673         u64 cycle_now, nsec;
1674         bool inject_sleeptime = false;
1675 
1676         read_persistent_clock64(&ts_new);
1677 
1678         clockevents_resume();
1679         clocksource_resume();
1680 
1681         raw_spin_lock_irqsave(&timekeeper_lock, flags);
1682         write_seqcount_begin(&tk_core.seq);
1683 
1684         /*
1685          * After system resumes, we need to calculate the suspended time and
1686          * compensate it for the OS time. There are 3 sources that could be
1687          * used: Nonstop clocksource during suspend, persistent clock and rtc
1688          * device.
1689          *
1690          * One specific platform may have 1 or 2 or all of them, and the
1691          * preference will be:
1692          *      suspend-nonstop clocksource -> persistent clock -> rtc
1693          * The less preferred source will only be tried if there is no better
1694          * usable source. The rtc part is handled separately in rtc core code.
1695          */
1696         cycle_now = tk_clock_read(&tk->tkr_mono);
1697         nsec = clocksource_stop_suspend_timing(clock, cycle_now);
1698         if (nsec > 0) {
1699                 ts_delta = ns_to_timespec64(nsec);
1700                 inject_sleeptime = true;
1701         } else if (timespec64_compare(&ts_new, &timekeeping_suspend_time) > 0) {
1702                 ts_delta = timespec64_sub(ts_new, timekeeping_suspend_time);
1703                 inject_sleeptime = true;
1704         }
1705 
1706         if (inject_sleeptime) {
1707                 suspend_timing_needed = false;
1708                 __timekeeping_inject_sleeptime(tk, &ts_delta);
1709         }
1710 
1711         /* Re-base the last cycle value */
1712         tk->tkr_mono.cycle_last = cycle_now;
1713         tk->tkr_raw.cycle_last  = cycle_now;
1714 
1715         tk->ntp_error = 0;
1716         timekeeping_suspended = 0;
1717         timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
1718         write_seqcount_end(&tk_core.seq);
1719         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1720 
1721         touch_softlockup_watchdog();
1722 
1723         tick_resume();
1724         hrtimers_resume();
1725 }
1726 
1727 int timekeeping_suspend(void)
1728 {
1729         struct timekeeper *tk = &tk_core.timekeeper;
1730         unsigned long flags;
1731         struct timespec64               delta, delta_delta;
1732         static struct timespec64        old_delta;
1733         struct clocksource *curr_clock;
1734         u64 cycle_now;
1735 
1736         read_persistent_clock64(&timekeeping_suspend_time);
1737 
1738         /*
1739          * On some systems the persistent_clock can not be detected at
1740          * timekeeping_init by its return value, so if we see a valid
1741          * value returned, update the persistent_clock_exists flag.
1742          */
1743         if (timekeeping_suspend_time.tv_sec || timekeeping_suspend_time.tv_nsec)
1744                 persistent_clock_exists = true;
1745 
1746         suspend_timing_needed = true;
1747 
1748         raw_spin_lock_irqsave(&timekeeper_lock, flags);
1749         write_seqcount_begin(&tk_core.seq);
1750         timekeeping_forward_now(tk);
1751         timekeeping_suspended = 1;
1752 
1753         /*
1754          * Since we've called forward_now, cycle_last stores the value
1755          * just read from the current clocksource. Save this to potentially
1756          * use in suspend timing.
1757          */
1758         curr_clock = tk->tkr_mono.clock;
1759         cycle_now = tk->tkr_mono.cycle_last;
1760         clocksource_start_suspend_timing(curr_clock, cycle_now);
1761 
1762         if (persistent_clock_exists) {
1763                 /*
1764                  * To avoid drift caused by repeated suspend/resumes,
1765                  * which each can add ~1 second drift error,
1766                  * try to compensate so the difference in system time
1767                  * and persistent_clock time stays close to constant.
1768                  */
1769                 delta = timespec64_sub(tk_xtime(tk), timekeeping_suspend_time);
1770                 delta_delta = timespec64_sub(delta, old_delta);
1771                 if (abs(delta_delta.tv_sec) >= 2) {
1772                         /*
1773                          * if delta_delta is too large, assume time correction
1774                          * has occurred and set old_delta to the current delta.
1775                          */
1776                         old_delta = delta;
1777                 } else {
1778                         /* Otherwise try to adjust old_system to compensate */
1779                         timekeeping_suspend_time =
1780                                 timespec64_add(timekeeping_suspend_time, delta_delta);
1781                 }
1782         }
1783 
1784         timekeeping_update(tk, TK_MIRROR);
1785         halt_fast_timekeeper(tk);
1786         write_seqcount_end(&tk_core.seq);
1787         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1788 
1789         tick_suspend();
1790         clocksource_suspend();
1791         clockevents_suspend();
1792 
1793         return 0;
1794 }
1795 
1796 /* sysfs resume/suspend bits for timekeeping */
1797 static struct syscore_ops timekeeping_syscore_ops = {
1798         .resume         = timekeeping_resume,
1799         .suspend        = timekeeping_suspend,
1800 };
1801 
1802 static int __init timekeeping_init_ops(void)
1803 {
1804         register_syscore_ops(&timekeeping_syscore_ops);
1805         return 0;
1806 }
1807 device_initcall(timekeeping_init_ops);
1808 
1809 /*
1810  * Apply a multiplier adjustment to the timekeeper
1811  */
1812 static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
1813                                                          s64 offset,
1814                                                          s32 mult_adj)
1815 {
1816         s64 interval = tk->cycle_interval;
1817 
1818         if (mult_adj == 0) {
1819                 return;
1820         } else if (mult_adj == -1) {
1821                 interval = -interval;
1822                 offset = -offset;
1823         } else if (mult_adj != 1) {
1824                 interval *= mult_adj;
1825                 offset *= mult_adj;
1826         }
1827 
1828         /*
1829          * So the following can be confusing.
1830          *
1831          * To keep things simple, lets assume mult_adj == 1 for now.
1832          *
1833          * When mult_adj != 1, remember that the interval and offset values
1834          * have been appropriately scaled so the math is the same.
1835          *
1836          * The basic idea here is that we're increasing the multiplier
1837          * by one, this causes the xtime_interval to be incremented by
1838          * one cycle_interval. This is because:
1839          *      xtime_interval = cycle_interval * mult
1840          * So if mult is being incremented by one:
1841          *      xtime_interval = cycle_interval * (mult + 1)
1842          * Its the same as:
1843          *      xtime_interval = (cycle_interval * mult) + cycle_interval
1844          * Which can be shortened to:
1845          *      xtime_interval += cycle_interval
1846          *
1847          * So offset stores the non-accumulated cycles. Thus the current
1848          * time (in shifted nanoseconds) is:
1849          *      now = (offset * adj) + xtime_nsec
1850          * Now, even though we're adjusting the clock frequency, we have
1851          * to keep time consistent. In other words, we can't jump back
1852          * in time, and we also want to avoid jumping forward in time.
1853          *
1854          * So given the same offset value, we need the time to be the same
1855          * both before and after the freq adjustment.
1856          *      now = (offset * adj_1) + xtime_nsec_1
1857          *      now = (offset * adj_2) + xtime_nsec_2
1858          * So:
1859          *      (offset * adj_1) + xtime_nsec_1 =
1860          *              (offset * adj_2) + xtime_nsec_2
1861          * And we know:
1862          *      adj_2 = adj_1 + 1
1863          * So:
1864          *      (offset * adj_1) + xtime_nsec_1 =
1865          *              (offset * (adj_1+1)) + xtime_nsec_2
1866          *      (offset * adj_1) + xtime_nsec_1 =
1867          *              (offset * adj_1) + offset + xtime_nsec_2
1868          * Canceling the sides:
1869          *      xtime_nsec_1 = offset + xtime_nsec_2
1870          * Which gives us:
1871          *      xtime_nsec_2 = xtime_nsec_1 - offset
1872          * Which simplfies to:
1873          *      xtime_nsec -= offset
1874          */
1875         if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) {
1876                 /* NTP adjustment caused clocksource mult overflow */
1877                 WARN_ON_ONCE(1);
1878                 return;
1879         }
1880 
1881         tk->tkr_mono.mult += mult_adj;
1882         tk->xtime_interval += interval;
1883         tk->tkr_mono.xtime_nsec -= offset;
1884 }
1885 
1886 /*
1887  * Adjust the timekeeper's multiplier to the correct frequency
1888  * and also to reduce the accumulated error value.
1889  */
1890 static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
1891 {
1892         u32 mult;
1893 
1894         /*
1895          * Determine the multiplier from the current NTP tick length.
1896          * Avoid expensive division when the tick length doesn't change.
1897          */
1898         if (likely(tk->ntp_tick == ntp_tick_length())) {
1899                 mult = tk->tkr_mono.mult - tk->ntp_err_mult;
1900         } else {
1901                 tk->ntp_tick = ntp_tick_length();
1902                 mult = div64_u64((tk->ntp_tick >> tk->ntp_error_shift) -
1903                                  tk->xtime_remainder, tk->cycle_interval);
1904         }
1905 
1906         /*
1907          * If the clock is behind the NTP time, increase the multiplier by 1
1908          * to catch up with it. If it's ahead and there was a remainder in the
1909          * tick division, the clock will slow down. Otherwise it will stay
1910          * ahead until the tick length changes to a non-divisible value.
1911          */
1912         tk->ntp_err_mult = tk->ntp_error > 0 ? 1 : 0;
1913         mult += tk->ntp_err_mult;
1914 
1915         timekeeping_apply_adjustment(tk, offset, mult - tk->tkr_mono.mult);
1916 
1917         if (unlikely(tk->tkr_mono.clock->maxadj &&
1918                 (abs(tk->tkr_mono.mult - tk->tkr_mono.clock->mult)
1919                         > tk->tkr_mono.clock->maxadj))) {
1920                 printk_once(KERN_WARNING
1921                         "Adjusting %s more than 11%% (%ld vs %ld)\n",
1922                         tk->tkr_mono.clock->name, (long)tk->tkr_mono.mult,
1923                         (long)tk->tkr_mono.clock->mult + tk->tkr_mono.clock->maxadj);
1924         }
1925 
1926         /*
1927          * It may be possible that when we entered this function, xtime_nsec
1928          * was very small.  Further, if we're slightly speeding the clocksource
1929          * in the code above, its possible the required corrective factor to
1930          * xtime_nsec could cause it to underflow.
1931          *
1932          * Now, since we have already accumulated the second and the NTP
1933          * subsystem has been notified via second_overflow(), we need to skip
1934          * the next update.
1935          */
1936         if (unlikely((s64)tk->tkr_mono.xtime_nsec < 0)) {
1937                 tk->tkr_mono.xtime_nsec += (u64)NSEC_PER_SEC <<
1938                                                         tk->tkr_mono.shift;
1939                 tk->xtime_sec--;
1940                 tk->skip_second_overflow = 1;
1941         }
1942 }
1943 
1944 /**
1945  * accumulate_nsecs_to_secs - Accumulates nsecs into secs
1946  *
1947  * Helper function that accumulates the nsecs greater than a second
1948  * from the xtime_nsec field to the xtime_secs field.
1949  * It also calls into the NTP code to handle leapsecond processing.
1950  *
1951  */
1952 static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
1953 {
1954         u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
1955         unsigned int clock_set = 0;
1956 
1957         while (tk->tkr_mono.xtime_nsec >= nsecps) {
1958                 int leap;
1959 
1960                 tk->tkr_mono.xtime_nsec -= nsecps;
1961                 tk->xtime_sec++;
1962 
1963                 /*
1964                  * Skip NTP update if this second was accumulated before,
1965                  * i.e. xtime_nsec underflowed in timekeeping_adjust()
1966                  */
1967                 if (unlikely(tk->skip_second_overflow)) {
1968                         tk->skip_second_overflow = 0;
1969                         continue;
1970                 }
1971 
1972                 /* Figure out if its a leap sec and apply if needed */
1973                 leap = second_overflow(tk->xtime_sec);
1974                 if (unlikely(leap)) {
1975                         struct timespec64 ts;
1976 
1977                         tk->xtime_sec += leap;
1978 
1979                         ts.tv_sec = leap;
1980                         ts.tv_nsec = 0;
1981                         tk_set_wall_to_mono(tk,
1982                                 timespec64_sub(tk->wall_to_monotonic, ts));
1983 
1984                         __timekeeping_set_tai_offset(tk, tk->tai_offset - leap);
1985 
1986                         clock_set = TK_CLOCK_WAS_SET;
1987                 }
1988         }
1989         return clock_set;
1990 }
1991 
1992 /**
1993  * logarithmic_accumulation - shifted accumulation of cycles
1994  *
1995  * This functions accumulates a shifted interval of cycles into
1996  * into a shifted interval nanoseconds. Allows for O(log) accumulation
1997  * loop.
1998  *
1999  * Returns the unconsumed cycles.
2000  */
2001 static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset,
2002                                     u32 shift, unsigned int *clock_set)
2003 {
2004         u64 interval = tk->cycle_interval << shift;
2005         u64 snsec_per_sec;
2006 
2007         /* If the offset is smaller than a shifted interval, do nothing */
2008         if (offset < interval)
2009                 return offset;
2010 
2011         /* Accumulate one shifted interval */
2012         offset -= interval;
2013         tk->tkr_mono.cycle_last += interval;
2014         tk->tkr_raw.cycle_last  += interval;
2015 
2016         tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift;
2017         *clock_set |= accumulate_nsecs_to_secs(tk);
2018 
2019         /* Accumulate raw time */
2020         tk->tkr_raw.xtime_nsec += tk->raw_interval << shift;
2021         snsec_per_sec = (u64)NSEC_PER_SEC << tk->tkr_raw.shift;
2022         while (tk->tkr_raw.xtime_nsec >= snsec_per_sec) {
2023                 tk->tkr_raw.xtime_nsec -= snsec_per_sec;
2024                 tk->raw_sec++;
2025         }
2026 
2027         /* Accumulate error between NTP and clock interval */
2028         tk->ntp_error += tk->ntp_tick << shift;
2029         tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
2030                                                 (tk->ntp_error_shift + shift);
2031 
2032         return offset;
2033 }
2034 
2035 /*
2036  * timekeeping_advance - Updates the timekeeper to the current time and
2037  * current NTP tick length
2038  */
2039 static void timekeeping_advance(enum timekeeping_adv_mode mode)
2040 {
2041         struct timekeeper *real_tk = &tk_core.timekeeper;
2042         struct timekeeper *tk = &shadow_timekeeper;
2043         u64 offset;
2044         int shift = 0, maxshift;
2045         unsigned int clock_set = 0;
2046         unsigned long flags;
2047 
2048         raw_spin_lock_irqsave(&timekeeper_lock, flags);
2049 
2050         /* Make sure we're fully resumed: */
2051         if (unlikely(timekeeping_suspended))
2052                 goto out;
2053 
2054 #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
2055         offset = real_tk->cycle_interval;
2056 
2057         if (mode != TK_ADV_TICK)
2058                 goto out;
2059 #else
2060         offset = clocksource_delta(tk_clock_read(&tk->tkr_mono),
2061                                    tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
2062 
2063         /* Check if there's really nothing to do */
2064         if (offset < real_tk->cycle_interval && mode == TK_ADV_TICK)
2065                 goto out;
2066 #endif
2067 
2068         /* Do some additional sanity checking */
2069         timekeeping_check_update(tk, offset);
2070 
2071         /*
2072          * With NO_HZ we may have to accumulate many cycle_intervals
2073          * (think "ticks") worth of time at once. To do this efficiently,
2074          * we calculate the largest doubling multiple of cycle_intervals
2075          * that is smaller than the offset.  We then accumulate that
2076          * chunk in one go, and then try to consume the next smaller
2077          * doubled multiple.
2078          */
2079         shift = ilog2(offset) - ilog2(tk->cycle_interval);
2080         shift = max(0, shift);
2081         /* Bound shift to one less than what overflows tick_length */
2082         maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;
2083         shift = min(shift, maxshift);
2084         while (offset >= tk->cycle_interval) {
2085                 offset = logarithmic_accumulation(tk, offset, shift,
2086                                                         &clock_set);
2087                 if (offset < tk->cycle_interval<<shift)
2088                         shift--;
2089         }
2090 
2091         /* Adjust the multiplier to correct NTP error */
2092         timekeeping_adjust(tk, offset);
2093 
2094         /*
2095          * Finally, make sure that after the rounding
2096          * xtime_nsec isn't larger than NSEC_PER_SEC
2097          */
2098         clock_set |= accumulate_nsecs_to_secs(tk);
2099 
2100         write_seqcount_begin(&tk_core.seq);
2101         /*
2102          * Update the real timekeeper.
2103          *
2104          * We could avoid this memcpy by switching pointers, but that
2105          * requires changes to all other timekeeper usage sites as
2106          * well, i.e. move the timekeeper pointer getter into the
2107          * spinlocked/seqcount protected sections. And we trade this
2108          * memcpy under the tk_core.seq against one before we start
2109          * updating.
2110          */
2111         timekeeping_update(tk, clock_set);
2112         memcpy(real_tk, tk, sizeof(*tk));
2113         /* The memcpy must come last. Do not put anything here! */
2114         write_seqcount_end(&tk_core.seq);
2115 out:
2116         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
2117         if (clock_set)
2118                 /* Have to call _delayed version, since in irq context*/
2119                 clock_was_set_delayed();
2120 }
2121 
2122 /**
2123  * update_wall_time - Uses the current clocksource to increment the wall time
2124  *
2125  */
2126 void update_wall_time(void)
2127 {
2128         timekeeping_advance(TK_ADV_TICK);
2129 }
2130 
2131 /**
2132  * getboottime64 - Return the real time of system boot.
2133  * @ts:         pointer to the timespec64 to be set
2134  *
2135  * Returns the wall-time of boot in a timespec64.
2136  *
2137  * This is based on the wall_to_monotonic offset and the total suspend
2138  * time. Calls to settimeofday will affect the value returned (which
2139  * basically means that however wrong your real time clock is at boot time,
2140  * you get the right time here).
2141  */
2142 void getboottime64(struct timespec64 *ts)
2143 {
2144         struct timekeeper *tk = &tk_core.timekeeper;
2145         ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot);
2146 
2147         *ts = ktime_to_timespec64(t);
2148 }
2149 EXPORT_SYMBOL_GPL(getboottime64);
2150 
2151 void ktime_get_coarse_real_ts64(struct timespec64 *ts)
2152 {
2153         struct timekeeper *tk = &tk_core.timekeeper;
2154         unsigned long seq;
2155 
2156         do {
2157                 seq = read_seqcount_begin(&tk_core.seq);
2158 
2159                 *ts = tk_xtime(tk);
2160         } while (read_seqcount_retry(&tk_core.seq, seq));
2161 }
2162 EXPORT_SYMBOL(ktime_get_coarse_real_ts64);
2163 
2164 void ktime_get_coarse_ts64(struct timespec64 *ts)
2165 {
2166         struct timekeeper *tk = &tk_core.timekeeper;
2167         struct timespec64 now, mono;
2168         unsigned long seq;
2169 
2170         do {
2171                 seq = read_seqcount_begin(&tk_core.seq);
2172 
2173                 now = tk_xtime(tk);
2174                 mono = tk->wall_to_monotonic;
2175         } while (read_seqcount_retry(&tk_core.seq, seq));
2176 
2177         set_normalized_timespec64(ts, now.tv_sec + mono.tv_sec,
2178                                 now.tv_nsec + mono.tv_nsec);
2179 }
2180 EXPORT_SYMBOL(ktime_get_coarse_ts64);
2181 
2182 /*
2183  * Must hold jiffies_lock
2184  */
2185 void do_timer(unsigned long ticks)
2186 {
2187         jiffies_64 += ticks;
2188         calc_global_load(ticks);
2189 }
2190 
2191 /**
2192  * ktime_get_update_offsets_now - hrtimer helper
2193  * @cwsseq:     pointer to check and store the clock was set sequence number
2194  * @offs_real:  pointer to storage for monotonic -> realtime offset
2195  * @offs_boot:  pointer to storage for monotonic -> boottime offset
2196  * @offs_tai:   pointer to storage for monotonic -> clock tai offset
2197  *
2198  * Returns current monotonic time and updates the offsets if the
2199  * sequence number in @cwsseq and timekeeper.clock_was_set_seq are
2200  * different.
2201  *
2202  * Called from hrtimer_interrupt() or retrigger_next_event()
2203  */
2204 ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, ktime_t *offs_real,
2205                                      ktime_t *offs_boot, ktime_t *offs_tai)
2206 {
2207         struct timekeeper *tk = &tk_core.timekeeper;
2208         unsigned int seq;
2209         ktime_t base;
2210         u64 nsecs;
2211 
2212         do {
2213                 seq = read_seqcount_begin(&tk_core.seq);
2214 
2215                 base = tk->tkr_mono.base;
2216                 nsecs = timekeeping_get_ns(&tk->tkr_mono);
2217                 base = ktime_add_ns(base, nsecs);
2218 
2219                 if (*cwsseq != tk->clock_was_set_seq) {
2220                         *cwsseq = tk->clock_was_set_seq;
2221                         *offs_real = tk->offs_real;
2222                         *offs_boot = tk->offs_boot;
2223                         *offs_tai = tk->offs_tai;
2224                 }
2225 
2226                 /* Handle leapsecond insertion adjustments */
2227                 if (unlikely(base >= tk->next_leap_ktime))
2228                         *offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0));
2229 
2230         } while (read_seqcount_retry(&tk_core.seq, seq));
2231 
2232         return base;
2233 }
2234 
2235 /**
2236  * timekeeping_validate_timex - Ensures the timex is ok for use in do_adjtimex
2237  */
2238 static int timekeeping_validate_timex(const struct timex *txc)
2239 {
2240         if (txc->modes & ADJ_ADJTIME) {
2241                 /* singleshot must not be used with any other mode bits */
2242                 if (!(txc->modes & ADJ_OFFSET_SINGLESHOT))
2243                         return -EINVAL;
2244                 if (!(txc->modes & ADJ_OFFSET_READONLY) &&
2245                     !capable(CAP_SYS_TIME))
2246                         return -EPERM;
2247                 if (!(txc->modes & ADJ_OFFSET_READONLY) &&
2248                     !ccs_capable(CCS_SYS_SETTIME))
2249                         return -EPERM;
2250         } else {
2251                 /* In order to modify anything, you gotta be super-user! */
2252                 if (txc->modes && !capable(CAP_SYS_TIME))
2253                         return -EPERM;
2254                 if (txc->modes && !ccs_capable(CCS_SYS_SETTIME))
2255                         return -EPERM;
2256                 /*
2257                  * if the quartz is off by more than 10% then
2258                  * something is VERY wrong!
2259                  */
2260                 if (txc->modes & ADJ_TICK &&
2261                     (txc->tick <  900000/USER_HZ ||
2262                      txc->tick > 1100000/USER_HZ))
2263                         return -EINVAL;
2264         }
2265 
2266         if (txc->modes & ADJ_SETOFFSET) {
2267                 /* In order to inject time, you gotta be super-user! */
2268                 if (!capable(CAP_SYS_TIME))
2269                         return -EPERM;
2270                 if (!ccs_capable(CCS_SYS_SETTIME))
2271                         return -EPERM;
2272 
2273                 /*
2274                  * Validate if a timespec/timeval used to inject a time
2275                  * offset is valid.  Offsets can be postive or negative, so
2276                  * we don't check tv_sec. The value of the timeval/timespec
2277                  * is the sum of its fields,but *NOTE*:
2278                  * The field tv_usec/tv_nsec must always be non-negative and
2279                  * we can't have more nanoseconds/microseconds than a second.
2280                  */
2281                 if (txc->time.tv_usec < 0)
2282                         return -EINVAL;
2283 
2284                 if (txc->modes & ADJ_NANO) {
2285                         if (txc->time.tv_usec >= NSEC_PER_SEC)
2286                                 return -EINVAL;
2287                 } else {
2288                         if (txc->time.tv_usec >= USEC_PER_SEC)
2289                                 return -EINVAL;
2290                 }
2291         }
2292 
2293         /*
2294          * Check for potential multiplication overflows that can
2295          * only happen on 64-bit systems:
2296          */
2297         if ((txc->modes & ADJ_FREQUENCY) && (BITS_PER_LONG == 64)) {
2298                 if (LLONG_MIN / PPM_SCALE > txc->freq)
2299                         return -EINVAL;
2300                 if (LLONG_MAX / PPM_SCALE < txc->freq)
2301                         return -EINVAL;
2302         }
2303 
2304         return 0;
2305 }
2306 
2307 
2308 /**
2309  * do_adjtimex() - Accessor function to NTP __do_adjtimex function
2310  */
2311 int do_adjtimex(struct timex *txc)
2312 {
2313         struct timekeeper *tk = &tk_core.timekeeper;
2314         unsigned long flags;
2315         struct timespec64 ts;
2316         s32 orig_tai, tai;
2317         int ret;
2318 
2319         /* Validate the data before disabling interrupts */
2320         ret = timekeeping_validate_timex(txc);
2321         if (ret)
2322                 return ret;
2323 
2324         if (txc->modes & ADJ_SETOFFSET) {
2325                 struct timespec64 delta;
2326                 delta.tv_sec  = txc->time.tv_sec;
2327                 delta.tv_nsec = txc->time.tv_usec;
2328                 if (!(txc->modes & ADJ_NANO))
2329                         delta.tv_nsec *= 1000;
2330                 ret = timekeeping_inject_offset(&delta);
2331                 if (ret)
2332                         return ret;
2333         }
2334 
2335         ktime_get_real_ts64(&ts);
2336 
2337         raw_spin_lock_irqsave(&timekeeper_lock, flags);
2338         write_seqcount_begin(&tk_core.seq);
2339 
2340         orig_tai = tai = tk->tai_offset;
2341         ret = __do_adjtimex(txc, &ts, &tai);
2342 
2343         if (tai != orig_tai) {
2344                 __timekeeping_set_tai_offset(tk, tai);
2345                 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
2346         }
2347         tk_update_leap_state(tk);
2348 
2349         write_seqcount_end(&tk_core.seq);
2350         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
2351 
2352         /* Update the multiplier immediately if frequency was set directly */
2353         if (txc->modes & (ADJ_FREQUENCY | ADJ_TICK))
2354                 timekeeping_advance(TK_ADV_FREQ);
2355 
2356         if (tai != orig_tai)
2357                 clock_was_set();
2358 
2359         ntp_notify_cmos_timer();
2360 
2361         return ret;
2362 }
2363 
2364 #ifdef CONFIG_NTP_PPS
2365 /**
2366  * hardpps() - Accessor function to NTP __hardpps function
2367  */
2368 void hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_ts)
2369 {
2370         unsigned long flags;
2371 
2372         raw_spin_lock_irqsave(&timekeeper_lock, flags);
2373         write_seqcount_begin(&tk_core.seq);
2374 
2375         __hardpps(phase_ts, raw_ts);
2376 
2377         write_seqcount_end(&tk_core.seq);
2378         raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
2379 }
2380 EXPORT_SYMBOL(hardpps);
2381 #endif /* CONFIG_NTP_PPS */
2382 
2383 /**
2384  * xtime_update() - advances the timekeeping infrastructure
2385  * @ticks:      number of ticks, that have elapsed since the last call.
2386  *
2387  * Must be called with interrupts disabled.
2388  */
2389 void xtime_update(unsigned long ticks)
2390 {
2391         write_seqlock(&jiffies_lock);
2392         do_timer(ticks);
2393         write_sequnlock(&jiffies_lock);
2394         update_wall_time();
2395 }
2396 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp