~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/time/clocksource.c

Version: ~ [ linux-5.2-rc1 ] ~ [ linux-5.1.2 ] ~ [ linux-5.0.16 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.43 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.119 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.176 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.179 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.139 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.67 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0+
  2 /*
  3  * This file contains the functions which manage clocksource drivers.
  4  *
  5  * Copyright (C) 2004, 2005 IBM, John Stultz (johnstul@us.ibm.com)
  6  */
  7 
  8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  9 
 10 #include <linux/device.h>
 11 #include <linux/clocksource.h>
 12 #include <linux/init.h>
 13 #include <linux/module.h>
 14 #include <linux/sched.h> /* for spin_unlock_irq() using preempt_count() m68k */
 15 #include <linux/tick.h>
 16 #include <linux/kthread.h>
 17 
 18 #include "tick-internal.h"
 19 #include "timekeeping_internal.h"
 20 
 21 /**
 22  * clocks_calc_mult_shift - calculate mult/shift factors for scaled math of clocks
 23  * @mult:       pointer to mult variable
 24  * @shift:      pointer to shift variable
 25  * @from:       frequency to convert from
 26  * @to:         frequency to convert to
 27  * @maxsec:     guaranteed runtime conversion range in seconds
 28  *
 29  * The function evaluates the shift/mult pair for the scaled math
 30  * operations of clocksources and clockevents.
 31  *
 32  * @to and @from are frequency values in HZ. For clock sources @to is
 33  * NSEC_PER_SEC == 1GHz and @from is the counter frequency. For clock
 34  * event @to is the counter frequency and @from is NSEC_PER_SEC.
 35  *
 36  * The @maxsec conversion range argument controls the time frame in
 37  * seconds which must be covered by the runtime conversion with the
 38  * calculated mult and shift factors. This guarantees that no 64bit
 39  * overflow happens when the input value of the conversion is
 40  * multiplied with the calculated mult factor. Larger ranges may
 41  * reduce the conversion accuracy by chosing smaller mult and shift
 42  * factors.
 43  */
 44 void
 45 clocks_calc_mult_shift(u32 *mult, u32 *shift, u32 from, u32 to, u32 maxsec)
 46 {
 47         u64 tmp;
 48         u32 sft, sftacc= 32;
 49 
 50         /*
 51          * Calculate the shift factor which is limiting the conversion
 52          * range:
 53          */
 54         tmp = ((u64)maxsec * from) >> 32;
 55         while (tmp) {
 56                 tmp >>=1;
 57                 sftacc--;
 58         }
 59 
 60         /*
 61          * Find the conversion shift/mult pair which has the best
 62          * accuracy and fits the maxsec conversion range:
 63          */
 64         for (sft = 32; sft > 0; sft--) {
 65                 tmp = (u64) to << sft;
 66                 tmp += from / 2;
 67                 do_div(tmp, from);
 68                 if ((tmp >> sftacc) == 0)
 69                         break;
 70         }
 71         *mult = tmp;
 72         *shift = sft;
 73 }
 74 EXPORT_SYMBOL_GPL(clocks_calc_mult_shift);
 75 
 76 /*[Clocksource internal variables]---------
 77  * curr_clocksource:
 78  *      currently selected clocksource.
 79  * suspend_clocksource:
 80  *      used to calculate the suspend time.
 81  * clocksource_list:
 82  *      linked list with the registered clocksources
 83  * clocksource_mutex:
 84  *      protects manipulations to curr_clocksource and the clocksource_list
 85  * override_name:
 86  *      Name of the user-specified clocksource.
 87  */
 88 static struct clocksource *curr_clocksource;
 89 static struct clocksource *suspend_clocksource;
 90 static LIST_HEAD(clocksource_list);
 91 static DEFINE_MUTEX(clocksource_mutex);
 92 static char override_name[CS_NAME_LEN];
 93 static int finished_booting;
 94 static u64 suspend_start;
 95 
 96 #ifdef CONFIG_CLOCKSOURCE_WATCHDOG
 97 static void clocksource_watchdog_work(struct work_struct *work);
 98 static void clocksource_select(void);
 99 
100 static LIST_HEAD(watchdog_list);
101 static struct clocksource *watchdog;
102 static struct timer_list watchdog_timer;
103 static DECLARE_WORK(watchdog_work, clocksource_watchdog_work);
104 static DEFINE_SPINLOCK(watchdog_lock);
105 static int watchdog_running;
106 static atomic_t watchdog_reset_pending;
107 
108 static void inline clocksource_watchdog_lock(unsigned long *flags)
109 {
110         spin_lock_irqsave(&watchdog_lock, *flags);
111 }
112 
113 static void inline clocksource_watchdog_unlock(unsigned long *flags)
114 {
115         spin_unlock_irqrestore(&watchdog_lock, *flags);
116 }
117 
118 static int clocksource_watchdog_kthread(void *data);
119 static void __clocksource_change_rating(struct clocksource *cs, int rating);
120 
121 /*
122  * Interval: 0.5sec Threshold: 0.0625s
123  */
124 #define WATCHDOG_INTERVAL (HZ >> 1)
125 #define WATCHDOG_THRESHOLD (NSEC_PER_SEC >> 4)
126 
127 static void clocksource_watchdog_work(struct work_struct *work)
128 {
129         /*
130          * We cannot directly run clocksource_watchdog_kthread() here, because
131          * clocksource_select() calls timekeeping_notify() which uses
132          * stop_machine(). One cannot use stop_machine() from a workqueue() due
133          * lock inversions wrt CPU hotplug.
134          *
135          * Also, we only ever run this work once or twice during the lifetime
136          * of the kernel, so there is no point in creating a more permanent
137          * kthread for this.
138          *
139          * If kthread_run fails the next watchdog scan over the
140          * watchdog_list will find the unstable clock again.
141          */
142         kthread_run(clocksource_watchdog_kthread, NULL, "kwatchdog");
143 }
144 
145 static void __clocksource_unstable(struct clocksource *cs)
146 {
147         cs->flags &= ~(CLOCK_SOURCE_VALID_FOR_HRES | CLOCK_SOURCE_WATCHDOG);
148         cs->flags |= CLOCK_SOURCE_UNSTABLE;
149 
150         /*
151          * If the clocksource is registered clocksource_watchdog_kthread() will
152          * re-rate and re-select.
153          */
154         if (list_empty(&cs->list)) {
155                 cs->rating = 0;
156                 return;
157         }
158 
159         if (cs->mark_unstable)
160                 cs->mark_unstable(cs);
161 
162         /* kick clocksource_watchdog_kthread() */
163         if (finished_booting)
164                 schedule_work(&watchdog_work);
165 }
166 
167 /**
168  * clocksource_mark_unstable - mark clocksource unstable via watchdog
169  * @cs:         clocksource to be marked unstable
170  *
171  * This function is called by the x86 TSC code to mark clocksources as unstable;
172  * it defers demotion and re-selection to a kthread.
173  */
174 void clocksource_mark_unstable(struct clocksource *cs)
175 {
176         unsigned long flags;
177 
178         spin_lock_irqsave(&watchdog_lock, flags);
179         if (!(cs->flags & CLOCK_SOURCE_UNSTABLE)) {
180                 if (!list_empty(&cs->list) && list_empty(&cs->wd_list))
181                         list_add(&cs->wd_list, &watchdog_list);
182                 __clocksource_unstable(cs);
183         }
184         spin_unlock_irqrestore(&watchdog_lock, flags);
185 }
186 
187 static void clocksource_watchdog(struct timer_list *unused)
188 {
189         struct clocksource *cs;
190         u64 csnow, wdnow, cslast, wdlast, delta;
191         int64_t wd_nsec, cs_nsec;
192         int next_cpu, reset_pending;
193 
194         spin_lock(&watchdog_lock);
195         if (!watchdog_running)
196                 goto out;
197 
198         reset_pending = atomic_read(&watchdog_reset_pending);
199 
200         list_for_each_entry(cs, &watchdog_list, wd_list) {
201 
202                 /* Clocksource already marked unstable? */
203                 if (cs->flags & CLOCK_SOURCE_UNSTABLE) {
204                         if (finished_booting)
205                                 schedule_work(&watchdog_work);
206                         continue;
207                 }
208 
209                 local_irq_disable();
210                 csnow = cs->read(cs);
211                 wdnow = watchdog->read(watchdog);
212                 local_irq_enable();
213 
214                 /* Clocksource initialized ? */
215                 if (!(cs->flags & CLOCK_SOURCE_WATCHDOG) ||
216                     atomic_read(&watchdog_reset_pending)) {
217                         cs->flags |= CLOCK_SOURCE_WATCHDOG;
218                         cs->wd_last = wdnow;
219                         cs->cs_last = csnow;
220                         continue;
221                 }
222 
223                 delta = clocksource_delta(wdnow, cs->wd_last, watchdog->mask);
224                 wd_nsec = clocksource_cyc2ns(delta, watchdog->mult,
225                                              watchdog->shift);
226 
227                 delta = clocksource_delta(csnow, cs->cs_last, cs->mask);
228                 cs_nsec = clocksource_cyc2ns(delta, cs->mult, cs->shift);
229                 wdlast = cs->wd_last; /* save these in case we print them */
230                 cslast = cs->cs_last;
231                 cs->cs_last = csnow;
232                 cs->wd_last = wdnow;
233 
234                 if (atomic_read(&watchdog_reset_pending))
235                         continue;
236 
237                 /* Check the deviation from the watchdog clocksource. */
238                 if (abs(cs_nsec - wd_nsec) > WATCHDOG_THRESHOLD) {
239                         pr_warn("timekeeping watchdog on CPU%d: Marking clocksource '%s' as unstable because the skew is too large:\n",
240                                 smp_processor_id(), cs->name);
241                         pr_warn("                      '%s' wd_now: %llx wd_last: %llx mask: %llx\n",
242                                 watchdog->name, wdnow, wdlast, watchdog->mask);
243                         pr_warn("                      '%s' cs_now: %llx cs_last: %llx mask: %llx\n",
244                                 cs->name, csnow, cslast, cs->mask);
245                         __clocksource_unstable(cs);
246                         continue;
247                 }
248 
249                 if (cs == curr_clocksource && cs->tick_stable)
250                         cs->tick_stable(cs);
251 
252                 if (!(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES) &&
253                     (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) &&
254                     (watchdog->flags & CLOCK_SOURCE_IS_CONTINUOUS)) {
255                         /* Mark it valid for high-res. */
256                         cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;
257 
258                         /*
259                          * clocksource_done_booting() will sort it if
260                          * finished_booting is not set yet.
261                          */
262                         if (!finished_booting)
263                                 continue;
264 
265                         /*
266                          * If this is not the current clocksource let
267                          * the watchdog thread reselect it. Due to the
268                          * change to high res this clocksource might
269                          * be preferred now. If it is the current
270                          * clocksource let the tick code know about
271                          * that change.
272                          */
273                         if (cs != curr_clocksource) {
274                                 cs->flags |= CLOCK_SOURCE_RESELECT;
275                                 schedule_work(&watchdog_work);
276                         } else {
277                                 tick_clock_notify();
278                         }
279                 }
280         }
281 
282         /*
283          * We only clear the watchdog_reset_pending, when we did a
284          * full cycle through all clocksources.
285          */
286         if (reset_pending)
287                 atomic_dec(&watchdog_reset_pending);
288 
289         /*
290          * Cycle through CPUs to check if the CPUs stay synchronized
291          * to each other.
292          */
293         next_cpu = cpumask_next(raw_smp_processor_id(), cpu_online_mask);
294         if (next_cpu >= nr_cpu_ids)
295                 next_cpu = cpumask_first(cpu_online_mask);
296         watchdog_timer.expires += WATCHDOG_INTERVAL;
297         add_timer_on(&watchdog_timer, next_cpu);
298 out:
299         spin_unlock(&watchdog_lock);
300 }
301 
302 static inline void clocksource_start_watchdog(void)
303 {
304         if (watchdog_running || !watchdog || list_empty(&watchdog_list))
305                 return;
306         timer_setup(&watchdog_timer, clocksource_watchdog, 0);
307         watchdog_timer.expires = jiffies + WATCHDOG_INTERVAL;
308         add_timer_on(&watchdog_timer, cpumask_first(cpu_online_mask));
309         watchdog_running = 1;
310 }
311 
312 static inline void clocksource_stop_watchdog(void)
313 {
314         if (!watchdog_running || (watchdog && !list_empty(&watchdog_list)))
315                 return;
316         del_timer(&watchdog_timer);
317         watchdog_running = 0;
318 }
319 
320 static inline void clocksource_reset_watchdog(void)
321 {
322         struct clocksource *cs;
323 
324         list_for_each_entry(cs, &watchdog_list, wd_list)
325                 cs->flags &= ~CLOCK_SOURCE_WATCHDOG;
326 }
327 
328 static void clocksource_resume_watchdog(void)
329 {
330         atomic_inc(&watchdog_reset_pending);
331 }
332 
333 static void clocksource_enqueue_watchdog(struct clocksource *cs)
334 {
335         INIT_LIST_HEAD(&cs->wd_list);
336 
337         if (cs->flags & CLOCK_SOURCE_MUST_VERIFY) {
338                 /* cs is a clocksource to be watched. */
339                 list_add(&cs->wd_list, &watchdog_list);
340                 cs->flags &= ~CLOCK_SOURCE_WATCHDOG;
341         } else {
342                 /* cs is a watchdog. */
343                 if (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS)
344                         cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;
345         }
346 }
347 
348 static void clocksource_select_watchdog(bool fallback)
349 {
350         struct clocksource *cs, *old_wd;
351         unsigned long flags;
352 
353         spin_lock_irqsave(&watchdog_lock, flags);
354         /* save current watchdog */
355         old_wd = watchdog;
356         if (fallback)
357                 watchdog = NULL;
358 
359         list_for_each_entry(cs, &clocksource_list, list) {
360                 /* cs is a clocksource to be watched. */
361                 if (cs->flags & CLOCK_SOURCE_MUST_VERIFY)
362                         continue;
363 
364                 /* Skip current if we were requested for a fallback. */
365                 if (fallback && cs == old_wd)
366                         continue;
367 
368                 /* Pick the best watchdog. */
369                 if (!watchdog || cs->rating > watchdog->rating)
370                         watchdog = cs;
371         }
372         /* If we failed to find a fallback restore the old one. */
373         if (!watchdog)
374                 watchdog = old_wd;
375 
376         /* If we changed the watchdog we need to reset cycles. */
377         if (watchdog != old_wd)
378                 clocksource_reset_watchdog();
379 
380         /* Check if the watchdog timer needs to be started. */
381         clocksource_start_watchdog();
382         spin_unlock_irqrestore(&watchdog_lock, flags);
383 }
384 
385 static void clocksource_dequeue_watchdog(struct clocksource *cs)
386 {
387         if (cs != watchdog) {
388                 if (cs->flags & CLOCK_SOURCE_MUST_VERIFY) {
389                         /* cs is a watched clocksource. */
390                         list_del_init(&cs->wd_list);
391                         /* Check if the watchdog timer needs to be stopped. */
392                         clocksource_stop_watchdog();
393                 }
394         }
395 }
396 
397 static int __clocksource_watchdog_kthread(void)
398 {
399         struct clocksource *cs, *tmp;
400         unsigned long flags;
401         int select = 0;
402 
403         spin_lock_irqsave(&watchdog_lock, flags);
404         list_for_each_entry_safe(cs, tmp, &watchdog_list, wd_list) {
405                 if (cs->flags & CLOCK_SOURCE_UNSTABLE) {
406                         list_del_init(&cs->wd_list);
407                         __clocksource_change_rating(cs, 0);
408                         select = 1;
409                 }
410                 if (cs->flags & CLOCK_SOURCE_RESELECT) {
411                         cs->flags &= ~CLOCK_SOURCE_RESELECT;
412                         select = 1;
413                 }
414         }
415         /* Check if the watchdog timer needs to be stopped. */
416         clocksource_stop_watchdog();
417         spin_unlock_irqrestore(&watchdog_lock, flags);
418 
419         return select;
420 }
421 
422 static int clocksource_watchdog_kthread(void *data)
423 {
424         mutex_lock(&clocksource_mutex);
425         if (__clocksource_watchdog_kthread())
426                 clocksource_select();
427         mutex_unlock(&clocksource_mutex);
428         return 0;
429 }
430 
431 static bool clocksource_is_watchdog(struct clocksource *cs)
432 {
433         return cs == watchdog;
434 }
435 
436 #else /* CONFIG_CLOCKSOURCE_WATCHDOG */
437 
438 static void clocksource_enqueue_watchdog(struct clocksource *cs)
439 {
440         if (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS)
441                 cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;
442 }
443 
444 static void clocksource_select_watchdog(bool fallback) { }
445 static inline void clocksource_dequeue_watchdog(struct clocksource *cs) { }
446 static inline void clocksource_resume_watchdog(void) { }
447 static inline int __clocksource_watchdog_kthread(void) { return 0; }
448 static bool clocksource_is_watchdog(struct clocksource *cs) { return false; }
449 void clocksource_mark_unstable(struct clocksource *cs) { }
450 
451 static inline void clocksource_watchdog_lock(unsigned long *flags) { }
452 static inline void clocksource_watchdog_unlock(unsigned long *flags) { }
453 
454 #endif /* CONFIG_CLOCKSOURCE_WATCHDOG */
455 
456 static bool clocksource_is_suspend(struct clocksource *cs)
457 {
458         return cs == suspend_clocksource;
459 }
460 
461 static void __clocksource_suspend_select(struct clocksource *cs)
462 {
463         /*
464          * Skip the clocksource which will be stopped in suspend state.
465          */
466         if (!(cs->flags & CLOCK_SOURCE_SUSPEND_NONSTOP))
467                 return;
468 
469         /*
470          * The nonstop clocksource can be selected as the suspend clocksource to
471          * calculate the suspend time, so it should not supply suspend/resume
472          * interfaces to suspend the nonstop clocksource when system suspends.
473          */
474         if (cs->suspend || cs->resume) {
475                 pr_warn("Nonstop clocksource %s should not supply suspend/resume interfaces\n",
476                         cs->name);
477         }
478 
479         /* Pick the best rating. */
480         if (!suspend_clocksource || cs->rating > suspend_clocksource->rating)
481                 suspend_clocksource = cs;
482 }
483 
484 /**
485  * clocksource_suspend_select - Select the best clocksource for suspend timing
486  * @fallback:   if select a fallback clocksource
487  */
488 static void clocksource_suspend_select(bool fallback)
489 {
490         struct clocksource *cs, *old_suspend;
491 
492         old_suspend = suspend_clocksource;
493         if (fallback)
494                 suspend_clocksource = NULL;
495 
496         list_for_each_entry(cs, &clocksource_list, list) {
497                 /* Skip current if we were requested for a fallback. */
498                 if (fallback && cs == old_suspend)
499                         continue;
500 
501                 __clocksource_suspend_select(cs);
502         }
503 }
504 
505 /**
506  * clocksource_start_suspend_timing - Start measuring the suspend timing
507  * @cs:                 current clocksource from timekeeping
508  * @start_cycles:       current cycles from timekeeping
509  *
510  * This function will save the start cycle values of suspend timer to calculate
511  * the suspend time when resuming system.
512  *
513  * This function is called late in the suspend process from timekeeping_suspend(),
514  * that means processes are freezed, non-boot cpus and interrupts are disabled
515  * now. It is therefore possible to start the suspend timer without taking the
516  * clocksource mutex.
517  */
518 void clocksource_start_suspend_timing(struct clocksource *cs, u64 start_cycles)
519 {
520         if (!suspend_clocksource)
521                 return;
522 
523         /*
524          * If current clocksource is the suspend timer, we should use the
525          * tkr_mono.cycle_last value as suspend_start to avoid same reading
526          * from suspend timer.
527          */
528         if (clocksource_is_suspend(cs)) {
529                 suspend_start = start_cycles;
530                 return;
531         }
532 
533         if (suspend_clocksource->enable &&
534             suspend_clocksource->enable(suspend_clocksource)) {
535                 pr_warn_once("Failed to enable the non-suspend-able clocksource.\n");
536                 return;
537         }
538 
539         suspend_start = suspend_clocksource->read(suspend_clocksource);
540 }
541 
542 /**
543  * clocksource_stop_suspend_timing - Stop measuring the suspend timing
544  * @cs:         current clocksource from timekeeping
545  * @cycle_now:  current cycles from timekeeping
546  *
547  * This function will calculate the suspend time from suspend timer.
548  *
549  * Returns nanoseconds since suspend started, 0 if no usable suspend clocksource.
550  *
551  * This function is called early in the resume process from timekeeping_resume(),
552  * that means there is only one cpu, no processes are running and the interrupts
553  * are disabled. It is therefore possible to stop the suspend timer without
554  * taking the clocksource mutex.
555  */
556 u64 clocksource_stop_suspend_timing(struct clocksource *cs, u64 cycle_now)
557 {
558         u64 now, delta, nsec = 0;
559 
560         if (!suspend_clocksource)
561                 return 0;
562 
563         /*
564          * If current clocksource is the suspend timer, we should use the
565          * tkr_mono.cycle_last value from timekeeping as current cycle to
566          * avoid same reading from suspend timer.
567          */
568         if (clocksource_is_suspend(cs))
569                 now = cycle_now;
570         else
571                 now = suspend_clocksource->read(suspend_clocksource);
572 
573         if (now > suspend_start) {
574                 delta = clocksource_delta(now, suspend_start,
575                                           suspend_clocksource->mask);
576                 nsec = mul_u64_u32_shr(delta, suspend_clocksource->mult,
577                                        suspend_clocksource->shift);
578         }
579 
580         /*
581          * Disable the suspend timer to save power if current clocksource is
582          * not the suspend timer.
583          */
584         if (!clocksource_is_suspend(cs) && suspend_clocksource->disable)
585                 suspend_clocksource->disable(suspend_clocksource);
586 
587         return nsec;
588 }
589 
590 /**
591  * clocksource_suspend - suspend the clocksource(s)
592  */
593 void clocksource_suspend(void)
594 {
595         struct clocksource *cs;
596 
597         list_for_each_entry_reverse(cs, &clocksource_list, list)
598                 if (cs->suspend)
599                         cs->suspend(cs);
600 }
601 
602 /**
603  * clocksource_resume - resume the clocksource(s)
604  */
605 void clocksource_resume(void)
606 {
607         struct clocksource *cs;
608 
609         list_for_each_entry(cs, &clocksource_list, list)
610                 if (cs->resume)
611                         cs->resume(cs);
612 
613         clocksource_resume_watchdog();
614 }
615 
616 /**
617  * clocksource_touch_watchdog - Update watchdog
618  *
619  * Update the watchdog after exception contexts such as kgdb so as not
620  * to incorrectly trip the watchdog. This might fail when the kernel
621  * was stopped in code which holds watchdog_lock.
622  */
623 void clocksource_touch_watchdog(void)
624 {
625         clocksource_resume_watchdog();
626 }
627 
628 /**
629  * clocksource_max_adjustment- Returns max adjustment amount
630  * @cs:         Pointer to clocksource
631  *
632  */
633 static u32 clocksource_max_adjustment(struct clocksource *cs)
634 {
635         u64 ret;
636         /*
637          * We won't try to correct for more than 11% adjustments (110,000 ppm),
638          */
639         ret = (u64)cs->mult * 11;
640         do_div(ret,100);
641         return (u32)ret;
642 }
643 
644 /**
645  * clocks_calc_max_nsecs - Returns maximum nanoseconds that can be converted
646  * @mult:       cycle to nanosecond multiplier
647  * @shift:      cycle to nanosecond divisor (power of two)
648  * @maxadj:     maximum adjustment value to mult (~11%)
649  * @mask:       bitmask for two's complement subtraction of non 64 bit counters
650  * @max_cyc:    maximum cycle value before potential overflow (does not include
651  *              any safety margin)
652  *
653  * NOTE: This function includes a safety margin of 50%, in other words, we
654  * return half the number of nanoseconds the hardware counter can technically
655  * cover. This is done so that we can potentially detect problems caused by
656  * delayed timers or bad hardware, which might result in time intervals that
657  * are larger than what the math used can handle without overflows.
658  */
659 u64 clocks_calc_max_nsecs(u32 mult, u32 shift, u32 maxadj, u64 mask, u64 *max_cyc)
660 {
661         u64 max_nsecs, max_cycles;
662 
663         /*
664          * Calculate the maximum number of cycles that we can pass to the
665          * cyc2ns() function without overflowing a 64-bit result.
666          */
667         max_cycles = ULLONG_MAX;
668         do_div(max_cycles, mult+maxadj);
669 
670         /*
671          * The actual maximum number of cycles we can defer the clocksource is
672          * determined by the minimum of max_cycles and mask.
673          * Note: Here we subtract the maxadj to make sure we don't sleep for
674          * too long if there's a large negative adjustment.
675          */
676         max_cycles = min(max_cycles, mask);
677         max_nsecs = clocksource_cyc2ns(max_cycles, mult - maxadj, shift);
678 
679         /* return the max_cycles value as well if requested */
680         if (max_cyc)
681                 *max_cyc = max_cycles;
682 
683         /* Return 50% of the actual maximum, so we can detect bad values */
684         max_nsecs >>= 1;
685 
686         return max_nsecs;
687 }
688 
689 /**
690  * clocksource_update_max_deferment - Updates the clocksource max_idle_ns & max_cycles
691  * @cs:         Pointer to clocksource to be updated
692  *
693  */
694 static inline void clocksource_update_max_deferment(struct clocksource *cs)
695 {
696         cs->max_idle_ns = clocks_calc_max_nsecs(cs->mult, cs->shift,
697                                                 cs->maxadj, cs->mask,
698                                                 &cs->max_cycles);
699 }
700 
701 #ifndef CONFIG_ARCH_USES_GETTIMEOFFSET
702 
703 static struct clocksource *clocksource_find_best(bool oneshot, bool skipcur)
704 {
705         struct clocksource *cs;
706 
707         if (!finished_booting || list_empty(&clocksource_list))
708                 return NULL;
709 
710         /*
711          * We pick the clocksource with the highest rating. If oneshot
712          * mode is active, we pick the highres valid clocksource with
713          * the best rating.
714          */
715         list_for_each_entry(cs, &clocksource_list, list) {
716                 if (skipcur && cs == curr_clocksource)
717                         continue;
718                 if (oneshot && !(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES))
719                         continue;
720                 return cs;
721         }
722         return NULL;
723 }
724 
725 static void __clocksource_select(bool skipcur)
726 {
727         bool oneshot = tick_oneshot_mode_active();
728         struct clocksource *best, *cs;
729 
730         /* Find the best suitable clocksource */
731         best = clocksource_find_best(oneshot, skipcur);
732         if (!best)
733                 return;
734 
735         if (!strlen(override_name))
736                 goto found;
737 
738         /* Check for the override clocksource. */
739         list_for_each_entry(cs, &clocksource_list, list) {
740                 if (skipcur && cs == curr_clocksource)
741                         continue;
742                 if (strcmp(cs->name, override_name) != 0)
743                         continue;
744                 /*
745                  * Check to make sure we don't switch to a non-highres
746                  * capable clocksource if the tick code is in oneshot
747                  * mode (highres or nohz)
748                  */
749                 if (!(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES) && oneshot) {
750                         /* Override clocksource cannot be used. */
751                         if (cs->flags & CLOCK_SOURCE_UNSTABLE) {
752                                 pr_warn("Override clocksource %s is unstable and not HRT compatible - cannot switch while in HRT/NOHZ mode\n",
753                                         cs->name);
754                                 override_name[0] = 0;
755                         } else {
756                                 /*
757                                  * The override cannot be currently verified.
758                                  * Deferring to let the watchdog check.
759                                  */
760                                 pr_info("Override clocksource %s is not currently HRT compatible - deferring\n",
761                                         cs->name);
762                         }
763                 } else
764                         /* Override clocksource can be used. */
765                         best = cs;
766                 break;
767         }
768 
769 found:
770         if (curr_clocksource != best && !timekeeping_notify(best)) {
771                 pr_info("Switched to clocksource %s\n", best->name);
772                 curr_clocksource = best;
773         }
774 }
775 
776 /**
777  * clocksource_select - Select the best clocksource available
778  *
779  * Private function. Must hold clocksource_mutex when called.
780  *
781  * Select the clocksource with the best rating, or the clocksource,
782  * which is selected by userspace override.
783  */
784 static void clocksource_select(void)
785 {
786         __clocksource_select(false);
787 }
788 
789 static void clocksource_select_fallback(void)
790 {
791         __clocksource_select(true);
792 }
793 
794 #else /* !CONFIG_ARCH_USES_GETTIMEOFFSET */
795 static inline void clocksource_select(void) { }
796 static inline void clocksource_select_fallback(void) { }
797 
798 #endif
799 
800 /*
801  * clocksource_done_booting - Called near the end of core bootup
802  *
803  * Hack to avoid lots of clocksource churn at boot time.
804  * We use fs_initcall because we want this to start before
805  * device_initcall but after subsys_initcall.
806  */
807 static int __init clocksource_done_booting(void)
808 {
809         mutex_lock(&clocksource_mutex);
810         curr_clocksource = clocksource_default_clock();
811         finished_booting = 1;
812         /*
813          * Run the watchdog first to eliminate unstable clock sources
814          */
815         __clocksource_watchdog_kthread();
816         clocksource_select();
817         mutex_unlock(&clocksource_mutex);
818         return 0;
819 }
820 fs_initcall(clocksource_done_booting);
821 
822 /*
823  * Enqueue the clocksource sorted by rating
824  */
825 static void clocksource_enqueue(struct clocksource *cs)
826 {
827         struct list_head *entry = &clocksource_list;
828         struct clocksource *tmp;
829 
830         list_for_each_entry(tmp, &clocksource_list, list) {
831                 /* Keep track of the place, where to insert */
832                 if (tmp->rating < cs->rating)
833                         break;
834                 entry = &tmp->list;
835         }
836         list_add(&cs->list, entry);
837 }
838 
839 /**
840  * __clocksource_update_freq_scale - Used update clocksource with new freq
841  * @cs:         clocksource to be registered
842  * @scale:      Scale factor multiplied against freq to get clocksource hz
843  * @freq:       clocksource frequency (cycles per second) divided by scale
844  *
845  * This should only be called from the clocksource->enable() method.
846  *
847  * This *SHOULD NOT* be called directly! Please use the
848  * __clocksource_update_freq_hz() or __clocksource_update_freq_khz() helper
849  * functions.
850  */
851 void __clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq)
852 {
853         u64 sec;
854 
855         /*
856          * Default clocksources are *special* and self-define their mult/shift.
857          * But, you're not special, so you should specify a freq value.
858          */
859         if (freq) {
860                 /*
861                  * Calc the maximum number of seconds which we can run before
862                  * wrapping around. For clocksources which have a mask > 32-bit
863                  * we need to limit the max sleep time to have a good
864                  * conversion precision. 10 minutes is still a reasonable
865                  * amount. That results in a shift value of 24 for a
866                  * clocksource with mask >= 40-bit and f >= 4GHz. That maps to
867                  * ~ 0.06ppm granularity for NTP.
868                  */
869                 sec = cs->mask;
870                 do_div(sec, freq);
871                 do_div(sec, scale);
872                 if (!sec)
873                         sec = 1;
874                 else if (sec > 600 && cs->mask > UINT_MAX)
875                         sec = 600;
876 
877                 clocks_calc_mult_shift(&cs->mult, &cs->shift, freq,
878                                        NSEC_PER_SEC / scale, sec * scale);
879         }
880         /*
881          * Ensure clocksources that have large 'mult' values don't overflow
882          * when adjusted.
883          */
884         cs->maxadj = clocksource_max_adjustment(cs);
885         while (freq && ((cs->mult + cs->maxadj < cs->mult)
886                 || (cs->mult - cs->maxadj > cs->mult))) {
887                 cs->mult >>= 1;
888                 cs->shift--;
889                 cs->maxadj = clocksource_max_adjustment(cs);
890         }
891 
892         /*
893          * Only warn for *special* clocksources that self-define
894          * their mult/shift values and don't specify a freq.
895          */
896         WARN_ONCE(cs->mult + cs->maxadj < cs->mult,
897                 "timekeeping: Clocksource %s might overflow on 11%% adjustment\n",
898                 cs->name);
899 
900         clocksource_update_max_deferment(cs);
901 
902         pr_info("%s: mask: 0x%llx max_cycles: 0x%llx, max_idle_ns: %lld ns\n",
903                 cs->name, cs->mask, cs->max_cycles, cs->max_idle_ns);
904 }
905 EXPORT_SYMBOL_GPL(__clocksource_update_freq_scale);
906 
907 /**
908  * __clocksource_register_scale - Used to install new clocksources
909  * @cs:         clocksource to be registered
910  * @scale:      Scale factor multiplied against freq to get clocksource hz
911  * @freq:       clocksource frequency (cycles per second) divided by scale
912  *
913  * Returns -EBUSY if registration fails, zero otherwise.
914  *
915  * This *SHOULD NOT* be called directly! Please use the
916  * clocksource_register_hz() or clocksource_register_khz helper functions.
917  */
918 int __clocksource_register_scale(struct clocksource *cs, u32 scale, u32 freq)
919 {
920         unsigned long flags;
921 
922         clocksource_arch_init(cs);
923 
924         /* Initialize mult/shift and max_idle_ns */
925         __clocksource_update_freq_scale(cs, scale, freq);
926 
927         /* Add clocksource to the clocksource list */
928         mutex_lock(&clocksource_mutex);
929 
930         clocksource_watchdog_lock(&flags);
931         clocksource_enqueue(cs);
932         clocksource_enqueue_watchdog(cs);
933         clocksource_watchdog_unlock(&flags);
934 
935         clocksource_select();
936         clocksource_select_watchdog(false);
937         __clocksource_suspend_select(cs);
938         mutex_unlock(&clocksource_mutex);
939         return 0;
940 }
941 EXPORT_SYMBOL_GPL(__clocksource_register_scale);
942 
943 static void __clocksource_change_rating(struct clocksource *cs, int rating)
944 {
945         list_del(&cs->list);
946         cs->rating = rating;
947         clocksource_enqueue(cs);
948 }
949 
950 /**
951  * clocksource_change_rating - Change the rating of a registered clocksource
952  * @cs:         clocksource to be changed
953  * @rating:     new rating
954  */
955 void clocksource_change_rating(struct clocksource *cs, int rating)
956 {
957         unsigned long flags;
958 
959         mutex_lock(&clocksource_mutex);
960         clocksource_watchdog_lock(&flags);
961         __clocksource_change_rating(cs, rating);
962         clocksource_watchdog_unlock(&flags);
963 
964         clocksource_select();
965         clocksource_select_watchdog(false);
966         clocksource_suspend_select(false);
967         mutex_unlock(&clocksource_mutex);
968 }
969 EXPORT_SYMBOL(clocksource_change_rating);
970 
971 /*
972  * Unbind clocksource @cs. Called with clocksource_mutex held
973  */
974 static int clocksource_unbind(struct clocksource *cs)
975 {
976         unsigned long flags;
977 
978         if (clocksource_is_watchdog(cs)) {
979                 /* Select and try to install a replacement watchdog. */
980                 clocksource_select_watchdog(true);
981                 if (clocksource_is_watchdog(cs))
982                         return -EBUSY;
983         }
984 
985         if (cs == curr_clocksource) {
986                 /* Select and try to install a replacement clock source */
987                 clocksource_select_fallback();
988                 if (curr_clocksource == cs)
989                         return -EBUSY;
990         }
991 
992         if (clocksource_is_suspend(cs)) {
993                 /*
994                  * Select and try to install a replacement suspend clocksource.
995                  * If no replacement suspend clocksource, we will just let the
996                  * clocksource go and have no suspend clocksource.
997                  */
998                 clocksource_suspend_select(true);
999         }
1000 
1001         clocksource_watchdog_lock(&flags);
1002         clocksource_dequeue_watchdog(cs);
1003         list_del_init(&cs->list);
1004         clocksource_watchdog_unlock(&flags);
1005 
1006         return 0;
1007 }
1008 
1009 /**
1010  * clocksource_unregister - remove a registered clocksource
1011  * @cs: clocksource to be unregistered
1012  */
1013 int clocksource_unregister(struct clocksource *cs)
1014 {
1015         int ret = 0;
1016 
1017         mutex_lock(&clocksource_mutex);
1018         if (!list_empty(&cs->list))
1019                 ret = clocksource_unbind(cs);
1020         mutex_unlock(&clocksource_mutex);
1021         return ret;
1022 }
1023 EXPORT_SYMBOL(clocksource_unregister);
1024 
1025 #ifdef CONFIG_SYSFS
1026 /**
1027  * current_clocksource_show - sysfs interface for current clocksource
1028  * @dev:        unused
1029  * @attr:       unused
1030  * @buf:        char buffer to be filled with clocksource list
1031  *
1032  * Provides sysfs interface for listing current clocksource.
1033  */
1034 static ssize_t current_clocksource_show(struct device *dev,
1035                                         struct device_attribute *attr,
1036                                         char *buf)
1037 {
1038         ssize_t count = 0;
1039 
1040         mutex_lock(&clocksource_mutex);
1041         count = snprintf(buf, PAGE_SIZE, "%s\n", curr_clocksource->name);
1042         mutex_unlock(&clocksource_mutex);
1043 
1044         return count;
1045 }
1046 
1047 ssize_t sysfs_get_uname(const char *buf, char *dst, size_t cnt)
1048 {
1049         size_t ret = cnt;
1050 
1051         /* strings from sysfs write are not 0 terminated! */
1052         if (!cnt || cnt >= CS_NAME_LEN)
1053                 return -EINVAL;
1054 
1055         /* strip of \n: */
1056         if (buf[cnt-1] == '\n')
1057                 cnt--;
1058         if (cnt > 0)
1059                 memcpy(dst, buf, cnt);
1060         dst[cnt] = 0;
1061         return ret;
1062 }
1063 
1064 /**
1065  * current_clocksource_store - interface for manually overriding clocksource
1066  * @dev:        unused
1067  * @attr:       unused
1068  * @buf:        name of override clocksource
1069  * @count:      length of buffer
1070  *
1071  * Takes input from sysfs interface for manually overriding the default
1072  * clocksource selection.
1073  */
1074 static ssize_t current_clocksource_store(struct device *dev,
1075                                          struct device_attribute *attr,
1076                                          const char *buf, size_t count)
1077 {
1078         ssize_t ret;
1079 
1080         mutex_lock(&clocksource_mutex);
1081 
1082         ret = sysfs_get_uname(buf, override_name, count);
1083         if (ret >= 0)
1084                 clocksource_select();
1085 
1086         mutex_unlock(&clocksource_mutex);
1087 
1088         return ret;
1089 }
1090 static DEVICE_ATTR_RW(current_clocksource);
1091 
1092 /**
1093  * unbind_clocksource_store - interface for manually unbinding clocksource
1094  * @dev:        unused
1095  * @attr:       unused
1096  * @buf:        unused
1097  * @count:      length of buffer
1098  *
1099  * Takes input from sysfs interface for manually unbinding a clocksource.
1100  */
1101 static ssize_t unbind_clocksource_store(struct device *dev,
1102                                         struct device_attribute *attr,
1103                                         const char *buf, size_t count)
1104 {
1105         struct clocksource *cs;
1106         char name[CS_NAME_LEN];
1107         ssize_t ret;
1108 
1109         ret = sysfs_get_uname(buf, name, count);
1110         if (ret < 0)
1111                 return ret;
1112 
1113         ret = -ENODEV;
1114         mutex_lock(&clocksource_mutex);
1115         list_for_each_entry(cs, &clocksource_list, list) {
1116                 if (strcmp(cs->name, name))
1117                         continue;
1118                 ret = clocksource_unbind(cs);
1119                 break;
1120         }
1121         mutex_unlock(&clocksource_mutex);
1122 
1123         return ret ? ret : count;
1124 }
1125 static DEVICE_ATTR_WO(unbind_clocksource);
1126 
1127 /**
1128  * available_clocksource_show - sysfs interface for listing clocksource
1129  * @dev:        unused
1130  * @attr:       unused
1131  * @buf:        char buffer to be filled with clocksource list
1132  *
1133  * Provides sysfs interface for listing registered clocksources
1134  */
1135 static ssize_t available_clocksource_show(struct device *dev,
1136                                           struct device_attribute *attr,
1137                                           char *buf)
1138 {
1139         struct clocksource *src;
1140         ssize_t count = 0;
1141 
1142         mutex_lock(&clocksource_mutex);
1143         list_for_each_entry(src, &clocksource_list, list) {
1144                 /*
1145                  * Don't show non-HRES clocksource if the tick code is
1146                  * in one shot mode (highres=on or nohz=on)
1147                  */
1148                 if (!tick_oneshot_mode_active() ||
1149                     (src->flags & CLOCK_SOURCE_VALID_FOR_HRES))
1150                         count += snprintf(buf + count,
1151                                   max((ssize_t)PAGE_SIZE - count, (ssize_t)0),
1152                                   "%s ", src->name);
1153         }
1154         mutex_unlock(&clocksource_mutex);
1155 
1156         count += snprintf(buf + count,
1157                           max((ssize_t)PAGE_SIZE - count, (ssize_t)0), "\n");
1158 
1159         return count;
1160 }
1161 static DEVICE_ATTR_RO(available_clocksource);
1162 
1163 static struct attribute *clocksource_attrs[] = {
1164         &dev_attr_current_clocksource.attr,
1165         &dev_attr_unbind_clocksource.attr,
1166         &dev_attr_available_clocksource.attr,
1167         NULL
1168 };
1169 ATTRIBUTE_GROUPS(clocksource);
1170 
1171 static struct bus_type clocksource_subsys = {
1172         .name = "clocksource",
1173         .dev_name = "clocksource",
1174 };
1175 
1176 static struct device device_clocksource = {
1177         .id     = 0,
1178         .bus    = &clocksource_subsys,
1179         .groups = clocksource_groups,
1180 };
1181 
1182 static int __init init_clocksource_sysfs(void)
1183 {
1184         int error = subsys_system_register(&clocksource_subsys, NULL);
1185 
1186         if (!error)
1187                 error = device_register(&device_clocksource);
1188 
1189         return error;
1190 }
1191 
1192 device_initcall(init_clocksource_sysfs);
1193 #endif /* CONFIG_SYSFS */
1194 
1195 /**
1196  * boot_override_clocksource - boot clock override
1197  * @str:        override name
1198  *
1199  * Takes a clocksource= boot argument and uses it
1200  * as the clocksource override name.
1201  */
1202 static int __init boot_override_clocksource(char* str)
1203 {
1204         mutex_lock(&clocksource_mutex);
1205         if (str)
1206                 strlcpy(override_name, str, sizeof(override_name));
1207         mutex_unlock(&clocksource_mutex);
1208         return 1;
1209 }
1210 
1211 __setup("clocksource=", boot_override_clocksource);
1212 
1213 /**
1214  * boot_override_clock - Compatibility layer for deprecated boot option
1215  * @str:        override name
1216  *
1217  * DEPRECATED! Takes a clock= boot argument and uses it
1218  * as the clocksource override name
1219  */
1220 static int __init boot_override_clock(char* str)
1221 {
1222         if (!strcmp(str, "pmtmr")) {
1223                 pr_warn("clock=pmtmr is deprecated - use clocksource=acpi_pm\n");
1224                 return boot_override_clocksource("acpi_pm");
1225         }
1226         pr_warn("clock= boot option is deprecated - use clocksource=xyz\n");
1227         return boot_override_clocksource(str);
1228 }
1229 
1230 __setup("clock=", boot_override_clock);
1231 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp