~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/time/sched_clock.c

Version: ~ [ linux-5.8-rc4 ] ~ [ linux-5.7.7 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.50 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.131 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.187 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.229 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.229 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * sched_clock.c: support for extending counters to full 64-bit ns counter
  3  *
  4  * This program is free software; you can redistribute it and/or modify
  5  * it under the terms of the GNU General Public License version 2 as
  6  * published by the Free Software Foundation.
  7  */
  8 #include <linux/clocksource.h>
  9 #include <linux/init.h>
 10 #include <linux/jiffies.h>
 11 #include <linux/kernel.h>
 12 #include <linux/moduleparam.h>
 13 #include <linux/sched.h>
 14 #include <linux/syscore_ops.h>
 15 #include <linux/timer.h>
 16 #include <linux/sched_clock.h>
 17 
 18 struct clock_data {
 19         u64 epoch_ns;
 20         u32 epoch_cyc;
 21         u32 epoch_cyc_copy;
 22         unsigned long rate;
 23         u32 mult;
 24         u32 shift;
 25         bool suspended;
 26 };
 27 
 28 static void sched_clock_poll(unsigned long wrap_ticks);
 29 static DEFINE_TIMER(sched_clock_timer, sched_clock_poll, 0, 0);
 30 static int irqtime = -1;
 31 
 32 core_param(irqtime, irqtime, int, 0400);
 33 
 34 static struct clock_data cd = {
 35         .mult   = NSEC_PER_SEC / HZ,
 36 };
 37 
 38 static u32 __read_mostly sched_clock_mask = 0xffffffff;
 39 
 40 static u32 notrace jiffy_sched_clock_read(void)
 41 {
 42         return (u32)(jiffies - INITIAL_JIFFIES);
 43 }
 44 
 45 static u32 __read_mostly (*read_sched_clock)(void) = jiffy_sched_clock_read;
 46 
 47 static inline u64 notrace cyc_to_ns(u64 cyc, u32 mult, u32 shift)
 48 {
 49         return (cyc * mult) >> shift;
 50 }
 51 
 52 static unsigned long long notrace sched_clock_32(void)
 53 {
 54         u64 epoch_ns;
 55         u32 epoch_cyc;
 56         u32 cyc;
 57 
 58         if (cd.suspended)
 59                 return cd.epoch_ns;
 60 
 61         /*
 62          * Load the epoch_cyc and epoch_ns atomically.  We do this by
 63          * ensuring that we always write epoch_cyc, epoch_ns and
 64          * epoch_cyc_copy in strict order, and read them in strict order.
 65          * If epoch_cyc and epoch_cyc_copy are not equal, then we're in
 66          * the middle of an update, and we should repeat the load.
 67          */
 68         do {
 69                 epoch_cyc = cd.epoch_cyc;
 70                 smp_rmb();
 71                 epoch_ns = cd.epoch_ns;
 72                 smp_rmb();
 73         } while (epoch_cyc != cd.epoch_cyc_copy);
 74 
 75         cyc = read_sched_clock();
 76         cyc = (cyc - epoch_cyc) & sched_clock_mask;
 77         return epoch_ns + cyc_to_ns(cyc, cd.mult, cd.shift);
 78 }
 79 
 80 /*
 81  * Atomically update the sched_clock epoch.
 82  */
 83 static void notrace update_sched_clock(void)
 84 {
 85         unsigned long flags;
 86         u32 cyc;
 87         u64 ns;
 88 
 89         cyc = read_sched_clock();
 90         ns = cd.epoch_ns +
 91                 cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask,
 92                           cd.mult, cd.shift);
 93         /*
 94          * Write epoch_cyc and epoch_ns in a way that the update is
 95          * detectable in cyc_to_fixed_sched_clock().
 96          */
 97         raw_local_irq_save(flags);
 98         cd.epoch_cyc_copy = cyc;
 99         smp_wmb();
100         cd.epoch_ns = ns;
101         smp_wmb();
102         cd.epoch_cyc = cyc;
103         raw_local_irq_restore(flags);
104 }
105 
106 static void sched_clock_poll(unsigned long wrap_ticks)
107 {
108         mod_timer(&sched_clock_timer, round_jiffies(jiffies + wrap_ticks));
109         update_sched_clock();
110 }
111 
112 void __init setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate)
113 {
114         unsigned long r, w;
115         u64 res, wrap;
116         char r_unit;
117 
118         if (cd.rate > rate)
119                 return;
120 
121         BUG_ON(bits > 32);
122         WARN_ON(!irqs_disabled());
123         read_sched_clock = read;
124         sched_clock_mask = (1ULL << bits) - 1;
125         cd.rate = rate;
126 
127         /* calculate the mult/shift to convert counter ticks to ns. */
128         clocks_calc_mult_shift(&cd.mult, &cd.shift, rate, NSEC_PER_SEC, 0);
129 
130         r = rate;
131         if (r >= 4000000) {
132                 r /= 1000000;
133                 r_unit = 'M';
134         } else if (r >= 1000) {
135                 r /= 1000;
136                 r_unit = 'k';
137         } else
138                 r_unit = ' ';
139 
140         /* calculate how many ns until we wrap */
141         wrap = cyc_to_ns((1ULL << bits) - 1, cd.mult, cd.shift);
142         do_div(wrap, NSEC_PER_MSEC);
143         w = wrap;
144 
145         /* calculate the ns resolution of this counter */
146         res = cyc_to_ns(1ULL, cd.mult, cd.shift);
147         pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lums\n",
148                 bits, r, r_unit, res, w);
149 
150         /*
151          * Start the timer to keep sched_clock() properly updated and
152          * sets the initial epoch.
153          */
154         sched_clock_timer.data = msecs_to_jiffies(w - (w / 10));
155         update_sched_clock();
156 
157         /*
158          * Ensure that sched_clock() starts off at 0ns
159          */
160         cd.epoch_ns = 0;
161 
162         /* Enable IRQ time accounting if we have a fast enough sched_clock */
163         if (irqtime > 0 || (irqtime == -1 && rate >= 1000000))
164                 enable_sched_clock_irqtime();
165 
166         pr_debug("Registered %pF as sched_clock source\n", read);
167 }
168 
169 unsigned long long __read_mostly (*sched_clock_func)(void) = sched_clock_32;
170 
171 unsigned long long notrace sched_clock(void)
172 {
173         return sched_clock_func();
174 }
175 
176 void __init sched_clock_postinit(void)
177 {
178         /*
179          * If no sched_clock function has been provided at that point,
180          * make it the final one one.
181          */
182         if (read_sched_clock == jiffy_sched_clock_read)
183                 setup_sched_clock(jiffy_sched_clock_read, 32, HZ);
184 
185         sched_clock_poll(sched_clock_timer.data);
186 }
187 
188 static int sched_clock_suspend(void)
189 {
190         sched_clock_poll(sched_clock_timer.data);
191         cd.suspended = true;
192         return 0;
193 }
194 
195 static void sched_clock_resume(void)
196 {
197         cd.epoch_cyc = read_sched_clock();
198         cd.epoch_cyc_copy = cd.epoch_cyc;
199         cd.suspended = false;
200 }
201 
202 static struct syscore_ops sched_clock_ops = {
203         .suspend = sched_clock_suspend,
204         .resume = sched_clock_resume,
205 };
206 
207 static int __init sched_clock_syscore_init(void)
208 {
209         register_syscore_ops(&sched_clock_ops);
210         return 0;
211 }
212 device_initcall(sched_clock_syscore_init);
213 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp