~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/xen/time.c

Version: ~ [ linux-5.9-rc6 ] ~ [ linux-5.8.10 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.66 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.146 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.198 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.236 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.236 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Xen time implementation.
  3  *
  4  * This is implemented in terms of a clocksource driver which uses
  5  * the hypervisor clock as a nanosecond timebase, and a clockevent
  6  * driver which uses the hypervisor's timer mechanism.
  7  *
  8  * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
  9  */
 10 #include <linux/kernel.h>
 11 #include <linux/interrupt.h>
 12 #include <linux/clocksource.h>
 13 #include <linux/clockchips.h>
 14 #include <linux/kernel_stat.h>
 15 #include <linux/math64.h>
 16 #include <linux/gfp.h>
 17 #include <linux/slab.h>
 18 #include <linux/pvclock_gtod.h>
 19 
 20 #include <asm/pvclock.h>
 21 #include <asm/xen/hypervisor.h>
 22 #include <asm/xen/hypercall.h>
 23 
 24 #include <xen/events.h>
 25 #include <xen/features.h>
 26 #include <xen/interface/xen.h>
 27 #include <xen/interface/vcpu.h>
 28 
 29 #include "xen-ops.h"
 30 
 31 /* Xen may fire a timer up to this many ns early */
 32 #define TIMER_SLOP      100000
 33 #define NS_PER_TICK     (1000000000LL / HZ)
 34 
 35 /* runstate info updated by Xen */
 36 static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate);
 37 
 38 /* snapshots of runstate info */
 39 static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate_snapshot);
 40 
 41 /* unused ns of stolen time */
 42 static DEFINE_PER_CPU(u64, xen_residual_stolen);
 43 
 44 /* return an consistent snapshot of 64-bit time/counter value */
 45 static u64 get64(const u64 *p)
 46 {
 47         u64 ret;
 48 
 49         if (BITS_PER_LONG < 64) {
 50                 u32 *p32 = (u32 *)p;
 51                 u32 h, l;
 52 
 53                 /*
 54                  * Read high then low, and then make sure high is
 55                  * still the same; this will only loop if low wraps
 56                  * and carries into high.
 57                  * XXX some clean way to make this endian-proof?
 58                  */
 59                 do {
 60                         h = p32[1];
 61                         barrier();
 62                         l = p32[0];
 63                         barrier();
 64                 } while (p32[1] != h);
 65 
 66                 ret = (((u64)h) << 32) | l;
 67         } else
 68                 ret = *p;
 69 
 70         return ret;
 71 }
 72 
 73 /*
 74  * Runstate accounting
 75  */
 76 static void get_runstate_snapshot(struct vcpu_runstate_info *res)
 77 {
 78         u64 state_time;
 79         struct vcpu_runstate_info *state;
 80 
 81         BUG_ON(preemptible());
 82 
 83         state = &__get_cpu_var(xen_runstate);
 84 
 85         /*
 86          * The runstate info is always updated by the hypervisor on
 87          * the current CPU, so there's no need to use anything
 88          * stronger than a compiler barrier when fetching it.
 89          */
 90         do {
 91                 state_time = get64(&state->state_entry_time);
 92                 barrier();
 93                 *res = *state;
 94                 barrier();
 95         } while (get64(&state->state_entry_time) != state_time);
 96 }
 97 
 98 /* return true when a vcpu could run but has no real cpu to run on */
 99 bool xen_vcpu_stolen(int vcpu)
100 {
101         return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable;
102 }
103 
104 void xen_setup_runstate_info(int cpu)
105 {
106         struct vcpu_register_runstate_memory_area area;
107 
108         area.addr.v = &per_cpu(xen_runstate, cpu);
109 
110         if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area,
111                                cpu, &area))
112                 BUG();
113 }
114 
115 static void do_stolen_accounting(void)
116 {
117         struct vcpu_runstate_info state;
118         struct vcpu_runstate_info *snap;
119         s64 runnable, offline, stolen;
120         cputime_t ticks;
121 
122         get_runstate_snapshot(&state);
123 
124         WARN_ON(state.state != RUNSTATE_running);
125 
126         snap = &__get_cpu_var(xen_runstate_snapshot);
127 
128         /* work out how much time the VCPU has not been runn*ing*  */
129         runnable = state.time[RUNSTATE_runnable] - snap->time[RUNSTATE_runnable];
130         offline = state.time[RUNSTATE_offline] - snap->time[RUNSTATE_offline];
131 
132         *snap = state;
133 
134         /* Add the appropriate number of ticks of stolen time,
135            including any left-overs from last time. */
136         stolen = runnable + offline + __this_cpu_read(xen_residual_stolen);
137 
138         if (stolen < 0)
139                 stolen = 0;
140 
141         ticks = iter_div_u64_rem(stolen, NS_PER_TICK, &stolen);
142         __this_cpu_write(xen_residual_stolen, stolen);
143         account_steal_ticks(ticks);
144 }
145 
146 /* Get the TSC speed from Xen */
147 static unsigned long xen_tsc_khz(void)
148 {
149         struct pvclock_vcpu_time_info *info =
150                 &HYPERVISOR_shared_info->vcpu_info[0].time;
151 
152         return pvclock_tsc_khz(info);
153 }
154 
155 cycle_t xen_clocksource_read(void)
156 {
157         struct pvclock_vcpu_time_info *src;
158         cycle_t ret;
159 
160         preempt_disable_notrace();
161         src = &__get_cpu_var(xen_vcpu)->time;
162         ret = pvclock_clocksource_read(src);
163         preempt_enable_notrace();
164         return ret;
165 }
166 
167 static cycle_t xen_clocksource_get_cycles(struct clocksource *cs)
168 {
169         return xen_clocksource_read();
170 }
171 
172 static void xen_read_wallclock(struct timespec *ts)
173 {
174         struct shared_info *s = HYPERVISOR_shared_info;
175         struct pvclock_wall_clock *wall_clock = &(s->wc);
176         struct pvclock_vcpu_time_info *vcpu_time;
177 
178         vcpu_time = &get_cpu_var(xen_vcpu)->time;
179         pvclock_read_wallclock(wall_clock, vcpu_time, ts);
180         put_cpu_var(xen_vcpu);
181 }
182 
183 static void xen_get_wallclock(struct timespec *now)
184 {
185         xen_read_wallclock(now);
186 }
187 
188 static int xen_set_wallclock(const struct timespec *now)
189 {
190         return -1;
191 }
192 
193 static int xen_pvclock_gtod_notify(struct notifier_block *nb,
194                                    unsigned long was_set, void *priv)
195 {
196         /* Protected by the calling core code serialization */
197         static struct timespec next_sync;
198 
199         struct xen_platform_op op;
200         struct timespec now;
201 
202         now = __current_kernel_time();
203 
204         /*
205          * We only take the expensive HV call when the clock was set
206          * or when the 11 minutes RTC synchronization time elapsed.
207          */
208         if (!was_set && timespec_compare(&now, &next_sync) < 0)
209                 return NOTIFY_OK;
210 
211         op.cmd = XENPF_settime;
212         op.u.settime.secs = now.tv_sec;
213         op.u.settime.nsecs = now.tv_nsec;
214         op.u.settime.system_time = xen_clocksource_read();
215 
216         (void)HYPERVISOR_dom0_op(&op);
217 
218         /*
219          * Move the next drift compensation time 11 minutes
220          * ahead. That's emulating the sync_cmos_clock() update for
221          * the hardware RTC.
222          */
223         next_sync = now;
224         next_sync.tv_sec += 11 * 60;
225 
226         return NOTIFY_OK;
227 }
228 
229 static struct notifier_block xen_pvclock_gtod_notifier = {
230         .notifier_call = xen_pvclock_gtod_notify,
231 };
232 
233 static struct clocksource xen_clocksource __read_mostly = {
234         .name = "xen",
235         .rating = 400,
236         .read = xen_clocksource_get_cycles,
237         .mask = ~0,
238         .flags = CLOCK_SOURCE_IS_CONTINUOUS,
239 };
240 
241 /*
242    Xen clockevent implementation
243 
244    Xen has two clockevent implementations:
245 
246    The old timer_op one works with all released versions of Xen prior
247    to version 3.0.4.  This version of the hypervisor provides a
248    single-shot timer with nanosecond resolution.  However, sharing the
249    same event channel is a 100Hz tick which is delivered while the
250    vcpu is running.  We don't care about or use this tick, but it will
251    cause the core time code to think the timer fired too soon, and
252    will end up resetting it each time.  It could be filtered, but
253    doing so has complications when the ktime clocksource is not yet
254    the xen clocksource (ie, at boot time).
255 
256    The new vcpu_op-based timer interface allows the tick timer period
257    to be changed or turned off.  The tick timer is not useful as a
258    periodic timer because events are only delivered to running vcpus.
259    The one-shot timer can report when a timeout is in the past, so
260    set_next_event is capable of returning -ETIME when appropriate.
261    This interface is used when available.
262 */
263 
264 
265 /*
266   Get a hypervisor absolute time.  In theory we could maintain an
267   offset between the kernel's time and the hypervisor's time, and
268   apply that to a kernel's absolute timeout.  Unfortunately the
269   hypervisor and kernel times can drift even if the kernel is using
270   the Xen clocksource, because ntp can warp the kernel's clocksource.
271 */
272 static s64 get_abs_timeout(unsigned long delta)
273 {
274         return xen_clocksource_read() + delta;
275 }
276 
277 static void xen_timerop_set_mode(enum clock_event_mode mode,
278                                  struct clock_event_device *evt)
279 {
280         switch (mode) {
281         case CLOCK_EVT_MODE_PERIODIC:
282                 /* unsupported */
283                 WARN_ON(1);
284                 break;
285 
286         case CLOCK_EVT_MODE_ONESHOT:
287         case CLOCK_EVT_MODE_RESUME:
288                 break;
289 
290         case CLOCK_EVT_MODE_UNUSED:
291         case CLOCK_EVT_MODE_SHUTDOWN:
292                 HYPERVISOR_set_timer_op(0);  /* cancel timeout */
293                 break;
294         }
295 }
296 
297 static int xen_timerop_set_next_event(unsigned long delta,
298                                       struct clock_event_device *evt)
299 {
300         WARN_ON(evt->mode != CLOCK_EVT_MODE_ONESHOT);
301 
302         if (HYPERVISOR_set_timer_op(get_abs_timeout(delta)) < 0)
303                 BUG();
304 
305         /* We may have missed the deadline, but there's no real way of
306            knowing for sure.  If the event was in the past, then we'll
307            get an immediate interrupt. */
308 
309         return 0;
310 }
311 
312 static const struct clock_event_device xen_timerop_clockevent = {
313         .name = "xen",
314         .features = CLOCK_EVT_FEAT_ONESHOT,
315 
316         .max_delta_ns = 0xffffffff,
317         .min_delta_ns = TIMER_SLOP,
318 
319         .mult = 1,
320         .shift = 0,
321         .rating = 500,
322 
323         .set_mode = xen_timerop_set_mode,
324         .set_next_event = xen_timerop_set_next_event,
325 };
326 
327 
328 
329 static void xen_vcpuop_set_mode(enum clock_event_mode mode,
330                                 struct clock_event_device *evt)
331 {
332         int cpu = smp_processor_id();
333 
334         switch (mode) {
335         case CLOCK_EVT_MODE_PERIODIC:
336                 WARN_ON(1);     /* unsupported */
337                 break;
338 
339         case CLOCK_EVT_MODE_ONESHOT:
340                 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
341                         BUG();
342                 break;
343 
344         case CLOCK_EVT_MODE_UNUSED:
345         case CLOCK_EVT_MODE_SHUTDOWN:
346                 if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, cpu, NULL) ||
347                     HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
348                         BUG();
349                 break;
350         case CLOCK_EVT_MODE_RESUME:
351                 break;
352         }
353 }
354 
355 static int xen_vcpuop_set_next_event(unsigned long delta,
356                                      struct clock_event_device *evt)
357 {
358         int cpu = smp_processor_id();
359         struct vcpu_set_singleshot_timer single;
360         int ret;
361 
362         WARN_ON(evt->mode != CLOCK_EVT_MODE_ONESHOT);
363 
364         single.timeout_abs_ns = get_abs_timeout(delta);
365         single.flags = VCPU_SSHOTTMR_future;
366 
367         ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, cpu, &single);
368 
369         BUG_ON(ret != 0 && ret != -ETIME);
370 
371         return ret;
372 }
373 
374 static const struct clock_event_device xen_vcpuop_clockevent = {
375         .name = "xen",
376         .features = CLOCK_EVT_FEAT_ONESHOT,
377 
378         .max_delta_ns = 0xffffffff,
379         .min_delta_ns = TIMER_SLOP,
380 
381         .mult = 1,
382         .shift = 0,
383         .rating = 500,
384 
385         .set_mode = xen_vcpuop_set_mode,
386         .set_next_event = xen_vcpuop_set_next_event,
387 };
388 
389 static const struct clock_event_device *xen_clockevent =
390         &xen_timerop_clockevent;
391 
392 struct xen_clock_event_device {
393         struct clock_event_device evt;
394         char *name;
395 };
396 static DEFINE_PER_CPU(struct xen_clock_event_device, xen_clock_events) = { .evt.irq = -1 };
397 
398 static irqreturn_t xen_timer_interrupt(int irq, void *dev_id)
399 {
400         struct clock_event_device *evt = &__get_cpu_var(xen_clock_events).evt;
401         irqreturn_t ret;
402 
403         ret = IRQ_NONE;
404         if (evt->event_handler) {
405                 evt->event_handler(evt);
406                 ret = IRQ_HANDLED;
407         }
408 
409         do_stolen_accounting();
410 
411         return ret;
412 }
413 
414 void xen_teardown_timer(int cpu)
415 {
416         struct clock_event_device *evt;
417         BUG_ON(cpu == 0);
418         evt = &per_cpu(xen_clock_events, cpu).evt;
419 
420         if (evt->irq >= 0) {
421                 unbind_from_irqhandler(evt->irq, NULL);
422                 evt->irq = -1;
423                 kfree(per_cpu(xen_clock_events, cpu).name);
424                 per_cpu(xen_clock_events, cpu).name = NULL;
425         }
426 }
427 
428 void xen_setup_timer(int cpu)
429 {
430         char *name;
431         struct clock_event_device *evt;
432         int irq;
433 
434         evt = &per_cpu(xen_clock_events, cpu).evt;
435         WARN(evt->irq >= 0, "IRQ%d for CPU%d is already allocated\n", evt->irq, cpu);
436         if (evt->irq >= 0)
437                 xen_teardown_timer(cpu);
438 
439         printk(KERN_INFO "installing Xen timer for CPU %d\n", cpu);
440 
441         name = kasprintf(GFP_KERNEL, "timer%d", cpu);
442         if (!name)
443                 name = "<timer kasprintf failed>";
444 
445         irq = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, xen_timer_interrupt,
446                                       IRQF_DISABLED|IRQF_PERCPU|
447                                       IRQF_NOBALANCING|IRQF_TIMER|
448                                       IRQF_FORCE_RESUME,
449                                       name, NULL);
450 
451         memcpy(evt, xen_clockevent, sizeof(*evt));
452 
453         evt->cpumask = cpumask_of(cpu);
454         evt->irq = irq;
455         per_cpu(xen_clock_events, cpu).name = name;
456 }
457 
458 
459 void xen_setup_cpu_clockevents(void)
460 {
461         BUG_ON(preemptible());
462 
463         clockevents_register_device(&__get_cpu_var(xen_clock_events).evt);
464 }
465 
466 void xen_timer_resume(void)
467 {
468         int cpu;
469 
470         pvclock_resume();
471 
472         if (xen_clockevent != &xen_vcpuop_clockevent)
473                 return;
474 
475         for_each_online_cpu(cpu) {
476                 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
477                         BUG();
478         }
479 }
480 
481 static const struct pv_time_ops xen_time_ops __initconst = {
482         .sched_clock = xen_clocksource_read,
483 };
484 
485 static void __init xen_time_init(void)
486 {
487         int cpu = smp_processor_id();
488         struct timespec tp;
489 
490         clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC);
491 
492         if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL) == 0) {
493                 /* Successfully turned off 100Hz tick, so we have the
494                    vcpuop-based timer interface */
495                 printk(KERN_DEBUG "Xen: using vcpuop timer interface\n");
496                 xen_clockevent = &xen_vcpuop_clockevent;
497         }
498 
499         /* Set initial system time with full resolution */
500         xen_read_wallclock(&tp);
501         do_settimeofday(&tp);
502 
503         setup_force_cpu_cap(X86_FEATURE_TSC);
504 
505         xen_setup_runstate_info(cpu);
506         xen_setup_timer(cpu);
507         xen_setup_cpu_clockevents();
508 
509         if (xen_initial_domain())
510                 pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier);
511 }
512 
513 void __init xen_init_time_ops(void)
514 {
515         pv_time_ops = xen_time_ops;
516 
517         x86_init.timers.timer_init = xen_time_init;
518         x86_init.timers.setup_percpu_clockev = x86_init_noop;
519         x86_cpuinit.setup_percpu_clockev = x86_init_noop;
520 
521         x86_platform.calibrate_tsc = xen_tsc_khz;
522         x86_platform.get_wallclock = xen_get_wallclock;
523         /* Dom0 uses the native method to set the hardware RTC. */
524         if (!xen_initial_domain())
525                 x86_platform.set_wallclock = xen_set_wallclock;
526 }
527 
528 #ifdef CONFIG_XEN_PVHVM
529 static void xen_hvm_setup_cpu_clockevents(void)
530 {
531         int cpu = smp_processor_id();
532         xen_setup_runstate_info(cpu);
533         /*
534          * xen_setup_timer(cpu) - snprintf is bad in atomic context. Hence
535          * doing it xen_hvm_cpu_notify (which gets called by smp_init during
536          * early bootup and also during CPU hotplug events).
537          */
538         xen_setup_cpu_clockevents();
539 }
540 
541 void __init xen_hvm_init_time_ops(void)
542 {
543         /* vector callback is needed otherwise we cannot receive interrupts
544          * on cpu > 0 and at this point we don't know how many cpus are
545          * available */
546         if (!xen_have_vector_callback)
547                 return;
548         if (!xen_feature(XENFEAT_hvm_safe_pvclock)) {
549                 printk(KERN_INFO "Xen doesn't support pvclock on HVM,"
550                                 "disable pv timer\n");
551                 return;
552         }
553 
554         pv_time_ops = xen_time_ops;
555         x86_init.timers.setup_percpu_clockev = xen_time_init;
556         x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents;
557 
558         x86_platform.calibrate_tsc = xen_tsc_khz;
559         x86_platform.get_wallclock = xen_get_wallclock;
560         x86_platform.set_wallclock = xen_set_wallclock;
561 }
562 #endif
563 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp