~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/xen/smp.c

Version: ~ [ linux-5.4-rc3 ] ~ [ linux-5.3.6 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.79 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.149 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.196 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.196 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.75 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Xen SMP support
  3  *
  4  * This file implements the Xen versions of smp_ops.  SMP under Xen is
  5  * very straightforward.  Bringing a CPU up is simply a matter of
  6  * loading its initial context and setting it running.
  7  *
  8  * IPIs are handled through the Xen event mechanism.
  9  *
 10  * Because virtual CPUs can be scheduled onto any real CPU, there's no
 11  * useful topology information for the kernel to make use of.  As a
 12  * result, all CPUs are treated as if they're single-core and
 13  * single-threaded.
 14  */
 15 #include <linux/sched.h>
 16 #include <linux/err.h>
 17 #include <linux/slab.h>
 18 #include <linux/smp.h>
 19 #include <linux/irq_work.h>
 20 #include <linux/tick.h>
 21 
 22 #include <asm/paravirt.h>
 23 #include <asm/desc.h>
 24 #include <asm/pgtable.h>
 25 #include <asm/cpu.h>
 26 
 27 #include <xen/interface/xen.h>
 28 #include <xen/interface/vcpu.h>
 29 #include <xen/interface/xenpmu.h>
 30 
 31 #include <asm/xen/interface.h>
 32 #include <asm/xen/hypercall.h>
 33 
 34 #include <xen/xen.h>
 35 #include <xen/page.h>
 36 #include <xen/events.h>
 37 
 38 #include <xen/hvc-console.h>
 39 #include "xen-ops.h"
 40 #include "mmu.h"
 41 #include "smp.h"
 42 #include "pmu.h"
 43 
 44 cpumask_var_t xen_cpu_initialized_map;
 45 
 46 struct xen_common_irq {
 47         int irq;
 48         char *name;
 49 };
 50 static DEFINE_PER_CPU(struct xen_common_irq, xen_resched_irq) = { .irq = -1 };
 51 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfunc_irq) = { .irq = -1 };
 52 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfuncsingle_irq) = { .irq = -1 };
 53 static DEFINE_PER_CPU(struct xen_common_irq, xen_irq_work) = { .irq = -1 };
 54 static DEFINE_PER_CPU(struct xen_common_irq, xen_debug_irq) = { .irq = -1 };
 55 static DEFINE_PER_CPU(struct xen_common_irq, xen_pmu_irq) = { .irq = -1 };
 56 
 57 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id);
 58 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
 59 static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id);
 60 
 61 /*
 62  * Reschedule call back.
 63  */
 64 static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
 65 {
 66         inc_irq_stat(irq_resched_count);
 67         scheduler_ipi();
 68 
 69         return IRQ_HANDLED;
 70 }
 71 
 72 static void cpu_bringup(void)
 73 {
 74         int cpu;
 75 
 76         cpu_init();
 77         touch_softlockup_watchdog();
 78         preempt_disable();
 79 
 80         /* PVH runs in ring 0 and allows us to do native syscalls. Yay! */
 81         if (!xen_feature(XENFEAT_supervisor_mode_kernel)) {
 82                 xen_enable_sysenter();
 83                 xen_enable_syscall();
 84         }
 85         cpu = smp_processor_id();
 86         smp_store_cpu_info(cpu);
 87         cpu_data(cpu).x86_max_cores = 1;
 88         set_cpu_sibling_map(cpu);
 89 
 90         xen_setup_cpu_clockevents();
 91 
 92         notify_cpu_starting(cpu);
 93 
 94         set_cpu_online(cpu, true);
 95 
 96         cpu_set_state_online(cpu);  /* Implies full memory barrier. */
 97 
 98         /* We can take interrupts now: we're officially "up". */
 99         local_irq_enable();
100 }
101 
102 /*
103  * Note: cpu parameter is only relevant for PVH. The reason for passing it
104  * is we can't do smp_processor_id until the percpu segments are loaded, for
105  * which we need the cpu number! So we pass it in rdi as first parameter.
106  */
107 asmlinkage __visible void cpu_bringup_and_idle(int cpu)
108 {
109 #ifdef CONFIG_XEN_PVH
110         if (xen_feature(XENFEAT_auto_translated_physmap) &&
111             xen_feature(XENFEAT_supervisor_mode_kernel))
112                 xen_pvh_secondary_vcpu_init(cpu);
113 #endif
114         cpu_bringup();
115         cpu_startup_entry(CPUHP_ONLINE);
116 }
117 
118 static void xen_smp_intr_free(unsigned int cpu)
119 {
120         if (per_cpu(xen_resched_irq, cpu).irq >= 0) {
121                 unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu).irq, NULL);
122                 per_cpu(xen_resched_irq, cpu).irq = -1;
123                 kfree(per_cpu(xen_resched_irq, cpu).name);
124                 per_cpu(xen_resched_irq, cpu).name = NULL;
125         }
126         if (per_cpu(xen_callfunc_irq, cpu).irq >= 0) {
127                 unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu).irq, NULL);
128                 per_cpu(xen_callfunc_irq, cpu).irq = -1;
129                 kfree(per_cpu(xen_callfunc_irq, cpu).name);
130                 per_cpu(xen_callfunc_irq, cpu).name = NULL;
131         }
132         if (per_cpu(xen_debug_irq, cpu).irq >= 0) {
133                 unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu).irq, NULL);
134                 per_cpu(xen_debug_irq, cpu).irq = -1;
135                 kfree(per_cpu(xen_debug_irq, cpu).name);
136                 per_cpu(xen_debug_irq, cpu).name = NULL;
137         }
138         if (per_cpu(xen_callfuncsingle_irq, cpu).irq >= 0) {
139                 unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu).irq,
140                                        NULL);
141                 per_cpu(xen_callfuncsingle_irq, cpu).irq = -1;
142                 kfree(per_cpu(xen_callfuncsingle_irq, cpu).name);
143                 per_cpu(xen_callfuncsingle_irq, cpu).name = NULL;
144         }
145         if (xen_hvm_domain())
146                 return;
147 
148         if (per_cpu(xen_irq_work, cpu).irq >= 0) {
149                 unbind_from_irqhandler(per_cpu(xen_irq_work, cpu).irq, NULL);
150                 per_cpu(xen_irq_work, cpu).irq = -1;
151                 kfree(per_cpu(xen_irq_work, cpu).name);
152                 per_cpu(xen_irq_work, cpu).name = NULL;
153         }
154 
155         if (per_cpu(xen_pmu_irq, cpu).irq >= 0) {
156                 unbind_from_irqhandler(per_cpu(xen_pmu_irq, cpu).irq, NULL);
157                 per_cpu(xen_pmu_irq, cpu).irq = -1;
158                 kfree(per_cpu(xen_pmu_irq, cpu).name);
159                 per_cpu(xen_pmu_irq, cpu).name = NULL;
160         }
161 };
162 static int xen_smp_intr_init(unsigned int cpu)
163 {
164         int rc;
165         char *resched_name, *callfunc_name, *debug_name, *pmu_name;
166 
167         resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu);
168         rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR,
169                                     cpu,
170                                     xen_reschedule_interrupt,
171                                     IRQF_PERCPU|IRQF_NOBALANCING,
172                                     resched_name,
173                                     NULL);
174         if (rc < 0)
175                 goto fail;
176         per_cpu(xen_resched_irq, cpu).irq = rc;
177         per_cpu(xen_resched_irq, cpu).name = resched_name;
178 
179         callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu);
180         rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR,
181                                     cpu,
182                                     xen_call_function_interrupt,
183                                     IRQF_PERCPU|IRQF_NOBALANCING,
184                                     callfunc_name,
185                                     NULL);
186         if (rc < 0)
187                 goto fail;
188         per_cpu(xen_callfunc_irq, cpu).irq = rc;
189         per_cpu(xen_callfunc_irq, cpu).name = callfunc_name;
190 
191         debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu);
192         rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu, xen_debug_interrupt,
193                                      IRQF_PERCPU | IRQF_NOBALANCING,
194                                      debug_name, NULL);
195         if (rc < 0)
196                 goto fail;
197         per_cpu(xen_debug_irq, cpu).irq = rc;
198         per_cpu(xen_debug_irq, cpu).name = debug_name;
199 
200         callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu);
201         rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR,
202                                     cpu,
203                                     xen_call_function_single_interrupt,
204                                     IRQF_PERCPU|IRQF_NOBALANCING,
205                                     callfunc_name,
206                                     NULL);
207         if (rc < 0)
208                 goto fail;
209         per_cpu(xen_callfuncsingle_irq, cpu).irq = rc;
210         per_cpu(xen_callfuncsingle_irq, cpu).name = callfunc_name;
211 
212         /*
213          * The IRQ worker on PVHVM goes through the native path and uses the
214          * IPI mechanism.
215          */
216         if (xen_hvm_domain())
217                 return 0;
218 
219         callfunc_name = kasprintf(GFP_KERNEL, "irqwork%d", cpu);
220         rc = bind_ipi_to_irqhandler(XEN_IRQ_WORK_VECTOR,
221                                     cpu,
222                                     xen_irq_work_interrupt,
223                                     IRQF_PERCPU|IRQF_NOBALANCING,
224                                     callfunc_name,
225                                     NULL);
226         if (rc < 0)
227                 goto fail;
228         per_cpu(xen_irq_work, cpu).irq = rc;
229         per_cpu(xen_irq_work, cpu).name = callfunc_name;
230 
231         if (is_xen_pmu(cpu)) {
232                 pmu_name = kasprintf(GFP_KERNEL, "pmu%d", cpu);
233                 rc = bind_virq_to_irqhandler(VIRQ_XENPMU, cpu,
234                                              xen_pmu_irq_handler,
235                                              IRQF_PERCPU|IRQF_NOBALANCING,
236                                              pmu_name, NULL);
237                 if (rc < 0)
238                         goto fail;
239                 per_cpu(xen_pmu_irq, cpu).irq = rc;
240                 per_cpu(xen_pmu_irq, cpu).name = pmu_name;
241         }
242 
243         return 0;
244 
245  fail:
246         xen_smp_intr_free(cpu);
247         return rc;
248 }
249 
250 static void __init xen_fill_possible_map(void)
251 {
252         int i, rc;
253 
254         if (xen_initial_domain())
255                 return;
256 
257         for (i = 0; i < nr_cpu_ids; i++) {
258                 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
259                 if (rc >= 0) {
260                         num_processors++;
261                         set_cpu_possible(i, true);
262                 }
263         }
264 }
265 
266 static void __init xen_filter_cpu_maps(void)
267 {
268         int i, rc;
269         unsigned int subtract = 0;
270 
271         if (!xen_initial_domain())
272                 return;
273 
274         num_processors = 0;
275         disabled_cpus = 0;
276         for (i = 0; i < nr_cpu_ids; i++) {
277                 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
278                 if (rc >= 0) {
279                         num_processors++;
280                         set_cpu_possible(i, true);
281                 } else {
282                         set_cpu_possible(i, false);
283                         set_cpu_present(i, false);
284                         subtract++;
285                 }
286         }
287 #ifdef CONFIG_HOTPLUG_CPU
288         /* This is akin to using 'nr_cpus' on the Linux command line.
289          * Which is OK as when we use 'dom0_max_vcpus=X' we can only
290          * have up to X, while nr_cpu_ids is greater than X. This
291          * normally is not a problem, except when CPU hotplugging
292          * is involved and then there might be more than X CPUs
293          * in the guest - which will not work as there is no
294          * hypercall to expand the max number of VCPUs an already
295          * running guest has. So cap it up to X. */
296         if (subtract)
297                 nr_cpu_ids = nr_cpu_ids - subtract;
298 #endif
299 
300 }
301 
302 static void __init xen_smp_prepare_boot_cpu(void)
303 {
304         BUG_ON(smp_processor_id() != 0);
305         native_smp_prepare_boot_cpu();
306 
307         if (xen_pv_domain()) {
308                 if (!xen_feature(XENFEAT_writable_page_tables))
309                         /* We've switched to the "real" per-cpu gdt, so make
310                          * sure the old memory can be recycled. */
311                         make_lowmem_page_readwrite(xen_initial_gdt);
312 
313 #ifdef CONFIG_X86_32
314                 /*
315                  * Xen starts us with XEN_FLAT_RING1_DS, but linux code
316                  * expects __USER_DS
317                  */
318                 loadsegment(ds, __USER_DS);
319                 loadsegment(es, __USER_DS);
320 #endif
321 
322                 xen_filter_cpu_maps();
323                 xen_setup_vcpu_info_placement();
324         }
325         /*
326          * The alternative logic (which patches the unlock/lock) runs before
327          * the smp bootup up code is activated. Hence we need to set this up
328          * the core kernel is being patched. Otherwise we will have only
329          * modules patched but not core code.
330          */
331         xen_init_spinlocks();
332 }
333 
334 static void __init xen_smp_prepare_cpus(unsigned int max_cpus)
335 {
336         unsigned cpu;
337         unsigned int i;
338 
339         if (skip_ioapic_setup) {
340                 char *m = (max_cpus == 0) ?
341                         "The nosmp parameter is incompatible with Xen; " \
342                         "use Xen dom0_max_vcpus=1 parameter" :
343                         "The noapic parameter is incompatible with Xen";
344 
345                 xen_raw_printk(m);
346                 panic(m);
347         }
348         xen_init_lock_cpu(0);
349 
350         smp_store_boot_cpu_info();
351         cpu_data(0).x86_max_cores = 1;
352 
353         for_each_possible_cpu(i) {
354                 zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
355                 zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
356                 zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
357         }
358         set_cpu_sibling_map(0);
359 
360         xen_pmu_init(0);
361 
362         if (xen_smp_intr_init(0))
363                 BUG();
364 
365         if (!alloc_cpumask_var(&xen_cpu_initialized_map, GFP_KERNEL))
366                 panic("could not allocate xen_cpu_initialized_map\n");
367 
368         cpumask_copy(xen_cpu_initialized_map, cpumask_of(0));
369 
370         /* Restrict the possible_map according to max_cpus. */
371         while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) {
372                 for (cpu = nr_cpu_ids - 1; !cpu_possible(cpu); cpu--)
373                         continue;
374                 set_cpu_possible(cpu, false);
375         }
376 
377         for_each_possible_cpu(cpu)
378                 set_cpu_present(cpu, true);
379 }
380 
381 static int
382 cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
383 {
384         struct vcpu_guest_context *ctxt;
385         struct desc_struct *gdt;
386         unsigned long gdt_mfn;
387 
388         /* used to tell cpu_init() that it can proceed with initialization */
389         cpumask_set_cpu(cpu, cpu_callout_mask);
390         if (cpumask_test_and_set_cpu(cpu, xen_cpu_initialized_map))
391                 return 0;
392 
393         ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
394         if (ctxt == NULL)
395                 return -ENOMEM;
396 
397         gdt = get_cpu_gdt_table(cpu);
398 
399 #ifdef CONFIG_X86_32
400         /* Note: PVH is not yet supported on x86_32. */
401         ctxt->user_regs.fs = __KERNEL_PERCPU;
402         ctxt->user_regs.gs = __KERNEL_STACK_CANARY;
403 #endif
404         memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));
405 
406         if (!xen_feature(XENFEAT_auto_translated_physmap)) {
407                 ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle;
408                 ctxt->flags = VGCF_IN_KERNEL;
409                 ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */
410                 ctxt->user_regs.ds = __USER_DS;
411                 ctxt->user_regs.es = __USER_DS;
412                 ctxt->user_regs.ss = __KERNEL_DS;
413 
414                 xen_copy_trap_info(ctxt->trap_ctxt);
415 
416                 ctxt->ldt_ents = 0;
417 
418                 BUG_ON((unsigned long)gdt & ~PAGE_MASK);
419 
420                 gdt_mfn = arbitrary_virt_to_mfn(gdt);
421                 make_lowmem_page_readonly(gdt);
422                 make_lowmem_page_readonly(mfn_to_virt(gdt_mfn));
423 
424                 ctxt->gdt_frames[0] = gdt_mfn;
425                 ctxt->gdt_ents      = GDT_ENTRIES;
426 
427                 ctxt->kernel_ss = __KERNEL_DS;
428                 ctxt->kernel_sp = idle->thread.sp0;
429 
430 #ifdef CONFIG_X86_32
431                 ctxt->event_callback_cs     = __KERNEL_CS;
432                 ctxt->failsafe_callback_cs  = __KERNEL_CS;
433 #else
434                 ctxt->gs_base_kernel = per_cpu_offset(cpu);
435 #endif
436                 ctxt->event_callback_eip    =
437                                         (unsigned long)xen_hypervisor_callback;
438                 ctxt->failsafe_callback_eip =
439                                         (unsigned long)xen_failsafe_callback;
440                 ctxt->user_regs.cs = __KERNEL_CS;
441                 per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir);
442         }
443 #ifdef CONFIG_XEN_PVH
444         else {
445                 /*
446                  * The vcpu comes on kernel page tables which have the NX pte
447                  * bit set. This means before DS/SS is touched, NX in
448                  * EFER must be set. Hence the following assembly glue code.
449                  */
450                 ctxt->user_regs.eip = (unsigned long)xen_pvh_early_cpu_init;
451                 ctxt->user_regs.rdi = cpu;
452                 ctxt->user_regs.rsi = true;  /* entry == true */
453         }
454 #endif
455         ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs);
456         ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_gfn(swapper_pg_dir));
457         if (HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, ctxt))
458                 BUG();
459 
460         kfree(ctxt);
461         return 0;
462 }
463 
464 static int xen_cpu_up(unsigned int cpu, struct task_struct *idle)
465 {
466         int rc;
467 
468         common_cpu_up(cpu, idle);
469 
470         xen_setup_runstate_info(cpu);
471         xen_setup_timer(cpu);
472         xen_init_lock_cpu(cpu);
473 
474         /*
475          * PV VCPUs are always successfully taken down (see 'while' loop
476          * in xen_cpu_die()), so -EBUSY is an error.
477          */
478         rc = cpu_check_up_prepare(cpu);
479         if (rc)
480                 return rc;
481 
482         /* make sure interrupts start blocked */
483         per_cpu(xen_vcpu, cpu)->evtchn_upcall_mask = 1;
484 
485         rc = cpu_initialize_context(cpu, idle);
486         if (rc)
487                 return rc;
488 
489         xen_pmu_init(cpu);
490 
491         rc = xen_smp_intr_init(cpu);
492         if (rc)
493                 return rc;
494 
495         rc = HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL);
496         BUG_ON(rc);
497 
498         while (cpu_report_state(cpu) != CPU_ONLINE)
499                 HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
500 
501         return 0;
502 }
503 
504 static void xen_smp_cpus_done(unsigned int max_cpus)
505 {
506 }
507 
508 #ifdef CONFIG_HOTPLUG_CPU
509 static int xen_cpu_disable(void)
510 {
511         unsigned int cpu = smp_processor_id();
512         if (cpu == 0)
513                 return -EBUSY;
514 
515         cpu_disable_common();
516 
517         load_cr3(swapper_pg_dir);
518         return 0;
519 }
520 
521 static void xen_cpu_die(unsigned int cpu)
522 {
523         while (xen_pv_domain() && HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) {
524                 __set_current_state(TASK_UNINTERRUPTIBLE);
525                 schedule_timeout(HZ/10);
526         }
527 
528         if (common_cpu_die(cpu) == 0) {
529                 xen_smp_intr_free(cpu);
530                 xen_uninit_lock_cpu(cpu);
531                 xen_teardown_timer(cpu);
532                 xen_pmu_finish(cpu);
533         }
534 }
535 
536 static void xen_play_dead(void) /* used only with HOTPLUG_CPU */
537 {
538         play_dead_common();
539         HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL);
540         cpu_bringup();
541         /*
542          * commit 4b0c0f294 (tick: Cleanup NOHZ per cpu data on cpu down)
543          * clears certain data that the cpu_idle loop (which called us
544          * and that we return from) expects. The only way to get that
545          * data back is to call:
546          */
547         tick_nohz_idle_enter();
548 }
549 
550 #else /* !CONFIG_HOTPLUG_CPU */
551 static int xen_cpu_disable(void)
552 {
553         return -ENOSYS;
554 }
555 
556 static void xen_cpu_die(unsigned int cpu)
557 {
558         BUG();
559 }
560 
561 static void xen_play_dead(void)
562 {
563         BUG();
564 }
565 
566 #endif
567 static void stop_self(void *v)
568 {
569         int cpu = smp_processor_id();
570 
571         /* make sure we're not pinning something down */
572         load_cr3(swapper_pg_dir);
573         /* should set up a minimal gdt */
574 
575         set_cpu_online(cpu, false);
576 
577         HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL);
578         BUG();
579 }
580 
581 static void xen_stop_other_cpus(int wait)
582 {
583         smp_call_function(stop_self, NULL, wait);
584 }
585 
586 static void xen_smp_send_reschedule(int cpu)
587 {
588         xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR);
589 }
590 
591 static void __xen_send_IPI_mask(const struct cpumask *mask,
592                               int vector)
593 {
594         unsigned cpu;
595 
596         for_each_cpu_and(cpu, mask, cpu_online_mask)
597                 xen_send_IPI_one(cpu, vector);
598 }
599 
600 static void xen_smp_send_call_function_ipi(const struct cpumask *mask)
601 {
602         int cpu;
603 
604         __xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);
605 
606         /* Make sure other vcpus get a chance to run if they need to. */
607         for_each_cpu(cpu, mask) {
608                 if (xen_vcpu_stolen(cpu)) {
609                         HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
610                         break;
611                 }
612         }
613 }
614 
615 static void xen_smp_send_call_function_single_ipi(int cpu)
616 {
617         __xen_send_IPI_mask(cpumask_of(cpu),
618                           XEN_CALL_FUNCTION_SINGLE_VECTOR);
619 }
620 
621 static inline int xen_map_vector(int vector)
622 {
623         int xen_vector;
624 
625         switch (vector) {
626         case RESCHEDULE_VECTOR:
627                 xen_vector = XEN_RESCHEDULE_VECTOR;
628                 break;
629         case CALL_FUNCTION_VECTOR:
630                 xen_vector = XEN_CALL_FUNCTION_VECTOR;
631                 break;
632         case CALL_FUNCTION_SINGLE_VECTOR:
633                 xen_vector = XEN_CALL_FUNCTION_SINGLE_VECTOR;
634                 break;
635         case IRQ_WORK_VECTOR:
636                 xen_vector = XEN_IRQ_WORK_VECTOR;
637                 break;
638 #ifdef CONFIG_X86_64
639         case NMI_VECTOR:
640         case APIC_DM_NMI: /* Some use that instead of NMI_VECTOR */
641                 xen_vector = XEN_NMI_VECTOR;
642                 break;
643 #endif
644         default:
645                 xen_vector = -1;
646                 printk(KERN_ERR "xen: vector 0x%x is not implemented\n",
647                         vector);
648         }
649 
650         return xen_vector;
651 }
652 
653 void xen_send_IPI_mask(const struct cpumask *mask,
654                               int vector)
655 {
656         int xen_vector = xen_map_vector(vector);
657 
658         if (xen_vector >= 0)
659                 __xen_send_IPI_mask(mask, xen_vector);
660 }
661 
662 void xen_send_IPI_all(int vector)
663 {
664         int xen_vector = xen_map_vector(vector);
665 
666         if (xen_vector >= 0)
667                 __xen_send_IPI_mask(cpu_online_mask, xen_vector);
668 }
669 
670 void xen_send_IPI_self(int vector)
671 {
672         int xen_vector = xen_map_vector(vector);
673 
674         if (xen_vector >= 0)
675                 xen_send_IPI_one(smp_processor_id(), xen_vector);
676 }
677 
678 void xen_send_IPI_mask_allbutself(const struct cpumask *mask,
679                                 int vector)
680 {
681         unsigned cpu;
682         unsigned int this_cpu = smp_processor_id();
683         int xen_vector = xen_map_vector(vector);
684 
685         if (!(num_online_cpus() > 1) || (xen_vector < 0))
686                 return;
687 
688         for_each_cpu_and(cpu, mask, cpu_online_mask) {
689                 if (this_cpu == cpu)
690                         continue;
691 
692                 xen_send_IPI_one(cpu, xen_vector);
693         }
694 }
695 
696 void xen_send_IPI_allbutself(int vector)
697 {
698         xen_send_IPI_mask_allbutself(cpu_online_mask, vector);
699 }
700 
701 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
702 {
703         irq_enter();
704         generic_smp_call_function_interrupt();
705         inc_irq_stat(irq_call_count);
706         irq_exit();
707 
708         return IRQ_HANDLED;
709 }
710 
711 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
712 {
713         irq_enter();
714         generic_smp_call_function_single_interrupt();
715         inc_irq_stat(irq_call_count);
716         irq_exit();
717 
718         return IRQ_HANDLED;
719 }
720 
721 static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id)
722 {
723         irq_enter();
724         irq_work_run();
725         inc_irq_stat(apic_irq_work_irqs);
726         irq_exit();
727 
728         return IRQ_HANDLED;
729 }
730 
731 static const struct smp_ops xen_smp_ops __initconst = {
732         .smp_prepare_boot_cpu = xen_smp_prepare_boot_cpu,
733         .smp_prepare_cpus = xen_smp_prepare_cpus,
734         .smp_cpus_done = xen_smp_cpus_done,
735 
736         .cpu_up = xen_cpu_up,
737         .cpu_die = xen_cpu_die,
738         .cpu_disable = xen_cpu_disable,
739         .play_dead = xen_play_dead,
740 
741         .stop_other_cpus = xen_stop_other_cpus,
742         .smp_send_reschedule = xen_smp_send_reschedule,
743 
744         .send_call_func_ipi = xen_smp_send_call_function_ipi,
745         .send_call_func_single_ipi = xen_smp_send_call_function_single_ipi,
746 };
747 
748 void __init xen_smp_init(void)
749 {
750         smp_ops = xen_smp_ops;
751         xen_fill_possible_map();
752 }
753 
754 static void __init xen_hvm_smp_prepare_cpus(unsigned int max_cpus)
755 {
756         native_smp_prepare_cpus(max_cpus);
757         WARN_ON(xen_smp_intr_init(0));
758 
759         xen_init_lock_cpu(0);
760 }
761 
762 static int xen_hvm_cpu_up(unsigned int cpu, struct task_struct *tidle)
763 {
764         int rc;
765 
766         /*
767          * This can happen if CPU was offlined earlier and
768          * offlining timed out in common_cpu_die().
769          */
770         if (cpu_report_state(cpu) == CPU_DEAD_FROZEN) {
771                 xen_smp_intr_free(cpu);
772                 xen_uninit_lock_cpu(cpu);
773         }
774 
775         /*
776          * xen_smp_intr_init() needs to run before native_cpu_up()
777          * so that IPI vectors are set up on the booting CPU before
778          * it is marked online in native_cpu_up().
779         */
780         rc = xen_smp_intr_init(cpu);
781         WARN_ON(rc);
782         if (!rc)
783                 rc =  native_cpu_up(cpu, tidle);
784 
785         /*
786          * We must initialize the slowpath CPU kicker _after_ the native
787          * path has executed. If we initialized it before none of the
788          * unlocker IPI kicks would reach the booting CPU as the booting
789          * CPU had not set itself 'online' in cpu_online_mask. That mask
790          * is checked when IPIs are sent (on HVM at least).
791          */
792         xen_init_lock_cpu(cpu);
793         return rc;
794 }
795 
796 void __init xen_hvm_smp_init(void)
797 {
798         if (!xen_have_vector_callback)
799                 return;
800         smp_ops.smp_prepare_cpus = xen_hvm_smp_prepare_cpus;
801         smp_ops.smp_send_reschedule = xen_smp_send_reschedule;
802         smp_ops.cpu_up = xen_hvm_cpu_up;
803         smp_ops.cpu_die = xen_cpu_die;
804         smp_ops.send_call_func_ipi = xen_smp_send_call_function_ipi;
805         smp_ops.send_call_func_single_ipi = xen_smp_send_call_function_single_ipi;
806         smp_ops.smp_prepare_boot_cpu = xen_smp_prepare_boot_cpu;
807 }
808 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp