~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/xen/smp_pv.c

Version: ~ [ linux-5.8 ] ~ [ linux-5.7.12 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.55 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.136 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.191 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.232 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.232 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  * Xen SMP support
  4  *
  5  * This file implements the Xen versions of smp_ops.  SMP under Xen is
  6  * very straightforward.  Bringing a CPU up is simply a matter of
  7  * loading its initial context and setting it running.
  8  *
  9  * IPIs are handled through the Xen event mechanism.
 10  *
 11  * Because virtual CPUs can be scheduled onto any real CPU, there's no
 12  * useful topology information for the kernel to make use of.  As a
 13  * result, all CPUs are treated as if they're single-core and
 14  * single-threaded.
 15  */
 16 #include <linux/sched.h>
 17 #include <linux/sched/task_stack.h>
 18 #include <linux/err.h>
 19 #include <linux/slab.h>
 20 #include <linux/smp.h>
 21 #include <linux/irq_work.h>
 22 #include <linux/tick.h>
 23 #include <linux/nmi.h>
 24 #include <linux/cpuhotplug.h>
 25 #include <linux/stackprotector.h>
 26 
 27 #include <asm/paravirt.h>
 28 #include <asm/desc.h>
 29 #include <asm/pgtable.h>
 30 #include <asm/cpu.h>
 31 
 32 #include <xen/interface/xen.h>
 33 #include <xen/interface/vcpu.h>
 34 #include <xen/interface/xenpmu.h>
 35 
 36 #include <asm/spec-ctrl.h>
 37 #include <asm/xen/interface.h>
 38 #include <asm/xen/hypercall.h>
 39 
 40 #include <xen/xen.h>
 41 #include <xen/page.h>
 42 #include <xen/events.h>
 43 
 44 #include <xen/hvc-console.h>
 45 #include "xen-ops.h"
 46 #include "mmu.h"
 47 #include "smp.h"
 48 #include "pmu.h"
 49 
 50 cpumask_var_t xen_cpu_initialized_map;
 51 
 52 static DEFINE_PER_CPU(struct xen_common_irq, xen_irq_work) = { .irq = -1 };
 53 static DEFINE_PER_CPU(struct xen_common_irq, xen_pmu_irq) = { .irq = -1 };
 54 
 55 static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id);
 56 
 57 static void cpu_bringup(void)
 58 {
 59         int cpu;
 60 
 61         cpu_init();
 62         touch_softlockup_watchdog();
 63         preempt_disable();
 64 
 65         /* PVH runs in ring 0 and allows us to do native syscalls. Yay! */
 66         if (!xen_feature(XENFEAT_supervisor_mode_kernel)) {
 67                 xen_enable_sysenter();
 68                 xen_enable_syscall();
 69         }
 70         cpu = smp_processor_id();
 71         smp_store_cpu_info(cpu);
 72         cpu_data(cpu).x86_max_cores = 1;
 73         set_cpu_sibling_map(cpu);
 74 
 75         speculative_store_bypass_ht_init();
 76 
 77         xen_setup_cpu_clockevents();
 78 
 79         notify_cpu_starting(cpu);
 80 
 81         set_cpu_online(cpu, true);
 82 
 83         cpu_set_state_online(cpu);  /* Implies full memory barrier. */
 84 
 85         /* We can take interrupts now: we're officially "up". */
 86         local_irq_enable();
 87 }
 88 
 89 asmlinkage __visible void cpu_bringup_and_idle(void)
 90 {
 91         cpu_bringup();
 92         boot_init_stack_canary();
 93         cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
 94 }
 95 
 96 void xen_smp_intr_free_pv(unsigned int cpu)
 97 {
 98         if (per_cpu(xen_irq_work, cpu).irq >= 0) {
 99                 unbind_from_irqhandler(per_cpu(xen_irq_work, cpu).irq, NULL);
100                 per_cpu(xen_irq_work, cpu).irq = -1;
101                 kfree(per_cpu(xen_irq_work, cpu).name);
102                 per_cpu(xen_irq_work, cpu).name = NULL;
103         }
104 
105         if (per_cpu(xen_pmu_irq, cpu).irq >= 0) {
106                 unbind_from_irqhandler(per_cpu(xen_pmu_irq, cpu).irq, NULL);
107                 per_cpu(xen_pmu_irq, cpu).irq = -1;
108                 kfree(per_cpu(xen_pmu_irq, cpu).name);
109                 per_cpu(xen_pmu_irq, cpu).name = NULL;
110         }
111 }
112 
113 int xen_smp_intr_init_pv(unsigned int cpu)
114 {
115         int rc;
116         char *callfunc_name, *pmu_name;
117 
118         callfunc_name = kasprintf(GFP_KERNEL, "irqwork%d", cpu);
119         rc = bind_ipi_to_irqhandler(XEN_IRQ_WORK_VECTOR,
120                                     cpu,
121                                     xen_irq_work_interrupt,
122                                     IRQF_PERCPU|IRQF_NOBALANCING,
123                                     callfunc_name,
124                                     NULL);
125         if (rc < 0)
126                 goto fail;
127         per_cpu(xen_irq_work, cpu).irq = rc;
128         per_cpu(xen_irq_work, cpu).name = callfunc_name;
129 
130         if (is_xen_pmu(cpu)) {
131                 pmu_name = kasprintf(GFP_KERNEL, "pmu%d", cpu);
132                 rc = bind_virq_to_irqhandler(VIRQ_XENPMU, cpu,
133                                              xen_pmu_irq_handler,
134                                              IRQF_PERCPU|IRQF_NOBALANCING,
135                                              pmu_name, NULL);
136                 if (rc < 0)
137                         goto fail;
138                 per_cpu(xen_pmu_irq, cpu).irq = rc;
139                 per_cpu(xen_pmu_irq, cpu).name = pmu_name;
140         }
141 
142         return 0;
143 
144  fail:
145         xen_smp_intr_free_pv(cpu);
146         return rc;
147 }
148 
149 static void __init xen_fill_possible_map(void)
150 {
151         int i, rc;
152 
153         if (xen_initial_domain())
154                 return;
155 
156         for (i = 0; i < nr_cpu_ids; i++) {
157                 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
158                 if (rc >= 0) {
159                         num_processors++;
160                         set_cpu_possible(i, true);
161                 }
162         }
163 }
164 
165 static void __init xen_filter_cpu_maps(void)
166 {
167         int i, rc;
168         unsigned int subtract = 0;
169 
170         if (!xen_initial_domain())
171                 return;
172 
173         num_processors = 0;
174         disabled_cpus = 0;
175         for (i = 0; i < nr_cpu_ids; i++) {
176                 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
177                 if (rc >= 0) {
178                         num_processors++;
179                         set_cpu_possible(i, true);
180                 } else {
181                         set_cpu_possible(i, false);
182                         set_cpu_present(i, false);
183                         subtract++;
184                 }
185         }
186 #ifdef CONFIG_HOTPLUG_CPU
187         /* This is akin to using 'nr_cpus' on the Linux command line.
188          * Which is OK as when we use 'dom0_max_vcpus=X' we can only
189          * have up to X, while nr_cpu_ids is greater than X. This
190          * normally is not a problem, except when CPU hotplugging
191          * is involved and then there might be more than X CPUs
192          * in the guest - which will not work as there is no
193          * hypercall to expand the max number of VCPUs an already
194          * running guest has. So cap it up to X. */
195         if (subtract)
196                 nr_cpu_ids = nr_cpu_ids - subtract;
197 #endif
198 
199 }
200 
201 static void __init xen_pv_smp_prepare_boot_cpu(void)
202 {
203         BUG_ON(smp_processor_id() != 0);
204         native_smp_prepare_boot_cpu();
205 
206         if (!xen_feature(XENFEAT_writable_page_tables))
207                 /* We've switched to the "real" per-cpu gdt, so make
208                  * sure the old memory can be recycled. */
209                 make_lowmem_page_readwrite(xen_initial_gdt);
210 
211 #ifdef CONFIG_X86_32
212         /*
213          * Xen starts us with XEN_FLAT_RING1_DS, but linux code
214          * expects __USER_DS
215          */
216         loadsegment(ds, __USER_DS);
217         loadsegment(es, __USER_DS);
218 #endif
219 
220         xen_filter_cpu_maps();
221         xen_setup_vcpu_info_placement();
222 
223         /*
224          * The alternative logic (which patches the unlock/lock) runs before
225          * the smp bootup up code is activated. Hence we need to set this up
226          * the core kernel is being patched. Otherwise we will have only
227          * modules patched but not core code.
228          */
229         xen_init_spinlocks();
230 }
231 
232 static void __init xen_pv_smp_prepare_cpus(unsigned int max_cpus)
233 {
234         unsigned cpu;
235         unsigned int i;
236 
237         if (skip_ioapic_setup) {
238                 char *m = (max_cpus == 0) ?
239                         "The nosmp parameter is incompatible with Xen; " \
240                         "use Xen dom0_max_vcpus=1 parameter" :
241                         "The noapic parameter is incompatible with Xen";
242 
243                 xen_raw_printk(m);
244                 panic(m);
245         }
246         xen_init_lock_cpu(0);
247 
248         smp_store_boot_cpu_info();
249         cpu_data(0).x86_max_cores = 1;
250 
251         for_each_possible_cpu(i) {
252                 zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
253                 zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
254                 zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
255         }
256         set_cpu_sibling_map(0);
257 
258         speculative_store_bypass_ht_init();
259 
260         xen_pmu_init(0);
261 
262         if (xen_smp_intr_init(0) || xen_smp_intr_init_pv(0))
263                 BUG();
264 
265         if (!alloc_cpumask_var(&xen_cpu_initialized_map, GFP_KERNEL))
266                 panic("could not allocate xen_cpu_initialized_map\n");
267 
268         cpumask_copy(xen_cpu_initialized_map, cpumask_of(0));
269 
270         /* Restrict the possible_map according to max_cpus. */
271         while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) {
272                 for (cpu = nr_cpu_ids - 1; !cpu_possible(cpu); cpu--)
273                         continue;
274                 set_cpu_possible(cpu, false);
275         }
276 
277         for_each_possible_cpu(cpu)
278                 set_cpu_present(cpu, true);
279 }
280 
281 static int
282 cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
283 {
284         struct vcpu_guest_context *ctxt;
285         struct desc_struct *gdt;
286         unsigned long gdt_mfn;
287 
288         /* used to tell cpu_init() that it can proceed with initialization */
289         cpumask_set_cpu(cpu, cpu_callout_mask);
290         if (cpumask_test_and_set_cpu(cpu, xen_cpu_initialized_map))
291                 return 0;
292 
293         ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
294         if (ctxt == NULL)
295                 return -ENOMEM;
296 
297         gdt = get_cpu_gdt_rw(cpu);
298 
299 #ifdef CONFIG_X86_32
300         ctxt->user_regs.fs = __KERNEL_PERCPU;
301         ctxt->user_regs.gs = __KERNEL_STACK_CANARY;
302 #endif
303         memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));
304 
305         /*
306          * Bring up the CPU in cpu_bringup_and_idle() with the stack
307          * pointing just below where pt_regs would be if it were a normal
308          * kernel entry.
309          */
310         ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle;
311         ctxt->flags = VGCF_IN_KERNEL;
312         ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */
313         ctxt->user_regs.ds = __USER_DS;
314         ctxt->user_regs.es = __USER_DS;
315         ctxt->user_regs.ss = __KERNEL_DS;
316         ctxt->user_regs.cs = __KERNEL_CS;
317         ctxt->user_regs.esp = (unsigned long)task_pt_regs(idle);
318 
319         xen_copy_trap_info(ctxt->trap_ctxt);
320 
321         ctxt->ldt_ents = 0;
322 
323         BUG_ON((unsigned long)gdt & ~PAGE_MASK);
324 
325         gdt_mfn = arbitrary_virt_to_mfn(gdt);
326         make_lowmem_page_readonly(gdt);
327         make_lowmem_page_readonly(mfn_to_virt(gdt_mfn));
328 
329         ctxt->gdt_frames[0] = gdt_mfn;
330         ctxt->gdt_ents      = GDT_ENTRIES;
331 
332         /*
333          * Set SS:SP that Xen will use when entering guest kernel mode
334          * from guest user mode.  Subsequent calls to load_sp0() can
335          * change this value.
336          */
337         ctxt->kernel_ss = __KERNEL_DS;
338         ctxt->kernel_sp = task_top_of_stack(idle);
339 
340 #ifdef CONFIG_X86_32
341         ctxt->event_callback_cs     = __KERNEL_CS;
342         ctxt->failsafe_callback_cs  = __KERNEL_CS;
343 #else
344         ctxt->gs_base_kernel = per_cpu_offset(cpu);
345 #endif
346         ctxt->event_callback_eip    =
347                 (unsigned long)xen_hypervisor_callback;
348         ctxt->failsafe_callback_eip =
349                 (unsigned long)xen_failsafe_callback;
350         per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir);
351 
352         ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_gfn(swapper_pg_dir));
353         if (HYPERVISOR_vcpu_op(VCPUOP_initialise, xen_vcpu_nr(cpu), ctxt))
354                 BUG();
355 
356         kfree(ctxt);
357         return 0;
358 }
359 
360 static int xen_pv_cpu_up(unsigned int cpu, struct task_struct *idle)
361 {
362         int rc;
363 
364         rc = common_cpu_up(cpu, idle);
365         if (rc)
366                 return rc;
367 
368         xen_setup_runstate_info(cpu);
369 
370         /*
371          * PV VCPUs are always successfully taken down (see 'while' loop
372          * in xen_cpu_die()), so -EBUSY is an error.
373          */
374         rc = cpu_check_up_prepare(cpu);
375         if (rc)
376                 return rc;
377 
378         /* make sure interrupts start blocked */
379         per_cpu(xen_vcpu, cpu)->evtchn_upcall_mask = 1;
380 
381         rc = cpu_initialize_context(cpu, idle);
382         if (rc)
383                 return rc;
384 
385         xen_pmu_init(cpu);
386 
387         rc = HYPERVISOR_vcpu_op(VCPUOP_up, xen_vcpu_nr(cpu), NULL);
388         BUG_ON(rc);
389 
390         while (cpu_report_state(cpu) != CPU_ONLINE)
391                 HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
392 
393         return 0;
394 }
395 
396 #ifdef CONFIG_HOTPLUG_CPU
397 static int xen_pv_cpu_disable(void)
398 {
399         unsigned int cpu = smp_processor_id();
400         if (cpu == 0)
401                 return -EBUSY;
402 
403         cpu_disable_common();
404 
405         load_cr3(swapper_pg_dir);
406         return 0;
407 }
408 
409 static void xen_pv_cpu_die(unsigned int cpu)
410 {
411         while (HYPERVISOR_vcpu_op(VCPUOP_is_up,
412                                   xen_vcpu_nr(cpu), NULL)) {
413                 __set_current_state(TASK_UNINTERRUPTIBLE);
414                 schedule_timeout(HZ/10);
415         }
416 
417         if (common_cpu_die(cpu) == 0) {
418                 xen_smp_intr_free(cpu);
419                 xen_uninit_lock_cpu(cpu);
420                 xen_teardown_timer(cpu);
421                 xen_pmu_finish(cpu);
422         }
423 }
424 
425 static void xen_pv_play_dead(void) /* used only with HOTPLUG_CPU */
426 {
427         play_dead_common();
428         HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(smp_processor_id()), NULL);
429         cpu_bringup();
430         /*
431          * commit 4b0c0f294 (tick: Cleanup NOHZ per cpu data on cpu down)
432          * clears certain data that the cpu_idle loop (which called us
433          * and that we return from) expects. The only way to get that
434          * data back is to call:
435          */
436         tick_nohz_idle_enter();
437         tick_nohz_idle_stop_tick_protected();
438 
439         cpuhp_online_idle(CPUHP_AP_ONLINE_IDLE);
440 }
441 
442 #else /* !CONFIG_HOTPLUG_CPU */
443 static int xen_pv_cpu_disable(void)
444 {
445         return -ENOSYS;
446 }
447 
448 static void xen_pv_cpu_die(unsigned int cpu)
449 {
450         BUG();
451 }
452 
453 static void xen_pv_play_dead(void)
454 {
455         BUG();
456 }
457 
458 #endif
459 static void stop_self(void *v)
460 {
461         int cpu = smp_processor_id();
462 
463         /* make sure we're not pinning something down */
464         load_cr3(swapper_pg_dir);
465         /* should set up a minimal gdt */
466 
467         set_cpu_online(cpu, false);
468 
469         HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(cpu), NULL);
470         BUG();
471 }
472 
473 static void xen_pv_stop_other_cpus(int wait)
474 {
475         smp_call_function(stop_self, NULL, wait);
476 }
477 
478 static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id)
479 {
480         irq_enter();
481         irq_work_run();
482         inc_irq_stat(apic_irq_work_irqs);
483         irq_exit();
484 
485         return IRQ_HANDLED;
486 }
487 
488 static const struct smp_ops xen_smp_ops __initconst = {
489         .smp_prepare_boot_cpu = xen_pv_smp_prepare_boot_cpu,
490         .smp_prepare_cpus = xen_pv_smp_prepare_cpus,
491         .smp_cpus_done = xen_smp_cpus_done,
492 
493         .cpu_up = xen_pv_cpu_up,
494         .cpu_die = xen_pv_cpu_die,
495         .cpu_disable = xen_pv_cpu_disable,
496         .play_dead = xen_pv_play_dead,
497 
498         .stop_other_cpus = xen_pv_stop_other_cpus,
499         .smp_send_reschedule = xen_smp_send_reschedule,
500 
501         .send_call_func_ipi = xen_smp_send_call_function_ipi,
502         .send_call_func_single_ipi = xen_smp_send_call_function_single_ipi,
503 };
504 
505 void __init xen_smp_init(void)
506 {
507         smp_ops = xen_smp_ops;
508         xen_fill_possible_map();
509 }
510 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp