~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/xen/enlighten.c

Version: ~ [ linux-5.2-rc1 ] ~ [ linux-5.1.2 ] ~ [ linux-5.0.16 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.43 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.119 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.176 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.179 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.139 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.67 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Core of Xen paravirt_ops implementation.
  3  *
  4  * This file contains the xen_paravirt_ops structure itself, and the
  5  * implementations for:
  6  * - privileged instructions
  7  * - interrupt flags
  8  * - segment operations
  9  * - booting and setup
 10  *
 11  * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
 12  */
 13 
 14 #include <linux/cpu.h>
 15 #include <linux/kernel.h>
 16 #include <linux/init.h>
 17 #include <linux/smp.h>
 18 #include <linux/preempt.h>
 19 #include <linux/hardirq.h>
 20 #include <linux/percpu.h>
 21 #include <linux/delay.h>
 22 #include <linux/start_kernel.h>
 23 #include <linux/sched.h>
 24 #include <linux/kprobes.h>
 25 #include <linux/bootmem.h>
 26 #include <linux/module.h>
 27 #include <linux/mm.h>
 28 #include <linux/page-flags.h>
 29 #include <linux/highmem.h>
 30 #include <linux/console.h>
 31 #include <linux/pci.h>
 32 #include <linux/gfp.h>
 33 #include <linux/memblock.h>
 34 
 35 #include <xen/xen.h>
 36 #include <xen/interface/xen.h>
 37 #include <xen/interface/version.h>
 38 #include <xen/interface/physdev.h>
 39 #include <xen/interface/vcpu.h>
 40 #include <xen/interface/memory.h>
 41 #include <xen/features.h>
 42 #include <xen/page.h>
 43 #include <xen/hvm.h>
 44 #include <xen/hvc-console.h>
 45 #include <xen/acpi.h>
 46 
 47 #include <asm/paravirt.h>
 48 #include <asm/apic.h>
 49 #include <asm/page.h>
 50 #include <asm/xen/pci.h>
 51 #include <asm/xen/hypercall.h>
 52 #include <asm/xen/hypervisor.h>
 53 #include <asm/fixmap.h>
 54 #include <asm/processor.h>
 55 #include <asm/proto.h>
 56 #include <asm/msr-index.h>
 57 #include <asm/traps.h>
 58 #include <asm/setup.h>
 59 #include <asm/desc.h>
 60 #include <asm/pgalloc.h>
 61 #include <asm/pgtable.h>
 62 #include <asm/tlbflush.h>
 63 #include <asm/reboot.h>
 64 #include <asm/stackprotector.h>
 65 #include <asm/hypervisor.h>
 66 #include <asm/mwait.h>
 67 #include <asm/pci_x86.h>
 68 
 69 #ifdef CONFIG_ACPI
 70 #include <linux/acpi.h>
 71 #include <asm/acpi.h>
 72 #include <acpi/pdc_intel.h>
 73 #include <acpi/processor.h>
 74 #include <xen/interface/platform.h>
 75 #endif
 76 
 77 #include "xen-ops.h"
 78 #include "mmu.h"
 79 #include "smp.h"
 80 #include "multicalls.h"
 81 
 82 EXPORT_SYMBOL_GPL(hypercall_page);
 83 
 84 DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu);
 85 DEFINE_PER_CPU(struct vcpu_info, xen_vcpu_info);
 86 
 87 enum xen_domain_type xen_domain_type = XEN_NATIVE;
 88 EXPORT_SYMBOL_GPL(xen_domain_type);
 89 
 90 unsigned long *machine_to_phys_mapping = (void *)MACH2PHYS_VIRT_START;
 91 EXPORT_SYMBOL(machine_to_phys_mapping);
 92 unsigned long  machine_to_phys_nr;
 93 EXPORT_SYMBOL(machine_to_phys_nr);
 94 
 95 struct start_info *xen_start_info;
 96 EXPORT_SYMBOL_GPL(xen_start_info);
 97 
 98 struct shared_info xen_dummy_shared_info;
 99 
100 void *xen_initial_gdt;
101 
102 RESERVE_BRK(shared_info_page_brk, PAGE_SIZE);
103 __read_mostly int xen_have_vector_callback;
104 EXPORT_SYMBOL_GPL(xen_have_vector_callback);
105 
106 /*
107  * Point at some empty memory to start with. We map the real shared_info
108  * page as soon as fixmap is up and running.
109  */
110 struct shared_info *HYPERVISOR_shared_info = (void *)&xen_dummy_shared_info;
111 
112 /*
113  * Flag to determine whether vcpu info placement is available on all
114  * VCPUs.  We assume it is to start with, and then set it to zero on
115  * the first failure.  This is because it can succeed on some VCPUs
116  * and not others, since it can involve hypervisor memory allocation,
117  * or because the guest failed to guarantee all the appropriate
118  * constraints on all VCPUs (ie buffer can't cross a page boundary).
119  *
120  * Note that any particular CPU may be using a placed vcpu structure,
121  * but we can only optimise if the all are.
122  *
123  * 0: not available, 1: available
124  */
125 static int have_vcpu_info_placement = 1;
126 
127 static void clamp_max_cpus(void)
128 {
129 #ifdef CONFIG_SMP
130         if (setup_max_cpus > MAX_VIRT_CPUS)
131                 setup_max_cpus = MAX_VIRT_CPUS;
132 #endif
133 }
134 
135 static void xen_vcpu_setup(int cpu)
136 {
137         struct vcpu_register_vcpu_info info;
138         int err;
139         struct vcpu_info *vcpup;
140 
141         BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info);
142 
143         if (cpu < MAX_VIRT_CPUS)
144                 per_cpu(xen_vcpu,cpu) = &HYPERVISOR_shared_info->vcpu_info[cpu];
145 
146         if (!have_vcpu_info_placement) {
147                 if (cpu >= MAX_VIRT_CPUS)
148                         clamp_max_cpus();
149                 return;
150         }
151 
152         vcpup = &per_cpu(xen_vcpu_info, cpu);
153         info.mfn = arbitrary_virt_to_mfn(vcpup);
154         info.offset = offset_in_page(vcpup);
155 
156         /* Check to see if the hypervisor will put the vcpu_info
157            structure where we want it, which allows direct access via
158            a percpu-variable. */
159         err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, cpu, &info);
160 
161         if (err) {
162                 printk(KERN_DEBUG "register_vcpu_info failed: err=%d\n", err);
163                 have_vcpu_info_placement = 0;
164                 clamp_max_cpus();
165         } else {
166                 /* This cpu is using the registered vcpu info, even if
167                    later ones fail to. */
168                 per_cpu(xen_vcpu, cpu) = vcpup;
169         }
170 }
171 
172 /*
173  * On restore, set the vcpu placement up again.
174  * If it fails, then we're in a bad state, since
175  * we can't back out from using it...
176  */
177 void xen_vcpu_restore(void)
178 {
179         int cpu;
180 
181         for_each_online_cpu(cpu) {
182                 bool other_cpu = (cpu != smp_processor_id());
183 
184                 if (other_cpu &&
185                     HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL))
186                         BUG();
187 
188                 xen_setup_runstate_info(cpu);
189 
190                 if (have_vcpu_info_placement)
191                         xen_vcpu_setup(cpu);
192 
193                 if (other_cpu &&
194                     HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL))
195                         BUG();
196         }
197 }
198 
199 static void __init xen_banner(void)
200 {
201         unsigned version = HYPERVISOR_xen_version(XENVER_version, NULL);
202         struct xen_extraversion extra;
203         HYPERVISOR_xen_version(XENVER_extraversion, &extra);
204 
205         printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
206                pv_info.name);
207         printk(KERN_INFO "Xen version: %d.%d%s%s\n",
208                version >> 16, version & 0xffff, extra.extraversion,
209                xen_feature(XENFEAT_mmu_pt_update_preserve_ad) ? " (preserve-AD)" : "");
210 }
211 
212 #define CPUID_THERM_POWER_LEAF 6
213 #define APERFMPERF_PRESENT 0
214 
215 static __read_mostly unsigned int cpuid_leaf1_edx_mask = ~0;
216 static __read_mostly unsigned int cpuid_leaf1_ecx_mask = ~0;
217 
218 static __read_mostly unsigned int cpuid_leaf1_ecx_set_mask;
219 static __read_mostly unsigned int cpuid_leaf5_ecx_val;
220 static __read_mostly unsigned int cpuid_leaf5_edx_val;
221 
222 static void xen_cpuid(unsigned int *ax, unsigned int *bx,
223                       unsigned int *cx, unsigned int *dx)
224 {
225         unsigned maskebx = ~0;
226         unsigned maskecx = ~0;
227         unsigned maskedx = ~0;
228         unsigned setecx = 0;
229         /*
230          * Mask out inconvenient features, to try and disable as many
231          * unsupported kernel subsystems as possible.
232          */
233         switch (*ax) {
234         case 1:
235                 maskecx = cpuid_leaf1_ecx_mask;
236                 setecx = cpuid_leaf1_ecx_set_mask;
237                 maskedx = cpuid_leaf1_edx_mask;
238                 break;
239 
240         case CPUID_MWAIT_LEAF:
241                 /* Synthesize the values.. */
242                 *ax = 0;
243                 *bx = 0;
244                 *cx = cpuid_leaf5_ecx_val;
245                 *dx = cpuid_leaf5_edx_val;
246                 return;
247 
248         case CPUID_THERM_POWER_LEAF:
249                 /* Disabling APERFMPERF for kernel usage */
250                 maskecx = ~(1 << APERFMPERF_PRESENT);
251                 break;
252 
253         case 0xb:
254                 /* Suppress extended topology stuff */
255                 maskebx = 0;
256                 break;
257         }
258 
259         asm(XEN_EMULATE_PREFIX "cpuid"
260                 : "=a" (*ax),
261                   "=b" (*bx),
262                   "=c" (*cx),
263                   "=d" (*dx)
264                 : "" (*ax), "2" (*cx));
265 
266         *bx &= maskebx;
267         *cx &= maskecx;
268         *cx |= setecx;
269         *dx &= maskedx;
270 
271 }
272 
273 static bool __init xen_check_mwait(void)
274 {
275 #if defined(CONFIG_ACPI) && !defined(CONFIG_ACPI_PROCESSOR_AGGREGATOR) && \
276         !defined(CONFIG_ACPI_PROCESSOR_AGGREGATOR_MODULE)
277         struct xen_platform_op op = {
278                 .cmd                    = XENPF_set_processor_pminfo,
279                 .u.set_pminfo.id        = -1,
280                 .u.set_pminfo.type      = XEN_PM_PDC,
281         };
282         uint32_t buf[3];
283         unsigned int ax, bx, cx, dx;
284         unsigned int mwait_mask;
285 
286         /* We need to determine whether it is OK to expose the MWAIT
287          * capability to the kernel to harvest deeper than C3 states from ACPI
288          * _CST using the processor_harvest_xen.c module. For this to work, we
289          * need to gather the MWAIT_LEAF values (which the cstate.c code
290          * checks against). The hypervisor won't expose the MWAIT flag because
291          * it would break backwards compatibility; so we will find out directly
292          * from the hardware and hypercall.
293          */
294         if (!xen_initial_domain())
295                 return false;
296 
297         ax = 1;
298         cx = 0;
299 
300         native_cpuid(&ax, &bx, &cx, &dx);
301 
302         mwait_mask = (1 << (X86_FEATURE_EST % 32)) |
303                      (1 << (X86_FEATURE_MWAIT % 32));
304 
305         if ((cx & mwait_mask) != mwait_mask)
306                 return false;
307 
308         /* We need to emulate the MWAIT_LEAF and for that we need both
309          * ecx and edx. The hypercall provides only partial information.
310          */
311 
312         ax = CPUID_MWAIT_LEAF;
313         bx = 0;
314         cx = 0;
315         dx = 0;
316 
317         native_cpuid(&ax, &bx, &cx, &dx);
318 
319         /* Ask the Hypervisor whether to clear ACPI_PDC_C_C2C3_FFH. If so,
320          * don't expose MWAIT_LEAF and let ACPI pick the IOPORT version of C3.
321          */
322         buf[0] = ACPI_PDC_REVISION_ID;
323         buf[1] = 1;
324         buf[2] = (ACPI_PDC_C_CAPABILITY_SMP | ACPI_PDC_EST_CAPABILITY_SWSMP);
325 
326         set_xen_guest_handle(op.u.set_pminfo.pdc, buf);
327 
328         if ((HYPERVISOR_dom0_op(&op) == 0) &&
329             (buf[2] & (ACPI_PDC_C_C1_FFH | ACPI_PDC_C_C2C3_FFH))) {
330                 cpuid_leaf5_ecx_val = cx;
331                 cpuid_leaf5_edx_val = dx;
332         }
333         return true;
334 #else
335         return false;
336 #endif
337 }
338 static void __init xen_init_cpuid_mask(void)
339 {
340         unsigned int ax, bx, cx, dx;
341         unsigned int xsave_mask;
342 
343         cpuid_leaf1_edx_mask =
344                 ~((1 << X86_FEATURE_MCE)  |  /* disable MCE */
345                   (1 << X86_FEATURE_MCA)  |  /* disable MCA */
346                   (1 << X86_FEATURE_MTRR) |  /* disable MTRR */
347                   (1 << X86_FEATURE_ACC));   /* thermal monitoring */
348 
349         if (!xen_initial_domain())
350                 cpuid_leaf1_edx_mask &=
351                         ~((1 << X86_FEATURE_APIC) |  /* disable local APIC */
352                           (1 << X86_FEATURE_ACPI));  /* disable ACPI */
353         ax = 1;
354         cx = 0;
355         xen_cpuid(&ax, &bx, &cx, &dx);
356 
357         xsave_mask =
358                 (1 << (X86_FEATURE_XSAVE % 32)) |
359                 (1 << (X86_FEATURE_OSXSAVE % 32));
360 
361         /* Xen will set CR4.OSXSAVE if supported and not disabled by force */
362         if ((cx & xsave_mask) != xsave_mask)
363                 cpuid_leaf1_ecx_mask &= ~xsave_mask; /* disable XSAVE & OSXSAVE */
364         if (xen_check_mwait())
365                 cpuid_leaf1_ecx_set_mask = (1 << (X86_FEATURE_MWAIT % 32));
366 }
367 
368 static void xen_set_debugreg(int reg, unsigned long val)
369 {
370         HYPERVISOR_set_debugreg(reg, val);
371 }
372 
373 static unsigned long xen_get_debugreg(int reg)
374 {
375         return HYPERVISOR_get_debugreg(reg);
376 }
377 
378 static void xen_end_context_switch(struct task_struct *next)
379 {
380         xen_mc_flush();
381         paravirt_end_context_switch(next);
382 }
383 
384 static unsigned long xen_store_tr(void)
385 {
386         return 0;
387 }
388 
389 /*
390  * Set the page permissions for a particular virtual address.  If the
391  * address is a vmalloc mapping (or other non-linear mapping), then
392  * find the linear mapping of the page and also set its protections to
393  * match.
394  */
395 static void set_aliased_prot(void *v, pgprot_t prot)
396 {
397         int level;
398         pte_t *ptep;
399         pte_t pte;
400         unsigned long pfn;
401         struct page *page;
402 
403         ptep = lookup_address((unsigned long)v, &level);
404         BUG_ON(ptep == NULL);
405 
406         pfn = pte_pfn(*ptep);
407         page = pfn_to_page(pfn);
408 
409         pte = pfn_pte(pfn, prot);
410 
411         if (HYPERVISOR_update_va_mapping((unsigned long)v, pte, 0))
412                 BUG();
413 
414         if (!PageHighMem(page)) {
415                 void *av = __va(PFN_PHYS(pfn));
416 
417                 if (av != v)
418                         if (HYPERVISOR_update_va_mapping((unsigned long)av, pte, 0))
419                                 BUG();
420         } else
421                 kmap_flush_unused();
422 }
423 
424 static void xen_alloc_ldt(struct desc_struct *ldt, unsigned entries)
425 {
426         const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE;
427         int i;
428 
429         for(i = 0; i < entries; i += entries_per_page)
430                 set_aliased_prot(ldt + i, PAGE_KERNEL_RO);
431 }
432 
433 static void xen_free_ldt(struct desc_struct *ldt, unsigned entries)
434 {
435         const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE;
436         int i;
437 
438         for(i = 0; i < entries; i += entries_per_page)
439                 set_aliased_prot(ldt + i, PAGE_KERNEL);
440 }
441 
442 static void xen_set_ldt(const void *addr, unsigned entries)
443 {
444         struct mmuext_op *op;
445         struct multicall_space mcs = xen_mc_entry(sizeof(*op));
446 
447         trace_xen_cpu_set_ldt(addr, entries);
448 
449         op = mcs.args;
450         op->cmd = MMUEXT_SET_LDT;
451         op->arg1.linear_addr = (unsigned long)addr;
452         op->arg2.nr_ents = entries;
453 
454         MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
455 
456         xen_mc_issue(PARAVIRT_LAZY_CPU);
457 }
458 
459 static void xen_load_gdt(const struct desc_ptr *dtr)
460 {
461         unsigned long va = dtr->address;
462         unsigned int size = dtr->size + 1;
463         unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
464         unsigned long frames[pages];
465         int f;
466 
467         /*
468          * A GDT can be up to 64k in size, which corresponds to 8192
469          * 8-byte entries, or 16 4k pages..
470          */
471 
472         BUG_ON(size > 65536);
473         BUG_ON(va & ~PAGE_MASK);
474 
475         for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
476                 int level;
477                 pte_t *ptep;
478                 unsigned long pfn, mfn;
479                 void *virt;
480 
481                 /*
482                  * The GDT is per-cpu and is in the percpu data area.
483                  * That can be virtually mapped, so we need to do a
484                  * page-walk to get the underlying MFN for the
485                  * hypercall.  The page can also be in the kernel's
486                  * linear range, so we need to RO that mapping too.
487                  */
488                 ptep = lookup_address(va, &level);
489                 BUG_ON(ptep == NULL);
490 
491                 pfn = pte_pfn(*ptep);
492                 mfn = pfn_to_mfn(pfn);
493                 virt = __va(PFN_PHYS(pfn));
494 
495                 frames[f] = mfn;
496 
497                 make_lowmem_page_readonly((void *)va);
498                 make_lowmem_page_readonly(virt);
499         }
500 
501         if (HYPERVISOR_set_gdt(frames, size / sizeof(struct desc_struct)))
502                 BUG();
503 }
504 
505 /*
506  * load_gdt for early boot, when the gdt is only mapped once
507  */
508 static void __init xen_load_gdt_boot(const struct desc_ptr *dtr)
509 {
510         unsigned long va = dtr->address;
511         unsigned int size = dtr->size + 1;
512         unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
513         unsigned long frames[pages];
514         int f;
515 
516         /*
517          * A GDT can be up to 64k in size, which corresponds to 8192
518          * 8-byte entries, or 16 4k pages..
519          */
520 
521         BUG_ON(size > 65536);
522         BUG_ON(va & ~PAGE_MASK);
523 
524         for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
525                 pte_t pte;
526                 unsigned long pfn, mfn;
527 
528                 pfn = virt_to_pfn(va);
529                 mfn = pfn_to_mfn(pfn);
530 
531                 pte = pfn_pte(pfn, PAGE_KERNEL_RO);
532 
533                 if (HYPERVISOR_update_va_mapping((unsigned long)va, pte, 0))
534                         BUG();
535 
536                 frames[f] = mfn;
537         }
538 
539         if (HYPERVISOR_set_gdt(frames, size / sizeof(struct desc_struct)))
540                 BUG();
541 }
542 
543 static void load_TLS_descriptor(struct thread_struct *t,
544                                 unsigned int cpu, unsigned int i)
545 {
546         struct desc_struct *gdt = get_cpu_gdt_table(cpu);
547         xmaddr_t maddr = arbitrary_virt_to_machine(&gdt[GDT_ENTRY_TLS_MIN+i]);
548         struct multicall_space mc = __xen_mc_entry(0);
549 
550         MULTI_update_descriptor(mc.mc, maddr.maddr, t->tls_array[i]);
551 }
552 
553 static void xen_load_tls(struct thread_struct *t, unsigned int cpu)
554 {
555         /*
556          * XXX sleazy hack: If we're being called in a lazy-cpu zone
557          * and lazy gs handling is enabled, it means we're in a
558          * context switch, and %gs has just been saved.  This means we
559          * can zero it out to prevent faults on exit from the
560          * hypervisor if the next process has no %gs.  Either way, it
561          * has been saved, and the new value will get loaded properly.
562          * This will go away as soon as Xen has been modified to not
563          * save/restore %gs for normal hypercalls.
564          *
565          * On x86_64, this hack is not used for %gs, because gs points
566          * to KERNEL_GS_BASE (and uses it for PDA references), so we
567          * must not zero %gs on x86_64
568          *
569          * For x86_64, we need to zero %fs, otherwise we may get an
570          * exception between the new %fs descriptor being loaded and
571          * %fs being effectively cleared at __switch_to().
572          */
573         if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_CPU) {
574 #ifdef CONFIG_X86_32
575                 lazy_load_gs(0);
576 #else
577                 loadsegment(fs, 0);
578 #endif
579         }
580 
581         xen_mc_batch();
582 
583         load_TLS_descriptor(t, cpu, 0);
584         load_TLS_descriptor(t, cpu, 1);
585         load_TLS_descriptor(t, cpu, 2);
586 
587         xen_mc_issue(PARAVIRT_LAZY_CPU);
588 }
589 
590 #ifdef CONFIG_X86_64
591 static void xen_load_gs_index(unsigned int idx)
592 {
593         if (HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, idx))
594                 BUG();
595 }
596 #endif
597 
598 static void xen_write_ldt_entry(struct desc_struct *dt, int entrynum,
599                                 const void *ptr)
600 {
601         xmaddr_t mach_lp = arbitrary_virt_to_machine(&dt[entrynum]);
602         u64 entry = *(u64 *)ptr;
603 
604         trace_xen_cpu_write_ldt_entry(dt, entrynum, entry);
605 
606         preempt_disable();
607 
608         xen_mc_flush();
609         if (HYPERVISOR_update_descriptor(mach_lp.maddr, entry))
610                 BUG();
611 
612         preempt_enable();
613 }
614 
615 static int cvt_gate_to_trap(int vector, const gate_desc *val,
616                             struct trap_info *info)
617 {
618         unsigned long addr;
619 
620         if (val->type != GATE_TRAP && val->type != GATE_INTERRUPT)
621                 return 0;
622 
623         info->vector = vector;
624 
625         addr = gate_offset(*val);
626 #ifdef CONFIG_X86_64
627         /*
628          * Look for known traps using IST, and substitute them
629          * appropriately.  The debugger ones are the only ones we care
630          * about.  Xen will handle faults like double_fault and
631          * machine_check, so we should never see them.  Warn if
632          * there's an unexpected IST-using fault handler.
633          */
634         if (addr == (unsigned long)debug)
635                 addr = (unsigned long)xen_debug;
636         else if (addr == (unsigned long)int3)
637                 addr = (unsigned long)xen_int3;
638         else if (addr == (unsigned long)stack_segment)
639                 addr = (unsigned long)xen_stack_segment;
640         else if (addr == (unsigned long)double_fault ||
641                  addr == (unsigned long)nmi) {
642                 /* Don't need to handle these */
643                 return 0;
644 #ifdef CONFIG_X86_MCE
645         } else if (addr == (unsigned long)machine_check) {
646                 return 0;
647 #endif
648         } else {
649                 /* Some other trap using IST? */
650                 if (WARN_ON(val->ist != 0))
651                         return 0;
652         }
653 #endif  /* CONFIG_X86_64 */
654         info->address = addr;
655 
656         info->cs = gate_segment(*val);
657         info->flags = val->dpl;
658         /* interrupt gates clear IF */
659         if (val->type == GATE_INTERRUPT)
660                 info->flags |= 1 << 2;
661 
662         return 1;
663 }
664 
665 /* Locations of each CPU's IDT */
666 static DEFINE_PER_CPU(struct desc_ptr, idt_desc);
667 
668 /* Set an IDT entry.  If the entry is part of the current IDT, then
669    also update Xen. */
670 static void xen_write_idt_entry(gate_desc *dt, int entrynum, const gate_desc *g)
671 {
672         unsigned long p = (unsigned long)&dt[entrynum];
673         unsigned long start, end;
674 
675         trace_xen_cpu_write_idt_entry(dt, entrynum, g);
676 
677         preempt_disable();
678 
679         start = __this_cpu_read(idt_desc.address);
680         end = start + __this_cpu_read(idt_desc.size) + 1;
681 
682         xen_mc_flush();
683 
684         native_write_idt_entry(dt, entrynum, g);
685 
686         if (p >= start && (p + 8) <= end) {
687                 struct trap_info info[2];
688 
689                 info[1].address = 0;
690 
691                 if (cvt_gate_to_trap(entrynum, g, &info[0]))
692                         if (HYPERVISOR_set_trap_table(info))
693                                 BUG();
694         }
695 
696         preempt_enable();
697 }
698 
699 static void xen_convert_trap_info(const struct desc_ptr *desc,
700                                   struct trap_info *traps)
701 {
702         unsigned in, out, count;
703 
704         count = (desc->size+1) / sizeof(gate_desc);
705         BUG_ON(count > 256);
706 
707         for (in = out = 0; in < count; in++) {
708                 gate_desc *entry = (gate_desc*)(desc->address) + in;
709 
710                 if (cvt_gate_to_trap(in, entry, &traps[out]))
711                         out++;
712         }
713         traps[out].address = 0;
714 }
715 
716 void xen_copy_trap_info(struct trap_info *traps)
717 {
718         const struct desc_ptr *desc = &__get_cpu_var(idt_desc);
719 
720         xen_convert_trap_info(desc, traps);
721 }
722 
723 /* Load a new IDT into Xen.  In principle this can be per-CPU, so we
724    hold a spinlock to protect the static traps[] array (static because
725    it avoids allocation, and saves stack space). */
726 static void xen_load_idt(const struct desc_ptr *desc)
727 {
728         static DEFINE_SPINLOCK(lock);
729         static struct trap_info traps[257];
730 
731         trace_xen_cpu_load_idt(desc);
732 
733         spin_lock(&lock);
734 
735         __get_cpu_var(idt_desc) = *desc;
736 
737         xen_convert_trap_info(desc, traps);
738 
739         xen_mc_flush();
740         if (HYPERVISOR_set_trap_table(traps))
741                 BUG();
742 
743         spin_unlock(&lock);
744 }
745 
746 /* Write a GDT descriptor entry.  Ignore LDT descriptors, since
747    they're handled differently. */
748 static void xen_write_gdt_entry(struct desc_struct *dt, int entry,
749                                 const void *desc, int type)
750 {
751         trace_xen_cpu_write_gdt_entry(dt, entry, desc, type);
752 
753         preempt_disable();
754 
755         switch (type) {
756         case DESC_LDT:
757         case DESC_TSS:
758                 /* ignore */
759                 break;
760 
761         default: {
762                 xmaddr_t maddr = arbitrary_virt_to_machine(&dt[entry]);
763 
764                 xen_mc_flush();
765                 if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc))
766                         BUG();
767         }
768 
769         }
770 
771         preempt_enable();
772 }
773 
774 /*
775  * Version of write_gdt_entry for use at early boot-time needed to
776  * update an entry as simply as possible.
777  */
778 static void __init xen_write_gdt_entry_boot(struct desc_struct *dt, int entry,
779                                             const void *desc, int type)
780 {
781         trace_xen_cpu_write_gdt_entry(dt, entry, desc, type);
782 
783         switch (type) {
784         case DESC_LDT:
785         case DESC_TSS:
786                 /* ignore */
787                 break;
788 
789         default: {
790                 xmaddr_t maddr = virt_to_machine(&dt[entry]);
791 
792                 if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc))
793                         dt[entry] = *(struct desc_struct *)desc;
794         }
795 
796         }
797 }
798 
799 static void xen_load_sp0(struct tss_struct *tss,
800                          struct thread_struct *thread)
801 {
802         struct multicall_space mcs;
803 
804         mcs = xen_mc_entry(0);
805         MULTI_stack_switch(mcs.mc, __KERNEL_DS, thread->sp0);
806         xen_mc_issue(PARAVIRT_LAZY_CPU);
807 }
808 
809 static void xen_set_iopl_mask(unsigned mask)
810 {
811         struct physdev_set_iopl set_iopl;
812 
813         /* Force the change at ring 0. */
814         set_iopl.iopl = (mask == 0) ? 1 : (mask >> 12) & 3;
815         HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
816 }
817 
818 static void xen_io_delay(void)
819 {
820 }
821 
822 #ifdef CONFIG_X86_LOCAL_APIC
823 static unsigned long xen_set_apic_id(unsigned int x)
824 {
825         WARN_ON(1);
826         return x;
827 }
828 static unsigned int xen_get_apic_id(unsigned long x)
829 {
830         return ((x)>>24) & 0xFFu;
831 }
832 static u32 xen_apic_read(u32 reg)
833 {
834         struct xen_platform_op op = {
835                 .cmd = XENPF_get_cpuinfo,
836                 .interface_version = XENPF_INTERFACE_VERSION,
837                 .u.pcpu_info.xen_cpuid = 0,
838         };
839         int ret = 0;
840 
841         /* Shouldn't need this as APIC is turned off for PV, and we only
842          * get called on the bootup processor. But just in case. */
843         if (!xen_initial_domain() || smp_processor_id())
844                 return 0;
845 
846         if (reg == APIC_LVR)
847                 return 0x10;
848 
849         if (reg != APIC_ID)
850                 return 0;
851 
852         ret = HYPERVISOR_dom0_op(&op);
853         if (ret)
854                 return 0;
855 
856         return op.u.pcpu_info.apic_id << 24;
857 }
858 
859 static void xen_apic_write(u32 reg, u32 val)
860 {
861         /* Warn to see if there's any stray references */
862         WARN_ON(1);
863 }
864 
865 static u64 xen_apic_icr_read(void)
866 {
867         return 0;
868 }
869 
870 static void xen_apic_icr_write(u32 low, u32 id)
871 {
872         /* Warn to see if there's any stray references */
873         WARN_ON(1);
874 }
875 
876 static void xen_apic_wait_icr_idle(void)
877 {
878         return;
879 }
880 
881 static u32 xen_safe_apic_wait_icr_idle(void)
882 {
883         return 0;
884 }
885 
886 static void set_xen_basic_apic_ops(void)
887 {
888         apic->read = xen_apic_read;
889         apic->write = xen_apic_write;
890         apic->icr_read = xen_apic_icr_read;
891         apic->icr_write = xen_apic_icr_write;
892         apic->wait_icr_idle = xen_apic_wait_icr_idle;
893         apic->safe_wait_icr_idle = xen_safe_apic_wait_icr_idle;
894         apic->set_apic_id = xen_set_apic_id;
895         apic->get_apic_id = xen_get_apic_id;
896 
897 #ifdef CONFIG_SMP
898         apic->send_IPI_allbutself = xen_send_IPI_allbutself;
899         apic->send_IPI_mask_allbutself = xen_send_IPI_mask_allbutself;
900         apic->send_IPI_mask = xen_send_IPI_mask;
901         apic->send_IPI_all = xen_send_IPI_all;
902         apic->send_IPI_self = xen_send_IPI_self;
903 #endif
904 }
905 
906 #endif
907 
908 static void xen_clts(void)
909 {
910         struct multicall_space mcs;
911 
912         mcs = xen_mc_entry(0);
913 
914         MULTI_fpu_taskswitch(mcs.mc, 0);
915 
916         xen_mc_issue(PARAVIRT_LAZY_CPU);
917 }
918 
919 static DEFINE_PER_CPU(unsigned long, xen_cr0_value);
920 
921 static unsigned long xen_read_cr0(void)
922 {
923         unsigned long cr0 = this_cpu_read(xen_cr0_value);
924 
925         if (unlikely(cr0 == 0)) {
926                 cr0 = native_read_cr0();
927                 this_cpu_write(xen_cr0_value, cr0);
928         }
929 
930         return cr0;
931 }
932 
933 static void xen_write_cr0(unsigned long cr0)
934 {
935         struct multicall_space mcs;
936 
937         this_cpu_write(xen_cr0_value, cr0);
938 
939         /* Only pay attention to cr0.TS; everything else is
940            ignored. */
941         mcs = xen_mc_entry(0);
942 
943         MULTI_fpu_taskswitch(mcs.mc, (cr0 & X86_CR0_TS) != 0);
944 
945         xen_mc_issue(PARAVIRT_LAZY_CPU);
946 }
947 
948 static void xen_write_cr4(unsigned long cr4)
949 {
950         cr4 &= ~X86_CR4_PGE;
951         cr4 &= ~X86_CR4_PSE;
952 
953         native_write_cr4(cr4);
954 }
955 
956 static int xen_write_msr_safe(unsigned int msr, unsigned low, unsigned high)
957 {
958         int ret;
959 
960         ret = 0;
961 
962         switch (msr) {
963 #ifdef CONFIG_X86_64
964                 unsigned which;
965                 u64 base;
966 
967         case MSR_FS_BASE:               which = SEGBASE_FS; goto set;
968         case MSR_KERNEL_GS_BASE:        which = SEGBASE_GS_USER; goto set;
969         case MSR_GS_BASE:               which = SEGBASE_GS_KERNEL; goto set;
970 
971         set:
972                 base = ((u64)high << 32) | low;
973                 if (HYPERVISOR_set_segment_base(which, base) != 0)
974                         ret = -EIO;
975                 break;
976 #endif
977 
978         case MSR_STAR:
979         case MSR_CSTAR:
980         case MSR_LSTAR:
981         case MSR_SYSCALL_MASK:
982         case MSR_IA32_SYSENTER_CS:
983         case MSR_IA32_SYSENTER_ESP:
984         case MSR_IA32_SYSENTER_EIP:
985                 /* Fast syscall setup is all done in hypercalls, so
986                    these are all ignored.  Stub them out here to stop
987                    Xen console noise. */
988                 break;
989 
990         case MSR_IA32_CR_PAT:
991                 if (smp_processor_id() == 0)
992                         xen_set_pat(((u64)high << 32) | low);
993                 break;
994 
995         default:
996                 ret = native_write_msr_safe(msr, low, high);
997         }
998 
999         return ret;
1000 }
1001 
1002 void xen_setup_shared_info(void)
1003 {
1004         if (!xen_feature(XENFEAT_auto_translated_physmap)) {
1005                 set_fixmap(FIX_PARAVIRT_BOOTMAP,
1006                            xen_start_info->shared_info);
1007 
1008                 HYPERVISOR_shared_info =
1009                         (struct shared_info *)fix_to_virt(FIX_PARAVIRT_BOOTMAP);
1010         } else
1011                 HYPERVISOR_shared_info =
1012                         (struct shared_info *)__va(xen_start_info->shared_info);
1013 
1014 #ifndef CONFIG_SMP
1015         /* In UP this is as good a place as any to set up shared info */
1016         xen_setup_vcpu_info_placement();
1017 #endif
1018 
1019         xen_setup_mfn_list_list();
1020 }
1021 
1022 /* This is called once we have the cpu_possible_mask */
1023 void xen_setup_vcpu_info_placement(void)
1024 {
1025         int cpu;
1026 
1027         for_each_possible_cpu(cpu)
1028                 xen_vcpu_setup(cpu);
1029 
1030         /* xen_vcpu_setup managed to place the vcpu_info within the
1031            percpu area for all cpus, so make use of it */
1032         if (have_vcpu_info_placement) {
1033                 pv_irq_ops.save_fl = __PV_IS_CALLEE_SAVE(xen_save_fl_direct);
1034                 pv_irq_ops.restore_fl = __PV_IS_CALLEE_SAVE(xen_restore_fl_direct);
1035                 pv_irq_ops.irq_disable = __PV_IS_CALLEE_SAVE(xen_irq_disable_direct);
1036                 pv_irq_ops.irq_enable = __PV_IS_CALLEE_SAVE(xen_irq_enable_direct);
1037                 pv_mmu_ops.read_cr2 = xen_read_cr2_direct;
1038         }
1039 }
1040 
1041 static unsigned xen_patch(u8 type, u16 clobbers, void *insnbuf,
1042                           unsigned long addr, unsigned len)
1043 {
1044         char *start, *end, *reloc;
1045         unsigned ret;
1046 
1047         start = end = reloc = NULL;
1048 
1049 #define SITE(op, x)                                                     \
1050         case PARAVIRT_PATCH(op.x):                                      \
1051         if (have_vcpu_info_placement) {                                 \
1052                 start = (char *)xen_##x##_direct;                       \
1053                 end = xen_##x##_direct_end;                             \
1054                 reloc = xen_##x##_direct_reloc;                         \
1055         }                                                               \
1056         goto patch_site
1057 
1058         switch (type) {
1059                 SITE(pv_irq_ops, irq_enable);
1060                 SITE(pv_irq_ops, irq_disable);
1061                 SITE(pv_irq_ops, save_fl);
1062                 SITE(pv_irq_ops, restore_fl);
1063 #undef SITE
1064 
1065         patch_site:
1066                 if (start == NULL || (end-start) > len)
1067                         goto default_patch;
1068 
1069                 ret = paravirt_patch_insns(insnbuf, len, start, end);
1070 
1071                 /* Note: because reloc is assigned from something that
1072                    appears to be an array, gcc assumes it's non-null,
1073                    but doesn't know its relationship with start and
1074                    end. */
1075                 if (reloc > start && reloc < end) {
1076                         int reloc_off = reloc - start;
1077                         long *relocp = (long *)(insnbuf + reloc_off);
1078                         long delta = start - (char *)addr;
1079 
1080                         *relocp += delta;
1081                 }
1082                 break;
1083 
1084         default_patch:
1085         default:
1086                 ret = paravirt_patch_default(type, clobbers, insnbuf,
1087                                              addr, len);
1088                 break;
1089         }
1090 
1091         return ret;
1092 }
1093 
1094 static const struct pv_info xen_info __initconst = {
1095         .paravirt_enabled = 1,
1096         .shared_kernel_pmd = 0,
1097 
1098 #ifdef CONFIG_X86_64
1099         .extra_user_64bit_cs = FLAT_USER_CS64,
1100 #endif
1101 
1102         .name = "Xen",
1103 };
1104 
1105 static const struct pv_init_ops xen_init_ops __initconst = {
1106         .patch = xen_patch,
1107 };
1108 
1109 static const struct pv_cpu_ops xen_cpu_ops __initconst = {
1110         .cpuid = xen_cpuid,
1111 
1112         .set_debugreg = xen_set_debugreg,
1113         .get_debugreg = xen_get_debugreg,
1114 
1115         .clts = xen_clts,
1116 
1117         .read_cr0 = xen_read_cr0,
1118         .write_cr0 = xen_write_cr0,
1119 
1120         .read_cr4 = native_read_cr4,
1121         .read_cr4_safe = native_read_cr4_safe,
1122         .write_cr4 = xen_write_cr4,
1123 
1124         .wbinvd = native_wbinvd,
1125 
1126         .read_msr = native_read_msr_safe,
1127         .rdmsr_regs = native_rdmsr_safe_regs,
1128         .write_msr = xen_write_msr_safe,
1129         .wrmsr_regs = native_wrmsr_safe_regs,
1130 
1131         .read_tsc = native_read_tsc,
1132         .read_pmc = native_read_pmc,
1133 
1134         .iret = xen_iret,
1135         .irq_enable_sysexit = xen_sysexit,
1136 #ifdef CONFIG_X86_64
1137         .usergs_sysret32 = xen_sysret32,
1138         .usergs_sysret64 = xen_sysret64,
1139 #endif
1140 
1141         .load_tr_desc = paravirt_nop,
1142         .set_ldt = xen_set_ldt,
1143         .load_gdt = xen_load_gdt,
1144         .load_idt = xen_load_idt,
1145         .load_tls = xen_load_tls,
1146 #ifdef CONFIG_X86_64
1147         .load_gs_index = xen_load_gs_index,
1148 #endif
1149 
1150         .alloc_ldt = xen_alloc_ldt,
1151         .free_ldt = xen_free_ldt,
1152 
1153         .store_gdt = native_store_gdt,
1154         .store_idt = native_store_idt,
1155         .store_tr = xen_store_tr,
1156 
1157         .write_ldt_entry = xen_write_ldt_entry,
1158         .write_gdt_entry = xen_write_gdt_entry,
1159         .write_idt_entry = xen_write_idt_entry,
1160         .load_sp0 = xen_load_sp0,
1161 
1162         .set_iopl_mask = xen_set_iopl_mask,
1163         .io_delay = xen_io_delay,
1164 
1165         /* Xen takes care of %gs when switching to usermode for us */
1166         .swapgs = paravirt_nop,
1167 
1168         .start_context_switch = paravirt_start_context_switch,
1169         .end_context_switch = xen_end_context_switch,
1170 };
1171 
1172 static const struct pv_apic_ops xen_apic_ops __initconst = {
1173 #ifdef CONFIG_X86_LOCAL_APIC
1174         .startup_ipi_hook = paravirt_nop,
1175 #endif
1176 };
1177 
1178 static void xen_reboot(int reason)
1179 {
1180         struct sched_shutdown r = { .reason = reason };
1181 
1182         if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r))
1183                 BUG();
1184 }
1185 
1186 static void xen_restart(char *msg)
1187 {
1188         xen_reboot(SHUTDOWN_reboot);
1189 }
1190 
1191 static void xen_emergency_restart(void)
1192 {
1193         xen_reboot(SHUTDOWN_reboot);
1194 }
1195 
1196 static void xen_machine_halt(void)
1197 {
1198         xen_reboot(SHUTDOWN_poweroff);
1199 }
1200 
1201 static void xen_machine_power_off(void)
1202 {
1203         if (pm_power_off)
1204                 pm_power_off();
1205         xen_reboot(SHUTDOWN_poweroff);
1206 }
1207 
1208 static void xen_crash_shutdown(struct pt_regs *regs)
1209 {
1210         xen_reboot(SHUTDOWN_crash);
1211 }
1212 
1213 static int
1214 xen_panic_event(struct notifier_block *this, unsigned long event, void *ptr)
1215 {
1216         xen_reboot(SHUTDOWN_crash);
1217         return NOTIFY_DONE;
1218 }
1219 
1220 static struct notifier_block xen_panic_block = {
1221         .notifier_call= xen_panic_event,
1222 };
1223 
1224 int xen_panic_handler_init(void)
1225 {
1226         atomic_notifier_chain_register(&panic_notifier_list, &xen_panic_block);
1227         return 0;
1228 }
1229 
1230 static const struct machine_ops xen_machine_ops __initconst = {
1231         .restart = xen_restart,
1232         .halt = xen_machine_halt,
1233         .power_off = xen_machine_power_off,
1234         .shutdown = xen_machine_halt,
1235         .crash_shutdown = xen_crash_shutdown,
1236         .emergency_restart = xen_emergency_restart,
1237 };
1238 
1239 /*
1240  * Set up the GDT and segment registers for -fstack-protector.  Until
1241  * we do this, we have to be careful not to call any stack-protected
1242  * function, which is most of the kernel.
1243  */
1244 static void __init xen_setup_stackprotector(void)
1245 {
1246         pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry_boot;
1247         pv_cpu_ops.load_gdt = xen_load_gdt_boot;
1248 
1249         setup_stack_canary_segment(0);
1250         switch_to_new_gdt(0);
1251 
1252         pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry;
1253         pv_cpu_ops.load_gdt = xen_load_gdt;
1254 }
1255 
1256 /* First C function to be called on Xen boot */
1257 asmlinkage void __init xen_start_kernel(void)
1258 {
1259         struct physdev_set_iopl set_iopl;
1260         int rc;
1261         pgd_t *pgd;
1262 
1263         if (!xen_start_info)
1264                 return;
1265 
1266         xen_domain_type = XEN_PV_DOMAIN;
1267 
1268         xen_setup_machphys_mapping();
1269 
1270         /* Install Xen paravirt ops */
1271         pv_info = xen_info;
1272         pv_init_ops = xen_init_ops;
1273         pv_cpu_ops = xen_cpu_ops;
1274         pv_apic_ops = xen_apic_ops;
1275 
1276         x86_init.resources.memory_setup = xen_memory_setup;
1277         x86_init.oem.arch_setup = xen_arch_setup;
1278         x86_init.oem.banner = xen_banner;
1279 
1280         xen_init_time_ops();
1281 
1282         /*
1283          * Set up some pagetable state before starting to set any ptes.
1284          */
1285 
1286         xen_init_mmu_ops();
1287 
1288         /* Prevent unwanted bits from being set in PTEs. */
1289         __supported_pte_mask &= ~_PAGE_GLOBAL;
1290 #if 0
1291         if (!xen_initial_domain())
1292 #endif
1293                 __supported_pte_mask &= ~(_PAGE_PWT | _PAGE_PCD);
1294 
1295         __supported_pte_mask |= _PAGE_IOMAP;
1296 
1297         /*
1298          * Prevent page tables from being allocated in highmem, even
1299          * if CONFIG_HIGHPTE is enabled.
1300          */
1301         __userpte_alloc_gfp &= ~__GFP_HIGHMEM;
1302 
1303         /* Work out if we support NX */
1304         x86_configure_nx();
1305 
1306         xen_setup_features();
1307 
1308         /* Get mfn list */
1309         if (!xen_feature(XENFEAT_auto_translated_physmap))
1310                 xen_build_dynamic_phys_to_machine();
1311 
1312         /*
1313          * Set up kernel GDT and segment registers, mainly so that
1314          * -fstack-protector code can be executed.
1315          */
1316         xen_setup_stackprotector();
1317 
1318         xen_init_irq_ops();
1319         xen_init_cpuid_mask();
1320 
1321 #ifdef CONFIG_X86_LOCAL_APIC
1322         /*
1323          * set up the basic apic ops.
1324          */
1325         set_xen_basic_apic_ops();
1326 #endif
1327 
1328         if (xen_feature(XENFEAT_mmu_pt_update_preserve_ad)) {
1329                 pv_mmu_ops.ptep_modify_prot_start = xen_ptep_modify_prot_start;
1330                 pv_mmu_ops.ptep_modify_prot_commit = xen_ptep_modify_prot_commit;
1331         }
1332 
1333         machine_ops = xen_machine_ops;
1334 
1335         /*
1336          * The only reliable way to retain the initial address of the
1337          * percpu gdt_page is to remember it here, so we can go and
1338          * mark it RW later, when the initial percpu area is freed.
1339          */
1340         xen_initial_gdt = &per_cpu(gdt_page, 0);
1341 
1342         xen_smp_init();
1343 
1344 #ifdef CONFIG_ACPI_NUMA
1345         /*
1346          * The pages we from Xen are not related to machine pages, so
1347          * any NUMA information the kernel tries to get from ACPI will
1348          * be meaningless.  Prevent it from trying.
1349          */
1350         acpi_numa = -1;
1351 #endif
1352 
1353         pgd = (pgd_t *)xen_start_info->pt_base;
1354 
1355         /* Don't do the full vcpu_info placement stuff until we have a
1356            possible map and a non-dummy shared_info. */
1357         per_cpu(xen_vcpu, 0) = &HYPERVISOR_shared_info->vcpu_info[0];
1358 
1359         local_irq_disable();
1360         early_boot_irqs_disabled = true;
1361 
1362         xen_raw_console_write("mapping kernel into physical memory\n");
1363         pgd = xen_setup_kernel_pagetable(pgd, xen_start_info->nr_pages);
1364 
1365         /* Allocate and initialize top and mid mfn levels for p2m structure */
1366         xen_build_mfn_list_list();
1367 
1368         /* keep using Xen gdt for now; no urgent need to change it */
1369 
1370 #ifdef CONFIG_X86_32
1371         pv_info.kernel_rpl = 1;
1372         if (xen_feature(XENFEAT_supervisor_mode_kernel))
1373                 pv_info.kernel_rpl = 0;
1374 #else
1375         pv_info.kernel_rpl = 0;
1376 #endif
1377         /* set the limit of our address space */
1378         xen_reserve_top();
1379 
1380         /* We used to do this in xen_arch_setup, but that is too late on AMD
1381          * were early_cpu_init (run before ->arch_setup()) calls early_amd_init
1382          * which pokes 0xcf8 port.
1383          */
1384         set_iopl.iopl = 1;
1385         rc = HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
1386         if (rc != 0)
1387                 xen_raw_printk("physdev_op failed %d\n", rc);
1388 
1389 #ifdef CONFIG_X86_32
1390         /* set up basic CPUID stuff */
1391         cpu_detect(&new_cpu_data);
1392         new_cpu_data.hard_math = 1;
1393         new_cpu_data.wp_works_ok = 1;
1394         new_cpu_data.x86_capability[0] = cpuid_edx(1);
1395 #endif
1396 
1397         /* Poke various useful things into boot_params */
1398         boot_params.hdr.type_of_loader = (9 << 4) | 0;
1399         boot_params.hdr.ramdisk_image = xen_start_info->mod_start
1400                 ? __pa(xen_start_info->mod_start) : 0;
1401         boot_params.hdr.ramdisk_size = xen_start_info->mod_len;
1402         boot_params.hdr.cmd_line_ptr = __pa(xen_start_info->cmd_line);
1403 
1404         if (!xen_initial_domain()) {
1405                 add_preferred_console("xenboot", 0, NULL);
1406                 add_preferred_console("tty", 0, NULL);
1407                 add_preferred_console("hvc", 0, NULL);
1408                 if (pci_xen)
1409                         x86_init.pci.arch_init = pci_xen_init;
1410         } else {
1411                 const struct dom0_vga_console_info *info =
1412                         (void *)((char *)xen_start_info +
1413                                  xen_start_info->console.dom0.info_off);
1414 
1415                 xen_init_vga(info, xen_start_info->console.dom0.info_size);
1416                 xen_start_info->console.domU.mfn = 0;
1417                 xen_start_info->console.domU.evtchn = 0;
1418 
1419                 xen_init_apic();
1420 
1421                 /* Make sure ACS will be enabled */
1422                 pci_request_acs();
1423 
1424                 xen_acpi_sleep_register();
1425 
1426                 /* Avoid searching for BIOS MP tables */
1427                 x86_init.mpparse.find_smp_config = x86_init_noop;
1428                 x86_init.mpparse.get_smp_config = x86_init_uint_noop;
1429         }
1430 #ifdef CONFIG_PCI
1431         /* PCI BIOS service won't work from a PV guest. */
1432         pci_probe &= ~PCI_PROBE_BIOS;
1433 #endif
1434         xen_raw_console_write("about to get started...\n");
1435 
1436         xen_setup_runstate_info(0);
1437 
1438         /* Start the world */
1439 #ifdef CONFIG_X86_32
1440         i386_start_kernel();
1441 #else
1442         x86_64_start_reservations((char *)__pa_symbol(&boot_params));
1443 #endif
1444 }
1445 
1446 static int init_hvm_pv_info(int *major, int *minor)
1447 {
1448         uint32_t eax, ebx, ecx, edx, pages, msr, base;
1449         u64 pfn;
1450 
1451         base = xen_cpuid_base();
1452         cpuid(base + 1, &eax, &ebx, &ecx, &edx);
1453 
1454         *major = eax >> 16;
1455         *minor = eax & 0xffff;
1456         printk(KERN_INFO "Xen version %d.%d.\n", *major, *minor);
1457 
1458         cpuid(base + 2, &pages, &msr, &ecx, &edx);
1459 
1460         pfn = __pa(hypercall_page);
1461         wrmsr_safe(msr, (u32)pfn, (u32)(pfn >> 32));
1462 
1463         xen_setup_features();
1464 
1465         pv_info.name = "Xen HVM";
1466 
1467         xen_domain_type = XEN_HVM_DOMAIN;
1468 
1469         return 0;
1470 }
1471 
1472 void __ref xen_hvm_init_shared_info(void)
1473 {
1474         int cpu;
1475         struct xen_add_to_physmap xatp;
1476         static struct shared_info *shared_info_page = 0;
1477 
1478         if (!shared_info_page)
1479                 shared_info_page = (struct shared_info *)
1480                         extend_brk(PAGE_SIZE, PAGE_SIZE);
1481         xatp.domid = DOMID_SELF;
1482         xatp.idx = 0;
1483         xatp.space = XENMAPSPACE_shared_info;
1484         xatp.gpfn = __pa(shared_info_page) >> PAGE_SHIFT;
1485         if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp))
1486                 BUG();
1487 
1488         HYPERVISOR_shared_info = (struct shared_info *)shared_info_page;
1489 
1490         /* xen_vcpu is a pointer to the vcpu_info struct in the shared_info
1491          * page, we use it in the event channel upcall and in some pvclock
1492          * related functions. We don't need the vcpu_info placement
1493          * optimizations because we don't use any pv_mmu or pv_irq op on
1494          * HVM.
1495          * When xen_hvm_init_shared_info is run at boot time only vcpu 0 is
1496          * online but xen_hvm_init_shared_info is run at resume time too and
1497          * in that case multiple vcpus might be online. */
1498         for_each_online_cpu(cpu) {
1499                 per_cpu(xen_vcpu, cpu) = &HYPERVISOR_shared_info->vcpu_info[cpu];
1500         }
1501 }
1502 
1503 #ifdef CONFIG_XEN_PVHVM
1504 static int __cpuinit xen_hvm_cpu_notify(struct notifier_block *self,
1505                                     unsigned long action, void *hcpu)
1506 {
1507         int cpu = (long)hcpu;
1508         switch (action) {
1509         case CPU_UP_PREPARE:
1510                 xen_vcpu_setup(cpu);
1511                 if (xen_have_vector_callback)
1512                         xen_init_lock_cpu(cpu);
1513                 break;
1514         default:
1515                 break;
1516         }
1517         return NOTIFY_OK;
1518 }
1519 
1520 static struct notifier_block xen_hvm_cpu_notifier __cpuinitdata = {
1521         .notifier_call  = xen_hvm_cpu_notify,
1522 };
1523 
1524 static void __init xen_hvm_guest_init(void)
1525 {
1526         int r;
1527         int major, minor;
1528 
1529         r = init_hvm_pv_info(&major, &minor);
1530         if (r < 0)
1531                 return;
1532 
1533         xen_hvm_init_shared_info();
1534 
1535         if (xen_feature(XENFEAT_hvm_callback_vector))
1536                 xen_have_vector_callback = 1;
1537         xen_hvm_smp_init();
1538         register_cpu_notifier(&xen_hvm_cpu_notifier);
1539         xen_unplug_emulated_devices();
1540         x86_init.irqs.intr_init = xen_init_IRQ;
1541         xen_hvm_init_time_ops();
1542         xen_hvm_init_mmu_ops();
1543 }
1544 
1545 static bool __init xen_hvm_platform(void)
1546 {
1547         if (xen_pv_domain())
1548                 return false;
1549 
1550         if (!xen_cpuid_base())
1551                 return false;
1552 
1553         return true;
1554 }
1555 
1556 bool xen_hvm_need_lapic(void)
1557 {
1558         if (xen_pv_domain())
1559                 return false;
1560         if (!xen_hvm_domain())
1561                 return false;
1562         if (xen_feature(XENFEAT_hvm_pirqs) && xen_have_vector_callback)
1563                 return false;
1564         return true;
1565 }
1566 EXPORT_SYMBOL_GPL(xen_hvm_need_lapic);
1567 
1568 const struct hypervisor_x86 x86_hyper_xen_hvm __refconst = {
1569         .name                   = "Xen HVM",
1570         .detect                 = xen_hvm_platform,
1571         .init_platform          = xen_hvm_guest_init,
1572 };
1573 EXPORT_SYMBOL(x86_hyper_xen_hvm);
1574 #endif
1575 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp