~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/xen/enlighten.c

Version: ~ [ linux-5.14-rc3 ] ~ [ linux-5.13.5 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.53 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.135 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.198 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.240 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.276 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.276 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Core of Xen paravirt_ops implementation.
  3  *
  4  * This file contains the xen_paravirt_ops structure itself, and the
  5  * implementations for:
  6  * - privileged instructions
  7  * - interrupt flags
  8  * - segment operations
  9  * - booting and setup
 10  *
 11  * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
 12  */
 13 
 14 #include <linux/kernel.h>
 15 #include <linux/init.h>
 16 #include <linux/smp.h>
 17 #include <linux/preempt.h>
 18 #include <linux/hardirq.h>
 19 #include <linux/percpu.h>
 20 #include <linux/delay.h>
 21 #include <linux/start_kernel.h>
 22 #include <linux/sched.h>
 23 #include <linux/kprobes.h>
 24 #include <linux/bootmem.h>
 25 #include <linux/module.h>
 26 #include <linux/mm.h>
 27 #include <linux/page-flags.h>
 28 #include <linux/highmem.h>
 29 #include <linux/console.h>
 30 
 31 #include <xen/interface/xen.h>
 32 #include <xen/interface/version.h>
 33 #include <xen/interface/physdev.h>
 34 #include <xen/interface/vcpu.h>
 35 #include <xen/features.h>
 36 #include <xen/page.h>
 37 #include <xen/hvc-console.h>
 38 
 39 #include <asm/paravirt.h>
 40 #include <asm/apic.h>
 41 #include <asm/page.h>
 42 #include <asm/xen/hypercall.h>
 43 #include <asm/xen/hypervisor.h>
 44 #include <asm/fixmap.h>
 45 #include <asm/processor.h>
 46 #include <asm/proto.h>
 47 #include <asm/msr-index.h>
 48 #include <asm/traps.h>
 49 #include <asm/setup.h>
 50 #include <asm/desc.h>
 51 #include <asm/pgalloc.h>
 52 #include <asm/pgtable.h>
 53 #include <asm/tlbflush.h>
 54 #include <asm/reboot.h>
 55 #include <asm/stackprotector.h>
 56 
 57 #include "xen-ops.h"
 58 #include "mmu.h"
 59 #include "multicalls.h"
 60 
 61 EXPORT_SYMBOL_GPL(hypercall_page);
 62 
 63 DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu);
 64 DEFINE_PER_CPU(struct vcpu_info, xen_vcpu_info);
 65 
 66 enum xen_domain_type xen_domain_type = XEN_NATIVE;
 67 EXPORT_SYMBOL_GPL(xen_domain_type);
 68 
 69 struct start_info *xen_start_info;
 70 EXPORT_SYMBOL_GPL(xen_start_info);
 71 
 72 struct shared_info xen_dummy_shared_info;
 73 
 74 void *xen_initial_gdt;
 75 
 76 /*
 77  * Point at some empty memory to start with. We map the real shared_info
 78  * page as soon as fixmap is up and running.
 79  */
 80 struct shared_info *HYPERVISOR_shared_info = (void *)&xen_dummy_shared_info;
 81 
 82 /*
 83  * Flag to determine whether vcpu info placement is available on all
 84  * VCPUs.  We assume it is to start with, and then set it to zero on
 85  * the first failure.  This is because it can succeed on some VCPUs
 86  * and not others, since it can involve hypervisor memory allocation,
 87  * or because the guest failed to guarantee all the appropriate
 88  * constraints on all VCPUs (ie buffer can't cross a page boundary).
 89  *
 90  * Note that any particular CPU may be using a placed vcpu structure,
 91  * but we can only optimise if the all are.
 92  *
 93  * 0: not available, 1: available
 94  */
 95 static int have_vcpu_info_placement = 1;
 96 
 97 static void xen_vcpu_setup(int cpu)
 98 {
 99         struct vcpu_register_vcpu_info info;
100         int err;
101         struct vcpu_info *vcpup;
102 
103         BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info);
104         per_cpu(xen_vcpu, cpu) = &HYPERVISOR_shared_info->vcpu_info[cpu];
105 
106         if (!have_vcpu_info_placement)
107                 return;         /* already tested, not available */
108 
109         vcpup = &per_cpu(xen_vcpu_info, cpu);
110 
111         info.mfn = arbitrary_virt_to_mfn(vcpup);
112         info.offset = offset_in_page(vcpup);
113 
114         printk(KERN_DEBUG "trying to map vcpu_info %d at %p, mfn %llx, offset %d\n",
115                cpu, vcpup, info.mfn, info.offset);
116 
117         /* Check to see if the hypervisor will put the vcpu_info
118            structure where we want it, which allows direct access via
119            a percpu-variable. */
120         err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, cpu, &info);
121 
122         if (err) {
123                 printk(KERN_DEBUG "register_vcpu_info failed: err=%d\n", err);
124                 have_vcpu_info_placement = 0;
125         } else {
126                 /* This cpu is using the registered vcpu info, even if
127                    later ones fail to. */
128                 per_cpu(xen_vcpu, cpu) = vcpup;
129 
130                 printk(KERN_DEBUG "cpu %d using vcpu_info at %p\n",
131                        cpu, vcpup);
132         }
133 }
134 
135 /*
136  * On restore, set the vcpu placement up again.
137  * If it fails, then we're in a bad state, since
138  * we can't back out from using it...
139  */
140 void xen_vcpu_restore(void)
141 {
142         int cpu;
143 
144         for_each_online_cpu(cpu) {
145                 bool other_cpu = (cpu != smp_processor_id());
146 
147                 if (other_cpu &&
148                     HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL))
149                         BUG();
150 
151                 xen_setup_runstate_info(cpu);
152 
153                 if (have_vcpu_info_placement)
154                         xen_vcpu_setup(cpu);
155 
156                 if (other_cpu &&
157                     HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL))
158                         BUG();
159         }
160 }
161 
162 static void __init xen_banner(void)
163 {
164         unsigned version = HYPERVISOR_xen_version(XENVER_version, NULL);
165         struct xen_extraversion extra;
166         HYPERVISOR_xen_version(XENVER_extraversion, &extra);
167 
168         printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
169                pv_info.name);
170         printk(KERN_INFO "Xen version: %d.%d%s%s\n",
171                version >> 16, version & 0xffff, extra.extraversion,
172                xen_feature(XENFEAT_mmu_pt_update_preserve_ad) ? " (preserve-AD)" : "");
173 }
174 
175 static __read_mostly unsigned int cpuid_leaf1_edx_mask = ~0;
176 static __read_mostly unsigned int cpuid_leaf1_ecx_mask = ~0;
177 
178 static void xen_cpuid(unsigned int *ax, unsigned int *bx,
179                       unsigned int *cx, unsigned int *dx)
180 {
181         unsigned maskebx = ~0;
182         unsigned maskecx = ~0;
183         unsigned maskedx = ~0;
184 
185         /*
186          * Mask out inconvenient features, to try and disable as many
187          * unsupported kernel subsystems as possible.
188          */
189         switch (*ax) {
190         case 1:
191                 maskecx = cpuid_leaf1_ecx_mask;
192                 maskedx = cpuid_leaf1_edx_mask;
193                 break;
194 
195         case 0xb:
196                 /* Suppress extended topology stuff */
197                 maskebx = 0;
198                 break;
199         }
200 
201         asm(XEN_EMULATE_PREFIX "cpuid"
202                 : "=a" (*ax),
203                   "=b" (*bx),
204                   "=c" (*cx),
205                   "=d" (*dx)
206                 : "" (*ax), "2" (*cx));
207 
208         *bx &= maskebx;
209         *cx &= maskecx;
210         *dx &= maskedx;
211 }
212 
213 static __init void xen_init_cpuid_mask(void)
214 {
215         unsigned int ax, bx, cx, dx;
216 
217         cpuid_leaf1_edx_mask =
218                 ~((1 << X86_FEATURE_MCE)  |  /* disable MCE */
219                   (1 << X86_FEATURE_MCA)  |  /* disable MCA */
220                   (1 << X86_FEATURE_ACC));   /* thermal monitoring */
221 
222         if (!xen_initial_domain())
223                 cpuid_leaf1_edx_mask &=
224                         ~((1 << X86_FEATURE_APIC) |  /* disable local APIC */
225                           (1 << X86_FEATURE_ACPI));  /* disable ACPI */
226 
227         ax = 1;
228         cx = 0;
229         xen_cpuid(&ax, &bx, &cx, &dx);
230 
231         /* cpuid claims we support xsave; try enabling it to see what happens */
232         if (cx & (1 << (X86_FEATURE_XSAVE % 32))) {
233                 unsigned long cr4;
234 
235                 set_in_cr4(X86_CR4_OSXSAVE);
236                 
237                 cr4 = read_cr4();
238 
239                 if ((cr4 & X86_CR4_OSXSAVE) == 0)
240                         cpuid_leaf1_ecx_mask &= ~(1 << (X86_FEATURE_XSAVE % 32));
241 
242                 clear_in_cr4(X86_CR4_OSXSAVE);
243         }
244 }
245 
246 static void xen_set_debugreg(int reg, unsigned long val)
247 {
248         HYPERVISOR_set_debugreg(reg, val);
249 }
250 
251 static unsigned long xen_get_debugreg(int reg)
252 {
253         return HYPERVISOR_get_debugreg(reg);
254 }
255 
256 static void xen_end_context_switch(struct task_struct *next)
257 {
258         xen_mc_flush();
259         paravirt_end_context_switch(next);
260 }
261 
262 static unsigned long xen_store_tr(void)
263 {
264         return 0;
265 }
266 
267 /*
268  * Set the page permissions for a particular virtual address.  If the
269  * address is a vmalloc mapping (or other non-linear mapping), then
270  * find the linear mapping of the page and also set its protections to
271  * match.
272  */
273 static void set_aliased_prot(void *v, pgprot_t prot)
274 {
275         int level;
276         pte_t *ptep;
277         pte_t pte;
278         unsigned long pfn;
279         struct page *page;
280         unsigned char dummy;
281 
282         ptep = lookup_address((unsigned long)v, &level);
283         BUG_ON(ptep == NULL);
284 
285         pfn = pte_pfn(*ptep);
286         page = pfn_to_page(pfn);
287 
288         pte = pfn_pte(pfn, prot);
289 
290         /*
291          * Careful: update_va_mapping() will fail if the virtual address
292          * we're poking isn't populated in the page tables.  We don't
293          * need to worry about the direct map (that's always in the page
294          * tables), but we need to be careful about vmap space.  In
295          * particular, the top level page table can lazily propagate
296          * entries between processes, so if we've switched mms since we
297          * vmapped the target in the first place, we might not have the
298          * top-level page table entry populated.
299          *
300          * We disable preemption because we want the same mm active when
301          * we probe the target and when we issue the hypercall.  We'll
302          * have the same nominal mm, but if we're a kernel thread, lazy
303          * mm dropping could change our pgd.
304          *
305          * Out of an abundance of caution, this uses __get_user() to fault
306          * in the target address just in case there's some obscure case
307          * in which the target address isn't readable.
308          */
309 
310         preempt_disable();
311 
312         pagefault_disable();    /* Avoid warnings due to being atomic. */
313         __get_user(dummy, (unsigned char __user __force *)v);
314         pagefault_enable();
315 
316         if (HYPERVISOR_update_va_mapping((unsigned long)v, pte, 0))
317                 BUG();
318 
319         if (!PageHighMem(page)) {
320                 void *av = __va(PFN_PHYS(pfn));
321 
322                 if (av != v)
323                         if (HYPERVISOR_update_va_mapping((unsigned long)av, pte, 0))
324                                 BUG();
325         } else
326                 kmap_flush_unused();
327 
328         preempt_enable();
329 }
330 
331 static void xen_alloc_ldt(struct desc_struct *ldt, unsigned entries)
332 {
333         const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE;
334         int i;
335 
336         /*
337          * We need to mark the all aliases of the LDT pages RO.  We
338          * don't need to call vm_flush_aliases(), though, since that's
339          * only responsible for flushing aliases out the TLBs, not the
340          * page tables, and Xen will flush the TLB for us if needed.
341          *
342          * To avoid confusing future readers: none of this is necessary
343          * to load the LDT.  The hypervisor only checks this when the
344          * LDT is faulted in due to subsequent descriptor access.
345          */
346 
347         for(i = 0; i < entries; i += entries_per_page)
348                 set_aliased_prot(ldt + i, PAGE_KERNEL_RO);
349 }
350 
351 static void xen_free_ldt(struct desc_struct *ldt, unsigned entries)
352 {
353         const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE;
354         int i;
355 
356         for(i = 0; i < entries; i += entries_per_page)
357                 set_aliased_prot(ldt + i, PAGE_KERNEL);
358 }
359 
360 static void xen_set_ldt(const void *addr, unsigned entries)
361 {
362         struct mmuext_op *op;
363         struct multicall_space mcs = xen_mc_entry(sizeof(*op));
364 
365         op = mcs.args;
366         op->cmd = MMUEXT_SET_LDT;
367         op->arg1.linear_addr = (unsigned long)addr;
368         op->arg2.nr_ents = entries;
369 
370         MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
371 
372         xen_mc_issue(PARAVIRT_LAZY_CPU);
373 }
374 
375 static void xen_load_gdt(const struct desc_ptr *dtr)
376 {
377         unsigned long va = dtr->address;
378         unsigned int size = dtr->size + 1;
379         unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
380         unsigned long frames[pages];
381         int f;
382 
383         /*
384          * A GDT can be up to 64k in size, which corresponds to 8192
385          * 8-byte entries, or 16 4k pages..
386          */
387 
388         BUG_ON(size > 65536);
389         BUG_ON(va & ~PAGE_MASK);
390 
391         for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
392                 int level;
393                 pte_t *ptep;
394                 unsigned long pfn, mfn;
395                 void *virt;
396 
397                 /*
398                  * The GDT is per-cpu and is in the percpu data area.
399                  * That can be virtually mapped, so we need to do a
400                  * page-walk to get the underlying MFN for the
401                  * hypercall.  The page can also be in the kernel's
402                  * linear range, so we need to RO that mapping too.
403                  */
404                 ptep = lookup_address(va, &level);
405                 BUG_ON(ptep == NULL);
406 
407                 pfn = pte_pfn(*ptep);
408                 mfn = pfn_to_mfn(pfn);
409                 virt = __va(PFN_PHYS(pfn));
410 
411                 frames[f] = mfn;
412 
413                 make_lowmem_page_readonly((void *)va);
414                 make_lowmem_page_readonly(virt);
415         }
416 
417         if (HYPERVISOR_set_gdt(frames, size / sizeof(struct desc_struct)))
418                 BUG();
419 }
420 
421 /*
422  * load_gdt for early boot, when the gdt is only mapped once
423  */
424 static __init void xen_load_gdt_boot(const struct desc_ptr *dtr)
425 {
426         unsigned long va = dtr->address;
427         unsigned int size = dtr->size + 1;
428         unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
429         unsigned long frames[pages];
430         int f;
431 
432         /*
433          * A GDT can be up to 64k in size, which corresponds to 8192
434          * 8-byte entries, or 16 4k pages..
435          */
436 
437         BUG_ON(size > 65536);
438         BUG_ON(va & ~PAGE_MASK);
439 
440         for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
441                 pte_t pte;
442                 unsigned long pfn, mfn;
443 
444                 pfn = virt_to_pfn(va);
445                 mfn = pfn_to_mfn(pfn);
446 
447                 pte = pfn_pte(pfn, PAGE_KERNEL_RO);
448 
449                 if (HYPERVISOR_update_va_mapping((unsigned long)va, pte, 0))
450                         BUG();
451 
452                 frames[f] = mfn;
453         }
454 
455         if (HYPERVISOR_set_gdt(frames, size / sizeof(struct desc_struct)))
456                 BUG();
457 }
458 
459 static void load_TLS_descriptor(struct thread_struct *t,
460                                 unsigned int cpu, unsigned int i)
461 {
462         struct desc_struct *gdt = get_cpu_gdt_table(cpu);
463         xmaddr_t maddr = arbitrary_virt_to_machine(&gdt[GDT_ENTRY_TLS_MIN+i]);
464         struct multicall_space mc = __xen_mc_entry(0);
465 
466         MULTI_update_descriptor(mc.mc, maddr.maddr, t->tls_array[i]);
467 }
468 
469 static void xen_load_tls(struct thread_struct *t, unsigned int cpu)
470 {
471         /*
472          * XXX sleazy hack: If we're being called in a lazy-cpu zone
473          * and lazy gs handling is enabled, it means we're in a
474          * context switch, and %gs has just been saved.  This means we
475          * can zero it out to prevent faults on exit from the
476          * hypervisor if the next process has no %gs.  Either way, it
477          * has been saved, and the new value will get loaded properly.
478          * This will go away as soon as Xen has been modified to not
479          * save/restore %gs for normal hypercalls.
480          *
481          * On x86_64, this hack is not used for %gs, because gs points
482          * to KERNEL_GS_BASE (and uses it for PDA references), so we
483          * must not zero %gs on x86_64
484          *
485          * For x86_64, we need to zero %fs, otherwise we may get an
486          * exception between the new %fs descriptor being loaded and
487          * %fs being effectively cleared at __switch_to().
488          */
489         if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_CPU) {
490 #ifdef CONFIG_X86_32
491                 lazy_load_gs(0);
492 #else
493                 loadsegment(fs, 0);
494 #endif
495         }
496 
497         xen_mc_batch();
498 
499         load_TLS_descriptor(t, cpu, 0);
500         load_TLS_descriptor(t, cpu, 1);
501         load_TLS_descriptor(t, cpu, 2);
502 
503         xen_mc_issue(PARAVIRT_LAZY_CPU);
504 }
505 
506 #ifdef CONFIG_X86_64
507 static void xen_load_gs_index(unsigned int idx)
508 {
509         if (HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, idx))
510                 BUG();
511 }
512 #endif
513 
514 static void xen_write_ldt_entry(struct desc_struct *dt, int entrynum,
515                                 const void *ptr)
516 {
517         xmaddr_t mach_lp = arbitrary_virt_to_machine(&dt[entrynum]);
518         u64 entry = *(u64 *)ptr;
519 
520         preempt_disable();
521 
522         xen_mc_flush();
523         if (HYPERVISOR_update_descriptor(mach_lp.maddr, entry))
524                 BUG();
525 
526         preempt_enable();
527 }
528 
529 static int cvt_gate_to_trap(int vector, const gate_desc *val,
530                             struct trap_info *info)
531 {
532         unsigned long addr;
533 
534         if (val->type != GATE_TRAP && val->type != GATE_INTERRUPT)
535                 return 0;
536 
537         info->vector = vector;
538 
539         addr = gate_offset(*val);
540 #ifdef CONFIG_X86_64
541         /*
542          * Look for known traps using IST, and substitute them
543          * appropriately.  The debugger ones are the only ones we care
544          * about.  Xen will handle faults like double_fault and
545          * machine_check, so we should never see them.  Warn if
546          * there's an unexpected IST-using fault handler.
547          */
548         if (addr == (unsigned long)debug)
549                 addr = (unsigned long)xen_debug;
550         else if (addr == (unsigned long)int3)
551                 addr = (unsigned long)xen_int3;
552         else if (addr == (unsigned long)stack_segment)
553                 addr = (unsigned long)xen_stack_segment;
554         else if (addr == (unsigned long)double_fault ||
555                  addr == (unsigned long)nmi) {
556                 /* Don't need to handle these */
557                 return 0;
558 #ifdef CONFIG_X86_MCE
559         } else if (addr == (unsigned long)machine_check) {
560                 return 0;
561 #endif
562         } else {
563                 /* Some other trap using IST? */
564                 if (WARN_ON(val->ist != 0))
565                         return 0;
566         }
567 #endif  /* CONFIG_X86_64 */
568         info->address = addr;
569 
570         info->cs = gate_segment(*val);
571         info->flags = val->dpl;
572         /* interrupt gates clear IF */
573         if (val->type == GATE_INTERRUPT)
574                 info->flags |= 1 << 2;
575 
576         return 1;
577 }
578 
579 /* Locations of each CPU's IDT */
580 static DEFINE_PER_CPU(struct desc_ptr, idt_desc);
581 
582 /* Set an IDT entry.  If the entry is part of the current IDT, then
583    also update Xen. */
584 static void xen_write_idt_entry(gate_desc *dt, int entrynum, const gate_desc *g)
585 {
586         unsigned long p = (unsigned long)&dt[entrynum];
587         unsigned long start, end;
588 
589         preempt_disable();
590 
591         start = __get_cpu_var(idt_desc).address;
592         end = start + __get_cpu_var(idt_desc).size + 1;
593 
594         xen_mc_flush();
595 
596         native_write_idt_entry(dt, entrynum, g);
597 
598         if (p >= start && (p + 8) <= end) {
599                 struct trap_info info[2];
600 
601                 info[1].address = 0;
602 
603                 if (cvt_gate_to_trap(entrynum, g, &info[0]))
604                         if (HYPERVISOR_set_trap_table(info))
605                                 BUG();
606         }
607 
608         preempt_enable();
609 }
610 
611 static void xen_convert_trap_info(const struct desc_ptr *desc,
612                                   struct trap_info *traps)
613 {
614         unsigned in, out, count;
615 
616         count = (desc->size+1) / sizeof(gate_desc);
617         BUG_ON(count > 256);
618 
619         for (in = out = 0; in < count; in++) {
620                 gate_desc *entry = (gate_desc*)(desc->address) + in;
621 
622                 if (cvt_gate_to_trap(in, entry, &traps[out]))
623                         out++;
624         }
625         traps[out].address = 0;
626 }
627 
628 void xen_copy_trap_info(struct trap_info *traps)
629 {
630         const struct desc_ptr *desc = &__get_cpu_var(idt_desc);
631 
632         xen_convert_trap_info(desc, traps);
633 }
634 
635 /* Load a new IDT into Xen.  In principle this can be per-CPU, so we
636    hold a spinlock to protect the static traps[] array (static because
637    it avoids allocation, and saves stack space). */
638 static void xen_load_idt(const struct desc_ptr *desc)
639 {
640         static DEFINE_SPINLOCK(lock);
641         static struct trap_info traps[257];
642 
643         spin_lock(&lock);
644 
645         __get_cpu_var(idt_desc) = *desc;
646 
647         xen_convert_trap_info(desc, traps);
648 
649         xen_mc_flush();
650         if (HYPERVISOR_set_trap_table(traps))
651                 BUG();
652 
653         spin_unlock(&lock);
654 }
655 
656 /* Write a GDT descriptor entry.  Ignore LDT descriptors, since
657    they're handled differently. */
658 static void xen_write_gdt_entry(struct desc_struct *dt, int entry,
659                                 const void *desc, int type)
660 {
661         preempt_disable();
662 
663         switch (type) {
664         case DESC_LDT:
665         case DESC_TSS:
666                 /* ignore */
667                 break;
668 
669         default: {
670                 xmaddr_t maddr = arbitrary_virt_to_machine(&dt[entry]);
671 
672                 xen_mc_flush();
673                 if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc))
674                         BUG();
675         }
676 
677         }
678 
679         preempt_enable();
680 }
681 
682 /*
683  * Version of write_gdt_entry for use at early boot-time needed to
684  * update an entry as simply as possible.
685  */
686 static __init void xen_write_gdt_entry_boot(struct desc_struct *dt, int entry,
687                                             const void *desc, int type)
688 {
689         switch (type) {
690         case DESC_LDT:
691         case DESC_TSS:
692                 /* ignore */
693                 break;
694 
695         default: {
696                 xmaddr_t maddr = virt_to_machine(&dt[entry]);
697 
698                 if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc))
699                         dt[entry] = *(struct desc_struct *)desc;
700         }
701 
702         }
703 }
704 
705 static void xen_load_sp0(struct tss_struct *tss,
706                          struct thread_struct *thread)
707 {
708         struct multicall_space mcs = xen_mc_entry(0);
709         MULTI_stack_switch(mcs.mc, __KERNEL_DS, thread->sp0);
710         xen_mc_issue(PARAVIRT_LAZY_CPU);
711 }
712 
713 static void xen_set_iopl_mask(unsigned mask)
714 {
715         struct physdev_set_iopl set_iopl;
716 
717         /* Force the change at ring 0. */
718         set_iopl.iopl = (mask == 0) ? 1 : (mask >> 12) & 3;
719         HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
720 }
721 
722 static void xen_io_delay(void)
723 {
724 }
725 
726 #ifdef CONFIG_X86_LOCAL_APIC
727 static u32 xen_apic_read(u32 reg)
728 {
729         return 0;
730 }
731 
732 static void xen_apic_write(u32 reg, u32 val)
733 {
734         /* Warn to see if there's any stray references */
735         WARN_ON(1);
736 }
737 
738 static u64 xen_apic_icr_read(void)
739 {
740         return 0;
741 }
742 
743 static void xen_apic_icr_write(u32 low, u32 id)
744 {
745         /* Warn to see if there's any stray references */
746         WARN_ON(1);
747 }
748 
749 static void xen_apic_wait_icr_idle(void)
750 {
751         return;
752 }
753 
754 static u32 xen_safe_apic_wait_icr_idle(void)
755 {
756         return 0;
757 }
758 
759 static void set_xen_basic_apic_ops(void)
760 {
761         apic->read = xen_apic_read;
762         apic->write = xen_apic_write;
763         apic->icr_read = xen_apic_icr_read;
764         apic->icr_write = xen_apic_icr_write;
765         apic->wait_icr_idle = xen_apic_wait_icr_idle;
766         apic->safe_wait_icr_idle = xen_safe_apic_wait_icr_idle;
767 }
768 
769 #endif
770 
771 
772 static void xen_clts(void)
773 {
774         struct multicall_space mcs;
775 
776         mcs = xen_mc_entry(0);
777 
778         MULTI_fpu_taskswitch(mcs.mc, 0);
779 
780         xen_mc_issue(PARAVIRT_LAZY_CPU);
781 }
782 
783 static DEFINE_PER_CPU(unsigned long, xen_cr0_value);
784 
785 static unsigned long xen_read_cr0(void)
786 {
787         unsigned long cr0 = percpu_read(xen_cr0_value);
788 
789         if (unlikely(cr0 == 0)) {
790                 cr0 = native_read_cr0();
791                 percpu_write(xen_cr0_value, cr0);
792         }
793 
794         return cr0;
795 }
796 
797 static void xen_write_cr0(unsigned long cr0)
798 {
799         struct multicall_space mcs;
800 
801         percpu_write(xen_cr0_value, cr0);
802 
803         /* Only pay attention to cr0.TS; everything else is
804            ignored. */
805         mcs = xen_mc_entry(0);
806 
807         MULTI_fpu_taskswitch(mcs.mc, (cr0 & X86_CR0_TS) != 0);
808 
809         xen_mc_issue(PARAVIRT_LAZY_CPU);
810 }
811 
812 static void xen_write_cr4(unsigned long cr4)
813 {
814         cr4 &= ~X86_CR4_PGE;
815         cr4 &= ~X86_CR4_PSE;
816 
817         native_write_cr4(cr4);
818 }
819 #ifdef CONFIG_X86_64
820 static inline unsigned long xen_read_cr8(void)
821 {
822         return 0;
823 }
824 static inline void xen_write_cr8(unsigned long val)
825 {
826         BUG_ON(val);
827 }
828 #endif
829 static int xen_write_msr_safe(unsigned int msr, unsigned low, unsigned high)
830 {
831         int ret;
832 
833         ret = 0;
834 
835         switch (msr) {
836 #ifdef CONFIG_X86_64
837                 unsigned which;
838                 u64 base;
839 
840         case MSR_FS_BASE:               which = SEGBASE_FS; goto set;
841         case MSR_KERNEL_GS_BASE:        which = SEGBASE_GS_USER; goto set;
842         case MSR_GS_BASE:               which = SEGBASE_GS_KERNEL; goto set;
843 
844         set:
845                 base = ((u64)high << 32) | low;
846                 if (HYPERVISOR_set_segment_base(which, base) != 0)
847                         ret = -EIO;
848                 break;
849 #endif
850 
851         case MSR_STAR:
852         case MSR_CSTAR:
853         case MSR_LSTAR:
854         case MSR_SYSCALL_MASK:
855         case MSR_IA32_SYSENTER_CS:
856         case MSR_IA32_SYSENTER_ESP:
857         case MSR_IA32_SYSENTER_EIP:
858                 /* Fast syscall setup is all done in hypercalls, so
859                    these are all ignored.  Stub them out here to stop
860                    Xen console noise. */
861                 break;
862 
863         default:
864                 ret = native_write_msr_safe(msr, low, high);
865         }
866 
867         return ret;
868 }
869 
870 void xen_setup_shared_info(void)
871 {
872         if (!xen_feature(XENFEAT_auto_translated_physmap)) {
873                 set_fixmap(FIX_PARAVIRT_BOOTMAP,
874                            xen_start_info->shared_info);
875 
876                 HYPERVISOR_shared_info =
877                         (struct shared_info *)fix_to_virt(FIX_PARAVIRT_BOOTMAP);
878         } else
879                 HYPERVISOR_shared_info =
880                         (struct shared_info *)__va(xen_start_info->shared_info);
881 
882 #ifndef CONFIG_SMP
883         /* In UP this is as good a place as any to set up shared info */
884         xen_setup_vcpu_info_placement();
885 #endif
886 
887         xen_setup_mfn_list_list();
888 }
889 
890 /* This is called once we have the cpu_possible_map */
891 void xen_setup_vcpu_info_placement(void)
892 {
893         int cpu;
894 
895         for_each_possible_cpu(cpu)
896                 xen_vcpu_setup(cpu);
897 
898         /* xen_vcpu_setup managed to place the vcpu_info within the
899            percpu area for all cpus, so make use of it */
900         if (have_vcpu_info_placement) {
901                 printk(KERN_INFO "Xen: using vcpu_info placement\n");
902 
903                 pv_irq_ops.save_fl = __PV_IS_CALLEE_SAVE(xen_save_fl_direct);
904                 pv_irq_ops.restore_fl = __PV_IS_CALLEE_SAVE(xen_restore_fl_direct);
905                 pv_irq_ops.irq_disable = __PV_IS_CALLEE_SAVE(xen_irq_disable_direct);
906                 pv_irq_ops.irq_enable = __PV_IS_CALLEE_SAVE(xen_irq_enable_direct);
907                 pv_mmu_ops.read_cr2 = xen_read_cr2_direct;
908         }
909 }
910 
911 static unsigned xen_patch(u8 type, u16 clobbers, void *insnbuf,
912                           unsigned long addr, unsigned len)
913 {
914         char *start, *end, *reloc;
915         unsigned ret;
916 
917         start = end = reloc = NULL;
918 
919 #define SITE(op, x)                                                     \
920         case PARAVIRT_PATCH(op.x):                                      \
921         if (have_vcpu_info_placement) {                                 \
922                 start = (char *)xen_##x##_direct;                       \
923                 end = xen_##x##_direct_end;                             \
924                 reloc = xen_##x##_direct_reloc;                         \
925         }                                                               \
926         goto patch_site
927 
928         switch (type) {
929                 SITE(pv_irq_ops, irq_enable);
930                 SITE(pv_irq_ops, irq_disable);
931                 SITE(pv_irq_ops, save_fl);
932                 SITE(pv_irq_ops, restore_fl);
933 #undef SITE
934 
935         patch_site:
936                 if (start == NULL || (end-start) > len)
937                         goto default_patch;
938 
939                 ret = paravirt_patch_insns(insnbuf, len, start, end);
940 
941                 /* Note: because reloc is assigned from something that
942                    appears to be an array, gcc assumes it's non-null,
943                    but doesn't know its relationship with start and
944                    end. */
945                 if (reloc > start && reloc < end) {
946                         int reloc_off = reloc - start;
947                         long *relocp = (long *)(insnbuf + reloc_off);
948                         long delta = start - (char *)addr;
949 
950                         *relocp += delta;
951                 }
952                 break;
953 
954         default_patch:
955         default:
956                 ret = paravirt_patch_default(type, clobbers, insnbuf,
957                                              addr, len);
958                 break;
959         }
960 
961         return ret;
962 }
963 
964 static const struct pv_info xen_info __initdata = {
965         .paravirt_enabled = 1,
966         .shared_kernel_pmd = 0,
967 
968         .name = "Xen",
969 };
970 
971 static const struct pv_init_ops xen_init_ops __initdata = {
972         .patch = xen_patch,
973 };
974 
975 static const struct pv_time_ops xen_time_ops __initdata = {
976         .sched_clock = xen_clocksource_read,
977 };
978 
979 static const struct pv_cpu_ops xen_cpu_ops __initdata = {
980         .cpuid = xen_cpuid,
981 
982         .set_debugreg = xen_set_debugreg,
983         .get_debugreg = xen_get_debugreg,
984 
985         .clts = xen_clts,
986 
987         .read_cr0 = xen_read_cr0,
988         .write_cr0 = xen_write_cr0,
989 
990         .read_cr4 = native_read_cr4,
991         .read_cr4_safe = native_read_cr4_safe,
992         .write_cr4 = xen_write_cr4,
993 
994 #ifdef CONFIG_X86_64
995         .read_cr8 = xen_read_cr8,
996         .write_cr8 = xen_write_cr8,
997 #endif
998 
999         .wbinvd = native_wbinvd,
1000 
1001         .read_msr = native_read_msr_safe,
1002         .rdmsr_regs = native_rdmsr_safe_regs,
1003         .write_msr = xen_write_msr_safe,
1004         .wrmsr_regs = native_wrmsr_safe_regs,
1005 
1006         .read_tsc = native_read_tsc,
1007         .read_pmc = native_read_pmc,
1008 
1009         .read_tscp = native_read_tscp,
1010 
1011         .iret = xen_iret,
1012         .irq_enable_sysexit = xen_sysexit,
1013 #ifdef CONFIG_X86_64
1014         .usergs_sysret32 = xen_sysret32,
1015         .usergs_sysret64 = xen_sysret64,
1016 #endif
1017 
1018         .load_tr_desc = paravirt_nop,
1019         .set_ldt = xen_set_ldt,
1020         .load_gdt = xen_load_gdt,
1021         .load_idt = xen_load_idt,
1022         .load_tls = xen_load_tls,
1023 #ifdef CONFIG_X86_64
1024         .load_gs_index = xen_load_gs_index,
1025 #endif
1026 
1027         .alloc_ldt = xen_alloc_ldt,
1028         .free_ldt = xen_free_ldt,
1029 
1030         .store_gdt = native_store_gdt,
1031         .store_idt = native_store_idt,
1032         .store_tr = xen_store_tr,
1033 
1034         .write_ldt_entry = xen_write_ldt_entry,
1035         .write_gdt_entry = xen_write_gdt_entry,
1036         .write_idt_entry = xen_write_idt_entry,
1037         .load_sp0 = xen_load_sp0,
1038 
1039         .set_iopl_mask = xen_set_iopl_mask,
1040         .io_delay = xen_io_delay,
1041 
1042         /* Xen takes care of %gs when switching to usermode for us */
1043         .swapgs = paravirt_nop,
1044 
1045         .start_context_switch = paravirt_start_context_switch,
1046         .end_context_switch = xen_end_context_switch,
1047 };
1048 
1049 static const struct pv_apic_ops xen_apic_ops __initdata = {
1050 #ifdef CONFIG_X86_LOCAL_APIC
1051         .startup_ipi_hook = paravirt_nop,
1052 #endif
1053 };
1054 
1055 static void xen_reboot(int reason)
1056 {
1057         struct sched_shutdown r = { .reason = reason };
1058 
1059         if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r))
1060                 BUG();
1061 }
1062 
1063 static void xen_restart(char *msg)
1064 {
1065         xen_reboot(SHUTDOWN_reboot);
1066 }
1067 
1068 static void xen_emergency_restart(void)
1069 {
1070         xen_reboot(SHUTDOWN_reboot);
1071 }
1072 
1073 static void xen_machine_halt(void)
1074 {
1075         xen_reboot(SHUTDOWN_poweroff);
1076 }
1077 
1078 static void xen_crash_shutdown(struct pt_regs *regs)
1079 {
1080         xen_reboot(SHUTDOWN_crash);
1081 }
1082 
1083 static const struct machine_ops __initdata xen_machine_ops = {
1084         .restart = xen_restart,
1085         .halt = xen_machine_halt,
1086         .power_off = xen_machine_halt,
1087         .shutdown = xen_machine_halt,
1088         .crash_shutdown = xen_crash_shutdown,
1089         .emergency_restart = xen_emergency_restart,
1090 };
1091 
1092 /*
1093  * Set up the GDT and segment registers for -fstack-protector.  Until
1094  * we do this, we have to be careful not to call any stack-protected
1095  * function, which is most of the kernel.
1096  */
1097 static void __init xen_setup_stackprotector(void)
1098 {
1099         pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry_boot;
1100         pv_cpu_ops.load_gdt = xen_load_gdt_boot;
1101 
1102         setup_stack_canary_segment(0);
1103         switch_to_new_gdt(0);
1104 
1105         pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry;
1106         pv_cpu_ops.load_gdt = xen_load_gdt;
1107 }
1108 
1109 /* First C function to be called on Xen boot */
1110 asmlinkage void __init xen_start_kernel(void)
1111 {
1112         pgd_t *pgd;
1113 
1114         if (!xen_start_info)
1115                 return;
1116 
1117         xen_domain_type = XEN_PV_DOMAIN;
1118 
1119         /* Install Xen paravirt ops */
1120         pv_info = xen_info;
1121         pv_init_ops = xen_init_ops;
1122         pv_time_ops = xen_time_ops;
1123         pv_cpu_ops = xen_cpu_ops;
1124         pv_apic_ops = xen_apic_ops;
1125 
1126         x86_init.resources.memory_setup = xen_memory_setup;
1127         x86_init.oem.arch_setup = xen_arch_setup;
1128         x86_init.oem.banner = xen_banner;
1129 
1130         x86_init.timers.timer_init = xen_time_init;
1131         x86_init.timers.setup_percpu_clockev = x86_init_noop;
1132         x86_cpuinit.setup_percpu_clockev = x86_init_noop;
1133 
1134         x86_platform.calibrate_tsc = xen_tsc_khz;
1135         x86_platform.get_wallclock = xen_get_wallclock;
1136         x86_platform.set_wallclock = xen_set_wallclock;
1137 
1138         /*
1139          * Set up some pagetable state before starting to set any ptes.
1140          */
1141 
1142         xen_init_mmu_ops();
1143 
1144         /* Prevent unwanted bits from being set in PTEs. */
1145         __supported_pte_mask &= ~_PAGE_GLOBAL;
1146         if (!xen_initial_domain())
1147                 __supported_pte_mask &= ~(_PAGE_PWT | _PAGE_PCD);
1148 
1149         __supported_pte_mask |= _PAGE_IOMAP;
1150 
1151         /*
1152          * Prevent page tables from being allocated in highmem, even
1153          * if CONFIG_HIGHPTE is enabled.
1154          */
1155         __userpte_alloc_gfp &= ~__GFP_HIGHMEM;
1156 
1157 #ifdef CONFIG_X86_64
1158         /* Work out if we support NX */
1159         check_efer();
1160 #endif
1161 
1162         xen_setup_features();
1163 
1164         /* Get mfn list */
1165         if (!xen_feature(XENFEAT_auto_translated_physmap))
1166                 xen_build_dynamic_phys_to_machine();
1167 
1168         /*
1169          * Set up kernel GDT and segment registers, mainly so that
1170          * -fstack-protector code can be executed.
1171          */
1172         xen_setup_stackprotector();
1173 
1174         xen_init_irq_ops();
1175         xen_init_cpuid_mask();
1176 
1177 #ifdef CONFIG_X86_LOCAL_APIC
1178         /*
1179          * set up the basic apic ops.
1180          */
1181         set_xen_basic_apic_ops();
1182 #endif
1183 
1184         if (xen_feature(XENFEAT_mmu_pt_update_preserve_ad)) {
1185                 pv_mmu_ops.ptep_modify_prot_start = xen_ptep_modify_prot_start;
1186                 pv_mmu_ops.ptep_modify_prot_commit = xen_ptep_modify_prot_commit;
1187         }
1188 
1189         machine_ops = xen_machine_ops;
1190 
1191         /*
1192          * The only reliable way to retain the initial address of the
1193          * percpu gdt_page is to remember it here, so we can go and
1194          * mark it RW later, when the initial percpu area is freed.
1195          */
1196         xen_initial_gdt = &per_cpu(gdt_page, 0);
1197 
1198         xen_smp_init();
1199 
1200         pgd = (pgd_t *)xen_start_info->pt_base;
1201 
1202         /* Don't do the full vcpu_info placement stuff until we have a
1203            possible map and a non-dummy shared_info. */
1204         per_cpu(xen_vcpu, 0) = &HYPERVISOR_shared_info->vcpu_info[0];
1205 
1206         local_irq_disable();
1207         early_boot_irqs_off();
1208 
1209         xen_raw_console_write("mapping kernel into physical memory\n");
1210         pgd = xen_setup_kernel_pagetable(pgd, xen_start_info->nr_pages);
1211 
1212         init_mm.pgd = pgd;
1213 
1214         /* keep using Xen gdt for now; no urgent need to change it */
1215 
1216         pv_info.kernel_rpl = 1;
1217         if (xen_feature(XENFEAT_supervisor_mode_kernel))
1218                 pv_info.kernel_rpl = 0;
1219 
1220         /* set the limit of our address space */
1221         xen_reserve_top();
1222 
1223 #ifdef CONFIG_X86_32
1224         /* set up basic CPUID stuff */
1225         cpu_detect(&new_cpu_data);
1226         new_cpu_data.hard_math = 1;
1227         new_cpu_data.wp_works_ok = 1;
1228         new_cpu_data.x86_capability[0] = cpuid_edx(1);
1229 #endif
1230 
1231         /* Poke various useful things into boot_params */
1232         boot_params.hdr.type_of_loader = (9 << 4) | 0;
1233         boot_params.hdr.ramdisk_image = xen_start_info->mod_start
1234                 ? __pa(xen_start_info->mod_start) : 0;
1235         boot_params.hdr.ramdisk_size = xen_start_info->mod_len;
1236         boot_params.hdr.cmd_line_ptr = __pa(xen_start_info->cmd_line);
1237 
1238         if (!xen_initial_domain()) {
1239                 add_preferred_console("xenboot", 0, NULL);
1240                 add_preferred_console("tty", 0, NULL);
1241                 add_preferred_console("hvc", 0, NULL);
1242         }
1243 
1244         xen_raw_console_write("about to get started...\n");
1245 
1246         xen_setup_runstate_info(0);
1247 
1248         /* Start the world */
1249 #ifdef CONFIG_X86_32
1250         i386_start_kernel();
1251 #else
1252         x86_64_start_reservations((char *)__pa_symbol(&boot_params));
1253 #endif
1254 }
1255 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp