~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/xen/mmu.c

Version: ~ [ linux-5.5-rc1 ] ~ [ linux-5.4.2 ] ~ [ linux-5.3.15 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.88 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.158 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.206 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.206 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.78 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Xen mmu operations
  3  *
  4  * This file contains the various mmu fetch and update operations.
  5  * The most important job they must perform is the mapping between the
  6  * domain's pfn and the overall machine mfns.
  7  *
  8  * Xen allows guests to directly update the pagetable, in a controlled
  9  * fashion.  In other words, the guest modifies the same pagetable
 10  * that the CPU actually uses, which eliminates the overhead of having
 11  * a separate shadow pagetable.
 12  *
 13  * In order to allow this, it falls on the guest domain to map its
 14  * notion of a "physical" pfn - which is just a domain-local linear
 15  * address - into a real "machine address" which the CPU's MMU can
 16  * use.
 17  *
 18  * A pgd_t/pmd_t/pte_t will typically contain an mfn, and so can be
 19  * inserted directly into the pagetable.  When creating a new
 20  * pte/pmd/pgd, it converts the passed pfn into an mfn.  Conversely,
 21  * when reading the content back with __(pgd|pmd|pte)_val, it converts
 22  * the mfn back into a pfn.
 23  *
 24  * The other constraint is that all pages which make up a pagetable
 25  * must be mapped read-only in the guest.  This prevents uncontrolled
 26  * guest updates to the pagetable.  Xen strictly enforces this, and
 27  * will disallow any pagetable update which will end up mapping a
 28  * pagetable page RW, and will disallow using any writable page as a
 29  * pagetable.
 30  *
 31  * Naively, when loading %cr3 with the base of a new pagetable, Xen
 32  * would need to validate the whole pagetable before going on.
 33  * Naturally, this is quite slow.  The solution is to "pin" a
 34  * pagetable, which enforces all the constraints on the pagetable even
 35  * when it is not actively in use.  This menas that Xen can be assured
 36  * that it is still valid when you do load it into %cr3, and doesn't
 37  * need to revalidate it.
 38  *
 39  * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
 40  */
 41 #include <linux/sched.h>
 42 #include <linux/highmem.h>
 43 #include <linux/debugfs.h>
 44 #include <linux/bug.h>
 45 #include <linux/vmalloc.h>
 46 #include <linux/module.h>
 47 #include <linux/gfp.h>
 48 #include <linux/memblock.h>
 49 #include <linux/seq_file.h>
 50 #include <linux/crash_dump.h>
 51 
 52 #include <trace/events/xen.h>
 53 
 54 #include <asm/pgtable.h>
 55 #include <asm/tlbflush.h>
 56 #include <asm/fixmap.h>
 57 #include <asm/mmu_context.h>
 58 #include <asm/setup.h>
 59 #include <asm/paravirt.h>
 60 #include <asm/e820.h>
 61 #include <asm/linkage.h>
 62 #include <asm/page.h>
 63 #include <asm/init.h>
 64 #include <asm/pat.h>
 65 #include <asm/smp.h>
 66 
 67 #include <asm/xen/hypercall.h>
 68 #include <asm/xen/hypervisor.h>
 69 
 70 #include <xen/xen.h>
 71 #include <xen/page.h>
 72 #include <xen/interface/xen.h>
 73 #include <xen/interface/hvm/hvm_op.h>
 74 #include <xen/interface/version.h>
 75 #include <xen/interface/memory.h>
 76 #include <xen/hvc-console.h>
 77 
 78 #include "multicalls.h"
 79 #include "mmu.h"
 80 #include "debugfs.h"
 81 
 82 /*
 83  * Protects atomic reservation decrease/increase against concurrent increases.
 84  * Also protects non-atomic updates of current_pages and balloon lists.
 85  */
 86 DEFINE_SPINLOCK(xen_reservation_lock);
 87 
 88 #ifdef CONFIG_X86_32
 89 /*
 90  * Identity map, in addition to plain kernel map.  This needs to be
 91  * large enough to allocate page table pages to allocate the rest.
 92  * Each page can map 2MB.
 93  */
 94 #define LEVEL1_IDENT_ENTRIES    (PTRS_PER_PTE * 4)
 95 static RESERVE_BRK_ARRAY(pte_t, level1_ident_pgt, LEVEL1_IDENT_ENTRIES);
 96 #endif
 97 #ifdef CONFIG_X86_64
 98 /* l3 pud for userspace vsyscall mapping */
 99 static pud_t level3_user_vsyscall[PTRS_PER_PUD] __page_aligned_bss;
100 #endif /* CONFIG_X86_64 */
101 
102 /*
103  * Note about cr3 (pagetable base) values:
104  *
105  * xen_cr3 contains the current logical cr3 value; it contains the
106  * last set cr3.  This may not be the current effective cr3, because
107  * its update may be being lazily deferred.  However, a vcpu looking
108  * at its own cr3 can use this value knowing that it everything will
109  * be self-consistent.
110  *
111  * xen_current_cr3 contains the actual vcpu cr3; it is set once the
112  * hypercall to set the vcpu cr3 is complete (so it may be a little
113  * out of date, but it will never be set early).  If one vcpu is
114  * looking at another vcpu's cr3 value, it should use this variable.
115  */
116 DEFINE_PER_CPU(unsigned long, xen_cr3);  /* cr3 stored as physaddr */
117 DEFINE_PER_CPU(unsigned long, xen_current_cr3);  /* actual vcpu cr3 */
118 
119 static phys_addr_t xen_pt_base, xen_pt_size __initdata;
120 
121 /*
122  * Just beyond the highest usermode address.  STACK_TOP_MAX has a
123  * redzone above it, so round it up to a PGD boundary.
124  */
125 #define USER_LIMIT      ((STACK_TOP_MAX + PGDIR_SIZE - 1) & PGDIR_MASK)
126 
127 unsigned long arbitrary_virt_to_mfn(void *vaddr)
128 {
129         xmaddr_t maddr = arbitrary_virt_to_machine(vaddr);
130 
131         return PFN_DOWN(maddr.maddr);
132 }
133 
134 xmaddr_t arbitrary_virt_to_machine(void *vaddr)
135 {
136         unsigned long address = (unsigned long)vaddr;
137         unsigned int level;
138         pte_t *pte;
139         unsigned offset;
140 
141         /*
142          * if the PFN is in the linear mapped vaddr range, we can just use
143          * the (quick) virt_to_machine() p2m lookup
144          */
145         if (virt_addr_valid(vaddr))
146                 return virt_to_machine(vaddr);
147 
148         /* otherwise we have to do a (slower) full page-table walk */
149 
150         pte = lookup_address(address, &level);
151         BUG_ON(pte == NULL);
152         offset = address & ~PAGE_MASK;
153         return XMADDR(((phys_addr_t)pte_mfn(*pte) << PAGE_SHIFT) + offset);
154 }
155 EXPORT_SYMBOL_GPL(arbitrary_virt_to_machine);
156 
157 void make_lowmem_page_readonly(void *vaddr)
158 {
159         pte_t *pte, ptev;
160         unsigned long address = (unsigned long)vaddr;
161         unsigned int level;
162 
163         pte = lookup_address(address, &level);
164         if (pte == NULL)
165                 return;         /* vaddr missing */
166 
167         ptev = pte_wrprotect(*pte);
168 
169         if (HYPERVISOR_update_va_mapping(address, ptev, 0))
170                 BUG();
171 }
172 
173 void make_lowmem_page_readwrite(void *vaddr)
174 {
175         pte_t *pte, ptev;
176         unsigned long address = (unsigned long)vaddr;
177         unsigned int level;
178 
179         pte = lookup_address(address, &level);
180         if (pte == NULL)
181                 return;         /* vaddr missing */
182 
183         ptev = pte_mkwrite(*pte);
184 
185         if (HYPERVISOR_update_va_mapping(address, ptev, 0))
186                 BUG();
187 }
188 
189 
190 static bool xen_page_pinned(void *ptr)
191 {
192         struct page *page = virt_to_page(ptr);
193 
194         return PagePinned(page);
195 }
196 
197 void xen_set_domain_pte(pte_t *ptep, pte_t pteval, unsigned domid)
198 {
199         struct multicall_space mcs;
200         struct mmu_update *u;
201 
202         trace_xen_mmu_set_domain_pte(ptep, pteval, domid);
203 
204         mcs = xen_mc_entry(sizeof(*u));
205         u = mcs.args;
206 
207         /* ptep might be kmapped when using 32-bit HIGHPTE */
208         u->ptr = virt_to_machine(ptep).maddr;
209         u->val = pte_val_ma(pteval);
210 
211         MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, domid);
212 
213         xen_mc_issue(PARAVIRT_LAZY_MMU);
214 }
215 EXPORT_SYMBOL_GPL(xen_set_domain_pte);
216 
217 static void xen_extend_mmu_update(const struct mmu_update *update)
218 {
219         struct multicall_space mcs;
220         struct mmu_update *u;
221 
222         mcs = xen_mc_extend_args(__HYPERVISOR_mmu_update, sizeof(*u));
223 
224         if (mcs.mc != NULL) {
225                 mcs.mc->args[1]++;
226         } else {
227                 mcs = __xen_mc_entry(sizeof(*u));
228                 MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, DOMID_SELF);
229         }
230 
231         u = mcs.args;
232         *u = *update;
233 }
234 
235 static void xen_extend_mmuext_op(const struct mmuext_op *op)
236 {
237         struct multicall_space mcs;
238         struct mmuext_op *u;
239 
240         mcs = xen_mc_extend_args(__HYPERVISOR_mmuext_op, sizeof(*u));
241 
242         if (mcs.mc != NULL) {
243                 mcs.mc->args[1]++;
244         } else {
245                 mcs = __xen_mc_entry(sizeof(*u));
246                 MULTI_mmuext_op(mcs.mc, mcs.args, 1, NULL, DOMID_SELF);
247         }
248 
249         u = mcs.args;
250         *u = *op;
251 }
252 
253 static void xen_set_pmd_hyper(pmd_t *ptr, pmd_t val)
254 {
255         struct mmu_update u;
256 
257         preempt_disable();
258 
259         xen_mc_batch();
260 
261         /* ptr may be ioremapped for 64-bit pagetable setup */
262         u.ptr = arbitrary_virt_to_machine(ptr).maddr;
263         u.val = pmd_val_ma(val);
264         xen_extend_mmu_update(&u);
265 
266         xen_mc_issue(PARAVIRT_LAZY_MMU);
267 
268         preempt_enable();
269 }
270 
271 static void xen_set_pmd(pmd_t *ptr, pmd_t val)
272 {
273         trace_xen_mmu_set_pmd(ptr, val);
274 
275         /* If page is not pinned, we can just update the entry
276            directly */
277         if (!xen_page_pinned(ptr)) {
278                 *ptr = val;
279                 return;
280         }
281 
282         xen_set_pmd_hyper(ptr, val);
283 }
284 
285 /*
286  * Associate a virtual page frame with a given physical page frame
287  * and protection flags for that frame.
288  */
289 void set_pte_mfn(unsigned long vaddr, unsigned long mfn, pgprot_t flags)
290 {
291         set_pte_vaddr(vaddr, mfn_pte(mfn, flags));
292 }
293 
294 static bool xen_batched_set_pte(pte_t *ptep, pte_t pteval)
295 {
296         struct mmu_update u;
297 
298         if (paravirt_get_lazy_mode() != PARAVIRT_LAZY_MMU)
299                 return false;
300 
301         xen_mc_batch();
302 
303         u.ptr = virt_to_machine(ptep).maddr | MMU_NORMAL_PT_UPDATE;
304         u.val = pte_val_ma(pteval);
305         xen_extend_mmu_update(&u);
306 
307         xen_mc_issue(PARAVIRT_LAZY_MMU);
308 
309         return true;
310 }
311 
312 static inline void __xen_set_pte(pte_t *ptep, pte_t pteval)
313 {
314         if (!xen_batched_set_pte(ptep, pteval)) {
315                 /*
316                  * Could call native_set_pte() here and trap and
317                  * emulate the PTE write but with 32-bit guests this
318                  * needs two traps (one for each of the two 32-bit
319                  * words in the PTE) so do one hypercall directly
320                  * instead.
321                  */
322                 struct mmu_update u;
323 
324                 u.ptr = virt_to_machine(ptep).maddr | MMU_NORMAL_PT_UPDATE;
325                 u.val = pte_val_ma(pteval);
326                 HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF);
327         }
328 }
329 
330 static void xen_set_pte(pte_t *ptep, pte_t pteval)
331 {
332         trace_xen_mmu_set_pte(ptep, pteval);
333         __xen_set_pte(ptep, pteval);
334 }
335 
336 static void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
337                     pte_t *ptep, pte_t pteval)
338 {
339         trace_xen_mmu_set_pte_at(mm, addr, ptep, pteval);
340         __xen_set_pte(ptep, pteval);
341 }
342 
343 pte_t xen_ptep_modify_prot_start(struct mm_struct *mm,
344                                  unsigned long addr, pte_t *ptep)
345 {
346         /* Just return the pte as-is.  We preserve the bits on commit */
347         trace_xen_mmu_ptep_modify_prot_start(mm, addr, ptep, *ptep);
348         return *ptep;
349 }
350 
351 void xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
352                                  pte_t *ptep, pte_t pte)
353 {
354         struct mmu_update u;
355 
356         trace_xen_mmu_ptep_modify_prot_commit(mm, addr, ptep, pte);
357         xen_mc_batch();
358 
359         u.ptr = virt_to_machine(ptep).maddr | MMU_PT_UPDATE_PRESERVE_AD;
360         u.val = pte_val_ma(pte);
361         xen_extend_mmu_update(&u);
362 
363         xen_mc_issue(PARAVIRT_LAZY_MMU);
364 }
365 
366 /* Assume pteval_t is equivalent to all the other *val_t types. */
367 static pteval_t pte_mfn_to_pfn(pteval_t val)
368 {
369         if (val & _PAGE_PRESENT) {
370                 unsigned long mfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
371                 unsigned long pfn = mfn_to_pfn(mfn);
372 
373                 pteval_t flags = val & PTE_FLAGS_MASK;
374                 if (unlikely(pfn == ~0))
375                         val = flags & ~_PAGE_PRESENT;
376                 else
377                         val = ((pteval_t)pfn << PAGE_SHIFT) | flags;
378         }
379 
380         return val;
381 }
382 
383 static pteval_t pte_pfn_to_mfn(pteval_t val)
384 {
385         if (val & _PAGE_PRESENT) {
386                 unsigned long pfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
387                 pteval_t flags = val & PTE_FLAGS_MASK;
388                 unsigned long mfn;
389 
390                 if (!xen_feature(XENFEAT_auto_translated_physmap))
391                         mfn = __pfn_to_mfn(pfn);
392                 else
393                         mfn = pfn;
394                 /*
395                  * If there's no mfn for the pfn, then just create an
396                  * empty non-present pte.  Unfortunately this loses
397                  * information about the original pfn, so
398                  * pte_mfn_to_pfn is asymmetric.
399                  */
400                 if (unlikely(mfn == INVALID_P2M_ENTRY)) {
401                         mfn = 0;
402                         flags = 0;
403                 } else
404                         mfn &= ~(FOREIGN_FRAME_BIT | IDENTITY_FRAME_BIT);
405                 val = ((pteval_t)mfn << PAGE_SHIFT) | flags;
406         }
407 
408         return val;
409 }
410 
411 __visible pteval_t xen_pte_val(pte_t pte)
412 {
413         pteval_t pteval = pte.pte;
414 
415         return pte_mfn_to_pfn(pteval);
416 }
417 PV_CALLEE_SAVE_REGS_THUNK(xen_pte_val);
418 
419 __visible pgdval_t xen_pgd_val(pgd_t pgd)
420 {
421         return pte_mfn_to_pfn(pgd.pgd);
422 }
423 PV_CALLEE_SAVE_REGS_THUNK(xen_pgd_val);
424 
425 __visible pte_t xen_make_pte(pteval_t pte)
426 {
427         pte = pte_pfn_to_mfn(pte);
428 
429         return native_make_pte(pte);
430 }
431 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte);
432 
433 __visible pgd_t xen_make_pgd(pgdval_t pgd)
434 {
435         pgd = pte_pfn_to_mfn(pgd);
436         return native_make_pgd(pgd);
437 }
438 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pgd);
439 
440 __visible pmdval_t xen_pmd_val(pmd_t pmd)
441 {
442         return pte_mfn_to_pfn(pmd.pmd);
443 }
444 PV_CALLEE_SAVE_REGS_THUNK(xen_pmd_val);
445 
446 static void xen_set_pud_hyper(pud_t *ptr, pud_t val)
447 {
448         struct mmu_update u;
449 
450         preempt_disable();
451 
452         xen_mc_batch();
453 
454         /* ptr may be ioremapped for 64-bit pagetable setup */
455         u.ptr = arbitrary_virt_to_machine(ptr).maddr;
456         u.val = pud_val_ma(val);
457         xen_extend_mmu_update(&u);
458 
459         xen_mc_issue(PARAVIRT_LAZY_MMU);
460 
461         preempt_enable();
462 }
463 
464 static void xen_set_pud(pud_t *ptr, pud_t val)
465 {
466         trace_xen_mmu_set_pud(ptr, val);
467 
468         /* If page is not pinned, we can just update the entry
469            directly */
470         if (!xen_page_pinned(ptr)) {
471                 *ptr = val;
472                 return;
473         }
474 
475         xen_set_pud_hyper(ptr, val);
476 }
477 
478 #ifdef CONFIG_X86_PAE
479 static void xen_set_pte_atomic(pte_t *ptep, pte_t pte)
480 {
481         trace_xen_mmu_set_pte_atomic(ptep, pte);
482         set_64bit((u64 *)ptep, native_pte_val(pte));
483 }
484 
485 static void xen_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
486 {
487         trace_xen_mmu_pte_clear(mm, addr, ptep);
488         if (!xen_batched_set_pte(ptep, native_make_pte(0)))
489                 native_pte_clear(mm, addr, ptep);
490 }
491 
492 static void xen_pmd_clear(pmd_t *pmdp)
493 {
494         trace_xen_mmu_pmd_clear(pmdp);
495         set_pmd(pmdp, __pmd(0));
496 }
497 #endif  /* CONFIG_X86_PAE */
498 
499 __visible pmd_t xen_make_pmd(pmdval_t pmd)
500 {
501         pmd = pte_pfn_to_mfn(pmd);
502         return native_make_pmd(pmd);
503 }
504 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pmd);
505 
506 #if CONFIG_PGTABLE_LEVELS == 4
507 __visible pudval_t xen_pud_val(pud_t pud)
508 {
509         return pte_mfn_to_pfn(pud.pud);
510 }
511 PV_CALLEE_SAVE_REGS_THUNK(xen_pud_val);
512 
513 __visible pud_t xen_make_pud(pudval_t pud)
514 {
515         pud = pte_pfn_to_mfn(pud);
516 
517         return native_make_pud(pud);
518 }
519 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pud);
520 
521 static pgd_t *xen_get_user_pgd(pgd_t *pgd)
522 {
523         pgd_t *pgd_page = (pgd_t *)(((unsigned long)pgd) & PAGE_MASK);
524         unsigned offset = pgd - pgd_page;
525         pgd_t *user_ptr = NULL;
526 
527         if (offset < pgd_index(USER_LIMIT)) {
528                 struct page *page = virt_to_page(pgd_page);
529                 user_ptr = (pgd_t *)page->private;
530                 if (user_ptr)
531                         user_ptr += offset;
532         }
533 
534         return user_ptr;
535 }
536 
537 static void __xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
538 {
539         struct mmu_update u;
540 
541         u.ptr = virt_to_machine(ptr).maddr;
542         u.val = pgd_val_ma(val);
543         xen_extend_mmu_update(&u);
544 }
545 
546 /*
547  * Raw hypercall-based set_pgd, intended for in early boot before
548  * there's a page structure.  This implies:
549  *  1. The only existing pagetable is the kernel's
550  *  2. It is always pinned
551  *  3. It has no user pagetable attached to it
552  */
553 static void __init xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
554 {
555         preempt_disable();
556 
557         xen_mc_batch();
558 
559         __xen_set_pgd_hyper(ptr, val);
560 
561         xen_mc_issue(PARAVIRT_LAZY_MMU);
562 
563         preempt_enable();
564 }
565 
566 static void xen_set_pgd(pgd_t *ptr, pgd_t val)
567 {
568         pgd_t *user_ptr = xen_get_user_pgd(ptr);
569 
570         trace_xen_mmu_set_pgd(ptr, user_ptr, val);
571 
572         /* If page is not pinned, we can just update the entry
573            directly */
574         if (!xen_page_pinned(ptr)) {
575                 *ptr = val;
576                 if (user_ptr) {
577                         WARN_ON(xen_page_pinned(user_ptr));
578                         *user_ptr = val;
579                 }
580                 return;
581         }
582 
583         /* If it's pinned, then we can at least batch the kernel and
584            user updates together. */
585         xen_mc_batch();
586 
587         __xen_set_pgd_hyper(ptr, val);
588         if (user_ptr)
589                 __xen_set_pgd_hyper(user_ptr, val);
590 
591         xen_mc_issue(PARAVIRT_LAZY_MMU);
592 }
593 #endif  /* CONFIG_PGTABLE_LEVELS == 4 */
594 
595 /*
596  * (Yet another) pagetable walker.  This one is intended for pinning a
597  * pagetable.  This means that it walks a pagetable and calls the
598  * callback function on each page it finds making up the page table,
599  * at every level.  It walks the entire pagetable, but it only bothers
600  * pinning pte pages which are below limit.  In the normal case this
601  * will be STACK_TOP_MAX, but at boot we need to pin up to
602  * FIXADDR_TOP.
603  *
604  * For 32-bit the important bit is that we don't pin beyond there,
605  * because then we start getting into Xen's ptes.
606  *
607  * For 64-bit, we must skip the Xen hole in the middle of the address
608  * space, just after the big x86-64 virtual hole.
609  */
610 static int __xen_pgd_walk(struct mm_struct *mm, pgd_t *pgd,
611                           int (*func)(struct mm_struct *mm, struct page *,
612                                       enum pt_level),
613                           unsigned long limit)
614 {
615         int flush = 0;
616         unsigned hole_low, hole_high;
617         unsigned pgdidx_limit, pudidx_limit, pmdidx_limit;
618         unsigned pgdidx, pudidx, pmdidx;
619 
620         /* The limit is the last byte to be touched */
621         limit--;
622         BUG_ON(limit >= FIXADDR_TOP);
623 
624         if (xen_feature(XENFEAT_auto_translated_physmap))
625                 return 0;
626 
627         /*
628          * 64-bit has a great big hole in the middle of the address
629          * space, which contains the Xen mappings.  On 32-bit these
630          * will end up making a zero-sized hole and so is a no-op.
631          */
632         hole_low = pgd_index(USER_LIMIT);
633         hole_high = pgd_index(PAGE_OFFSET);
634 
635         pgdidx_limit = pgd_index(limit);
636 #if PTRS_PER_PUD > 1
637         pudidx_limit = pud_index(limit);
638 #else
639         pudidx_limit = 0;
640 #endif
641 #if PTRS_PER_PMD > 1
642         pmdidx_limit = pmd_index(limit);
643 #else
644         pmdidx_limit = 0;
645 #endif
646 
647         for (pgdidx = 0; pgdidx <= pgdidx_limit; pgdidx++) {
648                 pud_t *pud;
649 
650                 if (pgdidx >= hole_low && pgdidx < hole_high)
651                         continue;
652 
653                 if (!pgd_val(pgd[pgdidx]))
654                         continue;
655 
656                 pud = pud_offset(&pgd[pgdidx], 0);
657 
658                 if (PTRS_PER_PUD > 1) /* not folded */
659                         flush |= (*func)(mm, virt_to_page(pud), PT_PUD);
660 
661                 for (pudidx = 0; pudidx < PTRS_PER_PUD; pudidx++) {
662                         pmd_t *pmd;
663 
664                         if (pgdidx == pgdidx_limit &&
665                             pudidx > pudidx_limit)
666                                 goto out;
667 
668                         if (pud_none(pud[pudidx]))
669                                 continue;
670 
671                         pmd = pmd_offset(&pud[pudidx], 0);
672 
673                         if (PTRS_PER_PMD > 1) /* not folded */
674                                 flush |= (*func)(mm, virt_to_page(pmd), PT_PMD);
675 
676                         for (pmdidx = 0; pmdidx < PTRS_PER_PMD; pmdidx++) {
677                                 struct page *pte;
678 
679                                 if (pgdidx == pgdidx_limit &&
680                                     pudidx == pudidx_limit &&
681                                     pmdidx > pmdidx_limit)
682                                         goto out;
683 
684                                 if (pmd_none(pmd[pmdidx]))
685                                         continue;
686 
687                                 pte = pmd_page(pmd[pmdidx]);
688                                 flush |= (*func)(mm, pte, PT_PTE);
689                         }
690                 }
691         }
692 
693 out:
694         /* Do the top level last, so that the callbacks can use it as
695            a cue to do final things like tlb flushes. */
696         flush |= (*func)(mm, virt_to_page(pgd), PT_PGD);
697 
698         return flush;
699 }
700 
701 static int xen_pgd_walk(struct mm_struct *mm,
702                         int (*func)(struct mm_struct *mm, struct page *,
703                                     enum pt_level),
704                         unsigned long limit)
705 {
706         return __xen_pgd_walk(mm, mm->pgd, func, limit);
707 }
708 
709 /* If we're using split pte locks, then take the page's lock and
710    return a pointer to it.  Otherwise return NULL. */
711 static spinlock_t *xen_pte_lock(struct page *page, struct mm_struct *mm)
712 {
713         spinlock_t *ptl = NULL;
714 
715 #if USE_SPLIT_PTE_PTLOCKS
716         ptl = ptlock_ptr(page);
717         spin_lock_nest_lock(ptl, &mm->page_table_lock);
718 #endif
719 
720         return ptl;
721 }
722 
723 static void xen_pte_unlock(void *v)
724 {
725         spinlock_t *ptl = v;
726         spin_unlock(ptl);
727 }
728 
729 static void xen_do_pin(unsigned level, unsigned long pfn)
730 {
731         struct mmuext_op op;
732 
733         op.cmd = level;
734         op.arg1.mfn = pfn_to_mfn(pfn);
735 
736         xen_extend_mmuext_op(&op);
737 }
738 
739 static int xen_pin_page(struct mm_struct *mm, struct page *page,
740                         enum pt_level level)
741 {
742         unsigned pgfl = TestSetPagePinned(page);
743         int flush;
744 
745         if (pgfl)
746                 flush = 0;              /* already pinned */
747         else if (PageHighMem(page))
748                 /* kmaps need flushing if we found an unpinned
749                    highpage */
750                 flush = 1;
751         else {
752                 void *pt = lowmem_page_address(page);
753                 unsigned long pfn = page_to_pfn(page);
754                 struct multicall_space mcs = __xen_mc_entry(0);
755                 spinlock_t *ptl;
756 
757                 flush = 0;
758 
759                 /*
760                  * We need to hold the pagetable lock between the time
761                  * we make the pagetable RO and when we actually pin
762                  * it.  If we don't, then other users may come in and
763                  * attempt to update the pagetable by writing it,
764                  * which will fail because the memory is RO but not
765                  * pinned, so Xen won't do the trap'n'emulate.
766                  *
767                  * If we're using split pte locks, we can't hold the
768                  * entire pagetable's worth of locks during the
769                  * traverse, because we may wrap the preempt count (8
770                  * bits).  The solution is to mark RO and pin each PTE
771                  * page while holding the lock.  This means the number
772                  * of locks we end up holding is never more than a
773                  * batch size (~32 entries, at present).
774                  *
775                  * If we're not using split pte locks, we needn't pin
776                  * the PTE pages independently, because we're
777                  * protected by the overall pagetable lock.
778                  */
779                 ptl = NULL;
780                 if (level == PT_PTE)
781                         ptl = xen_pte_lock(page, mm);
782 
783                 MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
784                                         pfn_pte(pfn, PAGE_KERNEL_RO),
785                                         level == PT_PGD ? UVMF_TLB_FLUSH : 0);
786 
787                 if (ptl) {
788                         xen_do_pin(MMUEXT_PIN_L1_TABLE, pfn);
789 
790                         /* Queue a deferred unlock for when this batch
791                            is completed. */
792                         xen_mc_callback(xen_pte_unlock, ptl);
793                 }
794         }
795 
796         return flush;
797 }
798 
799 /* This is called just after a mm has been created, but it has not
800    been used yet.  We need to make sure that its pagetable is all
801    read-only, and can be pinned. */
802 static void __xen_pgd_pin(struct mm_struct *mm, pgd_t *pgd)
803 {
804         trace_xen_mmu_pgd_pin(mm, pgd);
805 
806         xen_mc_batch();
807 
808         if (__xen_pgd_walk(mm, pgd, xen_pin_page, USER_LIMIT)) {
809                 /* re-enable interrupts for flushing */
810                 xen_mc_issue(0);
811 
812                 kmap_flush_unused();
813 
814                 xen_mc_batch();
815         }
816 
817 #ifdef CONFIG_X86_64
818         {
819                 pgd_t *user_pgd = xen_get_user_pgd(pgd);
820 
821                 xen_do_pin(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa(pgd)));
822 
823                 if (user_pgd) {
824                         xen_pin_page(mm, virt_to_page(user_pgd), PT_PGD);
825                         xen_do_pin(MMUEXT_PIN_L4_TABLE,
826                                    PFN_DOWN(__pa(user_pgd)));
827                 }
828         }
829 #else /* CONFIG_X86_32 */
830 #ifdef CONFIG_X86_PAE
831         /* Need to make sure unshared kernel PMD is pinnable */
832         xen_pin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]),
833                      PT_PMD);
834 #endif
835         xen_do_pin(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(pgd)));
836 #endif /* CONFIG_X86_64 */
837         xen_mc_issue(0);
838 }
839 
840 static void xen_pgd_pin(struct mm_struct *mm)
841 {
842         __xen_pgd_pin(mm, mm->pgd);
843 }
844 
845 /*
846  * On save, we need to pin all pagetables to make sure they get their
847  * mfns turned into pfns.  Search the list for any unpinned pgds and pin
848  * them (unpinned pgds are not currently in use, probably because the
849  * process is under construction or destruction).
850  *
851  * Expected to be called in stop_machine() ("equivalent to taking
852  * every spinlock in the system"), so the locking doesn't really
853  * matter all that much.
854  */
855 void xen_mm_pin_all(void)
856 {
857         struct page *page;
858 
859         spin_lock(&pgd_lock);
860 
861         list_for_each_entry(page, &pgd_list, lru) {
862                 if (!PagePinned(page)) {
863                         __xen_pgd_pin(&init_mm, (pgd_t *)page_address(page));
864                         SetPageSavePinned(page);
865                 }
866         }
867 
868         spin_unlock(&pgd_lock);
869 }
870 
871 /*
872  * The init_mm pagetable is really pinned as soon as its created, but
873  * that's before we have page structures to store the bits.  So do all
874  * the book-keeping now.
875  */
876 static int __init xen_mark_pinned(struct mm_struct *mm, struct page *page,
877                                   enum pt_level level)
878 {
879         SetPagePinned(page);
880         return 0;
881 }
882 
883 static void __init xen_mark_init_mm_pinned(void)
884 {
885         xen_pgd_walk(&init_mm, xen_mark_pinned, FIXADDR_TOP);
886 }
887 
888 static int xen_unpin_page(struct mm_struct *mm, struct page *page,
889                           enum pt_level level)
890 {
891         unsigned pgfl = TestClearPagePinned(page);
892 
893         if (pgfl && !PageHighMem(page)) {
894                 void *pt = lowmem_page_address(page);
895                 unsigned long pfn = page_to_pfn(page);
896                 spinlock_t *ptl = NULL;
897                 struct multicall_space mcs;
898 
899                 /*
900                  * Do the converse to pin_page.  If we're using split
901                  * pte locks, we must be holding the lock for while
902                  * the pte page is unpinned but still RO to prevent
903                  * concurrent updates from seeing it in this
904                  * partially-pinned state.
905                  */
906                 if (level == PT_PTE) {
907                         ptl = xen_pte_lock(page, mm);
908 
909                         if (ptl)
910                                 xen_do_pin(MMUEXT_UNPIN_TABLE, pfn);
911                 }
912 
913                 mcs = __xen_mc_entry(0);
914 
915                 MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
916                                         pfn_pte(pfn, PAGE_KERNEL),
917                                         level == PT_PGD ? UVMF_TLB_FLUSH : 0);
918 
919                 if (ptl) {
920                         /* unlock when batch completed */
921                         xen_mc_callback(xen_pte_unlock, ptl);
922                 }
923         }
924 
925         return 0;               /* never need to flush on unpin */
926 }
927 
928 /* Release a pagetables pages back as normal RW */
929 static void __xen_pgd_unpin(struct mm_struct *mm, pgd_t *pgd)
930 {
931         trace_xen_mmu_pgd_unpin(mm, pgd);
932 
933         xen_mc_batch();
934 
935         xen_do_pin(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
936 
937 #ifdef CONFIG_X86_64
938         {
939                 pgd_t *user_pgd = xen_get_user_pgd(pgd);
940 
941                 if (user_pgd) {
942                         xen_do_pin(MMUEXT_UNPIN_TABLE,
943                                    PFN_DOWN(__pa(user_pgd)));
944                         xen_unpin_page(mm, virt_to_page(user_pgd), PT_PGD);
945                 }
946         }
947 #endif
948 
949 #ifdef CONFIG_X86_PAE
950         /* Need to make sure unshared kernel PMD is unpinned */
951         xen_unpin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]),
952                        PT_PMD);
953 #endif
954 
955         __xen_pgd_walk(mm, pgd, xen_unpin_page, USER_LIMIT);
956 
957         xen_mc_issue(0);
958 }
959 
960 static void xen_pgd_unpin(struct mm_struct *mm)
961 {
962         __xen_pgd_unpin(mm, mm->pgd);
963 }
964 
965 /*
966  * On resume, undo any pinning done at save, so that the rest of the
967  * kernel doesn't see any unexpected pinned pagetables.
968  */
969 void xen_mm_unpin_all(void)
970 {
971         struct page *page;
972 
973         spin_lock(&pgd_lock);
974 
975         list_for_each_entry(page, &pgd_list, lru) {
976                 if (PageSavePinned(page)) {
977                         BUG_ON(!PagePinned(page));
978                         __xen_pgd_unpin(&init_mm, (pgd_t *)page_address(page));
979                         ClearPageSavePinned(page);
980                 }
981         }
982 
983         spin_unlock(&pgd_lock);
984 }
985 
986 static void xen_activate_mm(struct mm_struct *prev, struct mm_struct *next)
987 {
988         spin_lock(&next->page_table_lock);
989         xen_pgd_pin(next);
990         spin_unlock(&next->page_table_lock);
991 }
992 
993 static void xen_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm)
994 {
995         spin_lock(&mm->page_table_lock);
996         xen_pgd_pin(mm);
997         spin_unlock(&mm->page_table_lock);
998 }
999 
1000 
1001 #ifdef CONFIG_SMP
1002 /* Another cpu may still have their %cr3 pointing at the pagetable, so
1003    we need to repoint it somewhere else before we can unpin it. */
1004 static void drop_other_mm_ref(void *info)
1005 {
1006         struct mm_struct *mm = info;
1007         struct mm_struct *active_mm;
1008 
1009         active_mm = this_cpu_read(cpu_tlbstate.active_mm);
1010 
1011         if (active_mm == mm && this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
1012                 leave_mm(smp_processor_id());
1013 
1014         /* If this cpu still has a stale cr3 reference, then make sure
1015            it has been flushed. */
1016         if (this_cpu_read(xen_current_cr3) == __pa(mm->pgd))
1017                 load_cr3(swapper_pg_dir);
1018 }
1019 
1020 static void xen_drop_mm_ref(struct mm_struct *mm)
1021 {
1022         cpumask_var_t mask;
1023         unsigned cpu;
1024 
1025         if (current->active_mm == mm) {
1026                 if (current->mm == mm)
1027                         load_cr3(swapper_pg_dir);
1028                 else
1029                         leave_mm(smp_processor_id());
1030         }
1031 
1032         /* Get the "official" set of cpus referring to our pagetable. */
1033         if (!alloc_cpumask_var(&mask, GFP_ATOMIC)) {
1034                 for_each_online_cpu(cpu) {
1035                         if (!cpumask_test_cpu(cpu, mm_cpumask(mm))
1036                             && per_cpu(xen_current_cr3, cpu) != __pa(mm->pgd))
1037                                 continue;
1038                         smp_call_function_single(cpu, drop_other_mm_ref, mm, 1);
1039                 }
1040                 return;
1041         }
1042         cpumask_copy(mask, mm_cpumask(mm));
1043 
1044         /* It's possible that a vcpu may have a stale reference to our
1045            cr3, because its in lazy mode, and it hasn't yet flushed
1046            its set of pending hypercalls yet.  In this case, we can
1047            look at its actual current cr3 value, and force it to flush
1048            if needed. */
1049         for_each_online_cpu(cpu) {
1050                 if (per_cpu(xen_current_cr3, cpu) == __pa(mm->pgd))
1051                         cpumask_set_cpu(cpu, mask);
1052         }
1053 
1054         if (!cpumask_empty(mask))
1055                 smp_call_function_many(mask, drop_other_mm_ref, mm, 1);
1056         free_cpumask_var(mask);
1057 }
1058 #else
1059 static void xen_drop_mm_ref(struct mm_struct *mm)
1060 {
1061         if (current->active_mm == mm)
1062                 load_cr3(swapper_pg_dir);
1063 }
1064 #endif
1065 
1066 /*
1067  * While a process runs, Xen pins its pagetables, which means that the
1068  * hypervisor forces it to be read-only, and it controls all updates
1069  * to it.  This means that all pagetable updates have to go via the
1070  * hypervisor, which is moderately expensive.
1071  *
1072  * Since we're pulling the pagetable down, we switch to use init_mm,
1073  * unpin old process pagetable and mark it all read-write, which
1074  * allows further operations on it to be simple memory accesses.
1075  *
1076  * The only subtle point is that another CPU may be still using the
1077  * pagetable because of lazy tlb flushing.  This means we need need to
1078  * switch all CPUs off this pagetable before we can unpin it.
1079  */
1080 static void xen_exit_mmap(struct mm_struct *mm)
1081 {
1082         get_cpu();              /* make sure we don't move around */
1083         xen_drop_mm_ref(mm);
1084         put_cpu();
1085 
1086         spin_lock(&mm->page_table_lock);
1087 
1088         /* pgd may not be pinned in the error exit path of execve */
1089         if (xen_page_pinned(mm->pgd))
1090                 xen_pgd_unpin(mm);
1091 
1092         spin_unlock(&mm->page_table_lock);
1093 }
1094 
1095 static void xen_post_allocator_init(void);
1096 
1097 static void __init pin_pagetable_pfn(unsigned cmd, unsigned long pfn)
1098 {
1099         struct mmuext_op op;
1100 
1101         op.cmd = cmd;
1102         op.arg1.mfn = pfn_to_mfn(pfn);
1103         if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF))
1104                 BUG();
1105 }
1106 
1107 #ifdef CONFIG_X86_64
1108 static void __init xen_cleanhighmap(unsigned long vaddr,
1109                                     unsigned long vaddr_end)
1110 {
1111         unsigned long kernel_end = roundup((unsigned long)_brk_end, PMD_SIZE) - 1;
1112         pmd_t *pmd = level2_kernel_pgt + pmd_index(vaddr);
1113 
1114         /* NOTE: The loop is more greedy than the cleanup_highmap variant.
1115          * We include the PMD passed in on _both_ boundaries. */
1116         for (; vaddr <= vaddr_end && (pmd < (level2_kernel_pgt + PTRS_PER_PMD));
1117                         pmd++, vaddr += PMD_SIZE) {
1118                 if (pmd_none(*pmd))
1119                         continue;
1120                 if (vaddr < (unsigned long) _text || vaddr > kernel_end)
1121                         set_pmd(pmd, __pmd(0));
1122         }
1123         /* In case we did something silly, we should crash in this function
1124          * instead of somewhere later and be confusing. */
1125         xen_mc_flush();
1126 }
1127 
1128 /*
1129  * Make a page range writeable and free it.
1130  */
1131 static void __init xen_free_ro_pages(unsigned long paddr, unsigned long size)
1132 {
1133         void *vaddr = __va(paddr);
1134         void *vaddr_end = vaddr + size;
1135 
1136         for (; vaddr < vaddr_end; vaddr += PAGE_SIZE)
1137                 make_lowmem_page_readwrite(vaddr);
1138 
1139         memblock_free(paddr, size);
1140 }
1141 
1142 static void __init xen_cleanmfnmap_free_pgtbl(void *pgtbl, bool unpin)
1143 {
1144         unsigned long pa = __pa(pgtbl) & PHYSICAL_PAGE_MASK;
1145 
1146         if (unpin)
1147                 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(pa));
1148         ClearPagePinned(virt_to_page(__va(pa)));
1149         xen_free_ro_pages(pa, PAGE_SIZE);
1150 }
1151 
1152 /*
1153  * Since it is well isolated we can (and since it is perhaps large we should)
1154  * also free the page tables mapping the initial P->M table.
1155  */
1156 static void __init xen_cleanmfnmap(unsigned long vaddr)
1157 {
1158         unsigned long va = vaddr & PMD_MASK;
1159         unsigned long pa;
1160         pgd_t *pgd = pgd_offset_k(va);
1161         pud_t *pud_page = pud_offset(pgd, 0);
1162         pud_t *pud;
1163         pmd_t *pmd;
1164         pte_t *pte;
1165         unsigned int i;
1166         bool unpin;
1167 
1168         unpin = (vaddr == 2 * PGDIR_SIZE);
1169         set_pgd(pgd, __pgd(0));
1170         do {
1171                 pud = pud_page + pud_index(va);
1172                 if (pud_none(*pud)) {
1173                         va += PUD_SIZE;
1174                 } else if (pud_large(*pud)) {
1175                         pa = pud_val(*pud) & PHYSICAL_PAGE_MASK;
1176                         xen_free_ro_pages(pa, PUD_SIZE);
1177                         va += PUD_SIZE;
1178                 } else {
1179                         pmd = pmd_offset(pud, va);
1180                         if (pmd_large(*pmd)) {
1181                                 pa = pmd_val(*pmd) & PHYSICAL_PAGE_MASK;
1182                                 xen_free_ro_pages(pa, PMD_SIZE);
1183                         } else if (!pmd_none(*pmd)) {
1184                                 pte = pte_offset_kernel(pmd, va);
1185                                 set_pmd(pmd, __pmd(0));
1186                                 for (i = 0; i < PTRS_PER_PTE; ++i) {
1187                                         if (pte_none(pte[i]))
1188                                                 break;
1189                                         pa = pte_pfn(pte[i]) << PAGE_SHIFT;
1190                                         xen_free_ro_pages(pa, PAGE_SIZE);
1191                                 }
1192                                 xen_cleanmfnmap_free_pgtbl(pte, unpin);
1193                         }
1194                         va += PMD_SIZE;
1195                         if (pmd_index(va))
1196                                 continue;
1197                         set_pud(pud, __pud(0));
1198                         xen_cleanmfnmap_free_pgtbl(pmd, unpin);
1199                 }
1200 
1201         } while (pud_index(va) || pmd_index(va));
1202         xen_cleanmfnmap_free_pgtbl(pud_page, unpin);
1203 }
1204 
1205 static void __init xen_pagetable_p2m_free(void)
1206 {
1207         unsigned long size;
1208         unsigned long addr;
1209 
1210         size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long));
1211 
1212         /* No memory or already called. */
1213         if ((unsigned long)xen_p2m_addr == xen_start_info->mfn_list)
1214                 return;
1215 
1216         /* using __ka address and sticking INVALID_P2M_ENTRY! */
1217         memset((void *)xen_start_info->mfn_list, 0xff, size);
1218 
1219         addr = xen_start_info->mfn_list;
1220         /*
1221          * We could be in __ka space.
1222          * We roundup to the PMD, which means that if anybody at this stage is
1223          * using the __ka address of xen_start_info or
1224          * xen_start_info->shared_info they are in going to crash. Fortunatly
1225          * we have already revectored in xen_setup_kernel_pagetable and in
1226          * xen_setup_shared_info.
1227          */
1228         size = roundup(size, PMD_SIZE);
1229 
1230         if (addr >= __START_KERNEL_map) {
1231                 xen_cleanhighmap(addr, addr + size);
1232                 size = PAGE_ALIGN(xen_start_info->nr_pages *
1233                                   sizeof(unsigned long));
1234                 memblock_free(__pa(addr), size);
1235         } else {
1236                 xen_cleanmfnmap(addr);
1237         }
1238 }
1239 
1240 static void __init xen_pagetable_cleanhighmap(void)
1241 {
1242         unsigned long size;
1243         unsigned long addr;
1244 
1245         /* At this stage, cleanup_highmap has already cleaned __ka space
1246          * from _brk_limit way up to the max_pfn_mapped (which is the end of
1247          * the ramdisk). We continue on, erasing PMD entries that point to page
1248          * tables - do note that they are accessible at this stage via __va.
1249          * For good measure we also round up to the PMD - which means that if
1250          * anybody is using __ka address to the initial boot-stack - and try
1251          * to use it - they are going to crash. The xen_start_info has been
1252          * taken care of already in xen_setup_kernel_pagetable. */
1253         addr = xen_start_info->pt_base;
1254         size = roundup(xen_start_info->nr_pt_frames * PAGE_SIZE, PMD_SIZE);
1255 
1256         xen_cleanhighmap(addr, addr + size);
1257         xen_start_info->pt_base = (unsigned long)__va(__pa(xen_start_info->pt_base));
1258 #ifdef DEBUG
1259         /* This is superfluous and is not necessary, but you know what
1260          * lets do it. The MODULES_VADDR -> MODULES_END should be clear of
1261          * anything at this stage. */
1262         xen_cleanhighmap(MODULES_VADDR, roundup(MODULES_VADDR, PUD_SIZE) - 1);
1263 #endif
1264 }
1265 #endif
1266 
1267 static void __init xen_pagetable_p2m_setup(void)
1268 {
1269         if (xen_feature(XENFEAT_auto_translated_physmap))
1270                 return;
1271 
1272         xen_vmalloc_p2m_tree();
1273 
1274 #ifdef CONFIG_X86_64
1275         xen_pagetable_p2m_free();
1276 
1277         xen_pagetable_cleanhighmap();
1278 #endif
1279         /* And revector! Bye bye old array */
1280         xen_start_info->mfn_list = (unsigned long)xen_p2m_addr;
1281 }
1282 
1283 static void __init xen_pagetable_init(void)
1284 {
1285         paging_init();
1286         xen_post_allocator_init();
1287 
1288         xen_pagetable_p2m_setup();
1289 
1290         /* Allocate and initialize top and mid mfn levels for p2m structure */
1291         xen_build_mfn_list_list();
1292 
1293         /* Remap memory freed due to conflicts with E820 map */
1294         if (!xen_feature(XENFEAT_auto_translated_physmap))
1295                 xen_remap_memory();
1296 
1297         xen_setup_shared_info();
1298 }
1299 static void xen_write_cr2(unsigned long cr2)
1300 {
1301         this_cpu_read(xen_vcpu)->arch.cr2 = cr2;
1302 }
1303 
1304 static unsigned long xen_read_cr2(void)
1305 {
1306         return this_cpu_read(xen_vcpu)->arch.cr2;
1307 }
1308 
1309 unsigned long xen_read_cr2_direct(void)
1310 {
1311         return this_cpu_read(xen_vcpu_info.arch.cr2);
1312 }
1313 
1314 void xen_flush_tlb_all(void)
1315 {
1316         struct mmuext_op *op;
1317         struct multicall_space mcs;
1318 
1319         trace_xen_mmu_flush_tlb_all(0);
1320 
1321         preempt_disable();
1322 
1323         mcs = xen_mc_entry(sizeof(*op));
1324 
1325         op = mcs.args;
1326         op->cmd = MMUEXT_TLB_FLUSH_ALL;
1327         MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
1328 
1329         xen_mc_issue(PARAVIRT_LAZY_MMU);
1330 
1331         preempt_enable();
1332 }
1333 static void xen_flush_tlb(void)
1334 {
1335         struct mmuext_op *op;
1336         struct multicall_space mcs;
1337 
1338         trace_xen_mmu_flush_tlb(0);
1339 
1340         preempt_disable();
1341 
1342         mcs = xen_mc_entry(sizeof(*op));
1343 
1344         op = mcs.args;
1345         op->cmd = MMUEXT_TLB_FLUSH_LOCAL;
1346         MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
1347 
1348         xen_mc_issue(PARAVIRT_LAZY_MMU);
1349 
1350         preempt_enable();
1351 }
1352 
1353 static void xen_flush_tlb_single(unsigned long addr)
1354 {
1355         struct mmuext_op *op;
1356         struct multicall_space mcs;
1357 
1358         trace_xen_mmu_flush_tlb_single(addr);
1359 
1360         preempt_disable();
1361 
1362         mcs = xen_mc_entry(sizeof(*op));
1363         op = mcs.args;
1364         op->cmd = MMUEXT_INVLPG_LOCAL;
1365         op->arg1.linear_addr = addr & PAGE_MASK;
1366         MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
1367 
1368         xen_mc_issue(PARAVIRT_LAZY_MMU);
1369 
1370         preempt_enable();
1371 }
1372 
1373 static void xen_flush_tlb_others(const struct cpumask *cpus,
1374                                  struct mm_struct *mm, unsigned long start,
1375                                  unsigned long end)
1376 {
1377         struct {
1378                 struct mmuext_op op;
1379 #ifdef CONFIG_SMP
1380                 DECLARE_BITMAP(mask, num_processors);
1381 #else
1382                 DECLARE_BITMAP(mask, NR_CPUS);
1383 #endif
1384         } *args;
1385         struct multicall_space mcs;
1386 
1387         trace_xen_mmu_flush_tlb_others(cpus, mm, start, end);
1388 
1389         if (cpumask_empty(cpus))
1390                 return;         /* nothing to do */
1391 
1392         mcs = xen_mc_entry(sizeof(*args));
1393         args = mcs.args;
1394         args->op.arg2.vcpumask = to_cpumask(args->mask);
1395 
1396         /* Remove us, and any offline CPUS. */
1397         cpumask_and(to_cpumask(args->mask), cpus, cpu_online_mask);
1398         cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask));
1399 
1400         args->op.cmd = MMUEXT_TLB_FLUSH_MULTI;
1401         if (end != TLB_FLUSH_ALL && (end - start) <= PAGE_SIZE) {
1402                 args->op.cmd = MMUEXT_INVLPG_MULTI;
1403                 args->op.arg1.linear_addr = start;
1404         }
1405 
1406         MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);
1407 
1408         xen_mc_issue(PARAVIRT_LAZY_MMU);
1409 }
1410 
1411 static unsigned long xen_read_cr3(void)
1412 {
1413         return this_cpu_read(xen_cr3);
1414 }
1415 
1416 static void set_current_cr3(void *v)
1417 {
1418         this_cpu_write(xen_current_cr3, (unsigned long)v);
1419 }
1420 
1421 static void __xen_write_cr3(bool kernel, unsigned long cr3)
1422 {
1423         struct mmuext_op op;
1424         unsigned long mfn;
1425 
1426         trace_xen_mmu_write_cr3(kernel, cr3);
1427 
1428         if (cr3)
1429                 mfn = pfn_to_mfn(PFN_DOWN(cr3));
1430         else
1431                 mfn = 0;
1432 
1433         WARN_ON(mfn == 0 && kernel);
1434 
1435         op.cmd = kernel ? MMUEXT_NEW_BASEPTR : MMUEXT_NEW_USER_BASEPTR;
1436         op.arg1.mfn = mfn;
1437 
1438         xen_extend_mmuext_op(&op);
1439 
1440         if (kernel) {
1441                 this_cpu_write(xen_cr3, cr3);
1442 
1443                 /* Update xen_current_cr3 once the batch has actually
1444                    been submitted. */
1445                 xen_mc_callback(set_current_cr3, (void *)cr3);
1446         }
1447 }
1448 static void xen_write_cr3(unsigned long cr3)
1449 {
1450         BUG_ON(preemptible());
1451 
1452         xen_mc_batch();  /* disables interrupts */
1453 
1454         /* Update while interrupts are disabled, so its atomic with
1455            respect to ipis */
1456         this_cpu_write(xen_cr3, cr3);
1457 
1458         __xen_write_cr3(true, cr3);
1459 
1460 #ifdef CONFIG_X86_64
1461         {
1462                 pgd_t *user_pgd = xen_get_user_pgd(__va(cr3));
1463                 if (user_pgd)
1464                         __xen_write_cr3(false, __pa(user_pgd));
1465                 else
1466                         __xen_write_cr3(false, 0);
1467         }
1468 #endif
1469 
1470         xen_mc_issue(PARAVIRT_LAZY_CPU);  /* interrupts restored */
1471 }
1472 
1473 #ifdef CONFIG_X86_64
1474 /*
1475  * At the start of the day - when Xen launches a guest, it has already
1476  * built pagetables for the guest. We diligently look over them
1477  * in xen_setup_kernel_pagetable and graft as appropriate them in the
1478  * init_level4_pgt and its friends. Then when we are happy we load
1479  * the new init_level4_pgt - and continue on.
1480  *
1481  * The generic code starts (start_kernel) and 'init_mem_mapping' sets
1482  * up the rest of the pagetables. When it has completed it loads the cr3.
1483  * N.B. that baremetal would start at 'start_kernel' (and the early
1484  * #PF handler would create bootstrap pagetables) - so we are running
1485  * with the same assumptions as what to do when write_cr3 is executed
1486  * at this point.
1487  *
1488  * Since there are no user-page tables at all, we have two variants
1489  * of xen_write_cr3 - the early bootup (this one), and the late one
1490  * (xen_write_cr3). The reason we have to do that is that in 64-bit
1491  * the Linux kernel and user-space are both in ring 3 while the
1492  * hypervisor is in ring 0.
1493  */
1494 static void __init xen_write_cr3_init(unsigned long cr3)
1495 {
1496         BUG_ON(preemptible());
1497 
1498         xen_mc_batch();  /* disables interrupts */
1499 
1500         /* Update while interrupts are disabled, so its atomic with
1501            respect to ipis */
1502         this_cpu_write(xen_cr3, cr3);
1503 
1504         __xen_write_cr3(true, cr3);
1505 
1506         xen_mc_issue(PARAVIRT_LAZY_CPU);  /* interrupts restored */
1507 }
1508 #endif
1509 
1510 static int xen_pgd_alloc(struct mm_struct *mm)
1511 {
1512         pgd_t *pgd = mm->pgd;
1513         int ret = 0;
1514 
1515         BUG_ON(PagePinned(virt_to_page(pgd)));
1516 
1517 #ifdef CONFIG_X86_64
1518         {
1519                 struct page *page = virt_to_page(pgd);
1520                 pgd_t *user_pgd;
1521 
1522                 BUG_ON(page->private != 0);
1523 
1524                 ret = -ENOMEM;
1525 
1526                 user_pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
1527                 page->private = (unsigned long)user_pgd;
1528 
1529                 if (user_pgd != NULL) {
1530 #ifdef CONFIG_X86_VSYSCALL_EMULATION
1531                         user_pgd[pgd_index(VSYSCALL_ADDR)] =
1532                                 __pgd(__pa(level3_user_vsyscall) | _PAGE_TABLE);
1533 #endif
1534                         ret = 0;
1535                 }
1536 
1537                 BUG_ON(PagePinned(virt_to_page(xen_get_user_pgd(pgd))));
1538         }
1539 #endif
1540 
1541         return ret;
1542 }
1543 
1544 static void xen_pgd_free(struct mm_struct *mm, pgd_t *pgd)
1545 {
1546 #ifdef CONFIG_X86_64
1547         pgd_t *user_pgd = xen_get_user_pgd(pgd);
1548 
1549         if (user_pgd)
1550                 free_page((unsigned long)user_pgd);
1551 #endif
1552 }
1553 
1554 /*
1555  * Init-time set_pte while constructing initial pagetables, which
1556  * doesn't allow RO page table pages to be remapped RW.
1557  *
1558  * If there is no MFN for this PFN then this page is initially
1559  * ballooned out so clear the PTE (as in decrease_reservation() in
1560  * drivers/xen/balloon.c).
1561  *
1562  * Many of these PTE updates are done on unpinned and writable pages
1563  * and doing a hypercall for these is unnecessary and expensive.  At
1564  * this point it is not possible to tell if a page is pinned or not,
1565  * so always write the PTE directly and rely on Xen trapping and
1566  * emulating any updates as necessary.
1567  */
1568 __visible pte_t xen_make_pte_init(pteval_t pte)
1569 {
1570 #ifdef CONFIG_X86_64
1571         unsigned long pfn;
1572 
1573         /*
1574          * Pages belonging to the initial p2m list mapped outside the default
1575          * address range must be mapped read-only. This region contains the
1576          * page tables for mapping the p2m list, too, and page tables MUST be
1577          * mapped read-only.
1578          */
1579         pfn = (pte & PTE_PFN_MASK) >> PAGE_SHIFT;
1580         if (xen_start_info->mfn_list < __START_KERNEL_map &&
1581             pfn >= xen_start_info->first_p2m_pfn &&
1582             pfn < xen_start_info->first_p2m_pfn + xen_start_info->nr_p2m_frames)
1583                 pte &= ~_PAGE_RW;
1584 #endif
1585         pte = pte_pfn_to_mfn(pte);
1586         return native_make_pte(pte);
1587 }
1588 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte_init);
1589 
1590 static void __init xen_set_pte_init(pte_t *ptep, pte_t pte)
1591 {
1592 #ifdef CONFIG_X86_32
1593         /* If there's an existing pte, then don't allow _PAGE_RW to be set */
1594         if (pte_mfn(pte) != INVALID_P2M_ENTRY
1595             && pte_val_ma(*ptep) & _PAGE_PRESENT)
1596                 pte = __pte_ma(((pte_val_ma(*ptep) & _PAGE_RW) | ~_PAGE_RW) &
1597                                pte_val_ma(pte));
1598 #endif
1599         native_set_pte(ptep, pte);
1600 }
1601 
1602 /* Early in boot, while setting up the initial pagetable, assume
1603    everything is pinned. */
1604 static void __init xen_alloc_pte_init(struct mm_struct *mm, unsigned long pfn)
1605 {
1606 #ifdef CONFIG_FLATMEM
1607         BUG_ON(mem_map);        /* should only be used early */
1608 #endif
1609         make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
1610         pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn);
1611 }
1612 
1613 /* Used for pmd and pud */
1614 static void __init xen_alloc_pmd_init(struct mm_struct *mm, unsigned long pfn)
1615 {
1616 #ifdef CONFIG_FLATMEM
1617         BUG_ON(mem_map);        /* should only be used early */
1618 #endif
1619         make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
1620 }
1621 
1622 /* Early release_pte assumes that all pts are pinned, since there's
1623    only init_mm and anything attached to that is pinned. */
1624 static void __init xen_release_pte_init(unsigned long pfn)
1625 {
1626         pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn);
1627         make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
1628 }
1629 
1630 static void __init xen_release_pmd_init(unsigned long pfn)
1631 {
1632         make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
1633 }
1634 
1635 static inline void __pin_pagetable_pfn(unsigned cmd, unsigned long pfn)
1636 {
1637         struct multicall_space mcs;
1638         struct mmuext_op *op;
1639 
1640         mcs = __xen_mc_entry(sizeof(*op));
1641         op = mcs.args;
1642         op->cmd = cmd;
1643         op->arg1.mfn = pfn_to_mfn(pfn);
1644 
1645         MULTI_mmuext_op(mcs.mc, mcs.args, 1, NULL, DOMID_SELF);
1646 }
1647 
1648 static inline void __set_pfn_prot(unsigned long pfn, pgprot_t prot)
1649 {
1650         struct multicall_space mcs;
1651         unsigned long addr = (unsigned long)__va(pfn << PAGE_SHIFT);
1652 
1653         mcs = __xen_mc_entry(0);
1654         MULTI_update_va_mapping(mcs.mc, (unsigned long)addr,
1655                                 pfn_pte(pfn, prot), 0);
1656 }
1657 
1658 /* This needs to make sure the new pte page is pinned iff its being
1659    attached to a pinned pagetable. */
1660 static inline void xen_alloc_ptpage(struct mm_struct *mm, unsigned long pfn,
1661                                     unsigned level)
1662 {
1663         bool pinned = PagePinned(virt_to_page(mm->pgd));
1664 
1665         trace_xen_mmu_alloc_ptpage(mm, pfn, level, pinned);
1666 
1667         if (pinned) {
1668                 struct page *page = pfn_to_page(pfn);
1669 
1670                 SetPagePinned(page);
1671 
1672                 if (!PageHighMem(page)) {
1673                         xen_mc_batch();
1674 
1675                         __set_pfn_prot(pfn, PAGE_KERNEL_RO);
1676 
1677                         if (level == PT_PTE && USE_SPLIT_PTE_PTLOCKS)
1678                                 __pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn);
1679 
1680                         xen_mc_issue(PARAVIRT_LAZY_MMU);
1681                 } else {
1682                         /* make sure there are no stray mappings of
1683                            this page */
1684                         kmap_flush_unused();
1685                 }
1686         }
1687 }
1688 
1689 static void xen_alloc_pte(struct mm_struct *mm, unsigned long pfn)
1690 {
1691         xen_alloc_ptpage(mm, pfn, PT_PTE);
1692 }
1693 
1694 static void xen_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
1695 {
1696         xen_alloc_ptpage(mm, pfn, PT_PMD);
1697 }
1698 
1699 /* This should never happen until we're OK to use struct page */
1700 static inline void xen_release_ptpage(unsigned long pfn, unsigned level)
1701 {
1702         struct page *page = pfn_to_page(pfn);
1703         bool pinned = PagePinned(page);
1704 
1705         trace_xen_mmu_release_ptpage(pfn, level, pinned);
1706 
1707         if (pinned) {
1708                 if (!PageHighMem(page)) {
1709                         xen_mc_batch();
1710 
1711                         if (level == PT_PTE && USE_SPLIT_PTE_PTLOCKS)
1712                                 __pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn);
1713 
1714                         __set_pfn_prot(pfn, PAGE_KERNEL);
1715 
1716                         xen_mc_issue(PARAVIRT_LAZY_MMU);
1717                 }
1718                 ClearPagePinned(page);
1719         }
1720 }
1721 
1722 static void xen_release_pte(unsigned long pfn)
1723 {
1724         xen_release_ptpage(pfn, PT_PTE);
1725 }
1726 
1727 static void xen_release_pmd(unsigned long pfn)
1728 {
1729         xen_release_ptpage(pfn, PT_PMD);
1730 }
1731 
1732 #if CONFIG_PGTABLE_LEVELS == 4
1733 static void xen_alloc_pud(struct mm_struct *mm, unsigned long pfn)
1734 {
1735         xen_alloc_ptpage(mm, pfn, PT_PUD);
1736 }
1737 
1738 static void xen_release_pud(unsigned long pfn)
1739 {
1740         xen_release_ptpage(pfn, PT_PUD);
1741 }
1742 #endif
1743 
1744 void __init xen_reserve_top(void)
1745 {
1746 #ifdef CONFIG_X86_32
1747         unsigned long top = HYPERVISOR_VIRT_START;
1748         struct xen_platform_parameters pp;
1749 
1750         if (HYPERVISOR_xen_version(XENVER_platform_parameters, &pp) == 0)
1751                 top = pp.virt_start;
1752 
1753         reserve_top_address(-top);
1754 #endif  /* CONFIG_X86_32 */
1755 }
1756 
1757 /*
1758  * Like __va(), but returns address in the kernel mapping (which is
1759  * all we have until the physical memory mapping has been set up.
1760  */
1761 static void * __init __ka(phys_addr_t paddr)
1762 {
1763 #ifdef CONFIG_X86_64
1764         return (void *)(paddr + __START_KERNEL_map);
1765 #else
1766         return __va(paddr);
1767 #endif
1768 }
1769 
1770 /* Convert a machine address to physical address */
1771 static unsigned long __init m2p(phys_addr_t maddr)
1772 {
1773         phys_addr_t paddr;
1774 
1775         maddr &= PTE_PFN_MASK;
1776         paddr = mfn_to_pfn(maddr >> PAGE_SHIFT) << PAGE_SHIFT;
1777 
1778         return paddr;
1779 }
1780 
1781 /* Convert a machine address to kernel virtual */
1782 static void * __init m2v(phys_addr_t maddr)
1783 {
1784         return __ka(m2p(maddr));
1785 }
1786 
1787 /* Set the page permissions on an identity-mapped pages */
1788 static void __init set_page_prot_flags(void *addr, pgprot_t prot,
1789                                        unsigned long flags)
1790 {
1791         unsigned long pfn = __pa(addr) >> PAGE_SHIFT;
1792         pte_t pte = pfn_pte(pfn, prot);
1793 
1794         /* For PVH no need to set R/O or R/W to pin them or unpin them. */
1795         if (xen_feature(XENFEAT_auto_translated_physmap))
1796                 return;
1797 
1798         if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, flags))
1799                 BUG();
1800 }
1801 static void __init set_page_prot(void *addr, pgprot_t prot)
1802 {
1803         return set_page_prot_flags(addr, prot, UVMF_NONE);
1804 }
1805 #ifdef CONFIG_X86_32
1806 static void __init xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn)
1807 {
1808         unsigned pmdidx, pteidx;
1809         unsigned ident_pte;
1810         unsigned long pfn;
1811 
1812         level1_ident_pgt = extend_brk(sizeof(pte_t) * LEVEL1_IDENT_ENTRIES,
1813                                       PAGE_SIZE);
1814 
1815         ident_pte = 0;
1816         pfn = 0;
1817         for (pmdidx = 0; pmdidx < PTRS_PER_PMD && pfn < max_pfn; pmdidx++) {
1818                 pte_t *pte_page;
1819 
1820                 /* Reuse or allocate a page of ptes */
1821                 if (pmd_present(pmd[pmdidx]))
1822                         pte_page = m2v(pmd[pmdidx].pmd);
1823                 else {
1824                         /* Check for free pte pages */
1825                         if (ident_pte == LEVEL1_IDENT_ENTRIES)
1826                                 break;
1827 
1828                         pte_page = &level1_ident_pgt[ident_pte];
1829                         ident_pte += PTRS_PER_PTE;
1830 
1831                         pmd[pmdidx] = __pmd(__pa(pte_page) | _PAGE_TABLE);
1832                 }
1833 
1834                 /* Install mappings */
1835                 for (pteidx = 0; pteidx < PTRS_PER_PTE; pteidx++, pfn++) {
1836                         pte_t pte;
1837 
1838                         if (pfn > max_pfn_mapped)
1839                                 max_pfn_mapped = pfn;
1840 
1841                         if (!pte_none(pte_page[pteidx]))
1842                                 continue;
1843 
1844                         pte = pfn_pte(pfn, PAGE_KERNEL_EXEC);
1845                         pte_page[pteidx] = pte;
1846                 }
1847         }
1848 
1849         for (pteidx = 0; pteidx < ident_pte; pteidx += PTRS_PER_PTE)
1850                 set_page_prot(&level1_ident_pgt[pteidx], PAGE_KERNEL_RO);
1851 
1852         set_page_prot(pmd, PAGE_KERNEL_RO);
1853 }
1854 #endif
1855 void __init xen_setup_machphys_mapping(void)
1856 {
1857         struct xen_machphys_mapping mapping;
1858 
1859         if (HYPERVISOR_memory_op(XENMEM_machphys_mapping, &mapping) == 0) {
1860                 machine_to_phys_mapping = (unsigned long *)mapping.v_start;
1861                 machine_to_phys_nr = mapping.max_mfn + 1;
1862         } else {
1863                 machine_to_phys_nr = MACH2PHYS_NR_ENTRIES;
1864         }
1865 #ifdef CONFIG_X86_32
1866         WARN_ON((machine_to_phys_mapping + (machine_to_phys_nr - 1))
1867                 < machine_to_phys_mapping);
1868 #endif
1869 }
1870 
1871 #ifdef CONFIG_X86_64
1872 static void __init convert_pfn_mfn(void *v)
1873 {
1874         pte_t *pte = v;
1875         int i;
1876 
1877         /* All levels are converted the same way, so just treat them
1878            as ptes. */
1879         for (i = 0; i < PTRS_PER_PTE; i++)
1880                 pte[i] = xen_make_pte(pte[i].pte);
1881 }
1882 static void __init check_pt_base(unsigned long *pt_base, unsigned long *pt_end,
1883                                  unsigned long addr)
1884 {
1885         if (*pt_base == PFN_DOWN(__pa(addr))) {
1886                 set_page_prot_flags((void *)addr, PAGE_KERNEL, UVMF_INVLPG);
1887                 clear_page((void *)addr);
1888                 (*pt_base)++;
1889         }
1890         if (*pt_end == PFN_DOWN(__pa(addr))) {
1891                 set_page_prot_flags((void *)addr, PAGE_KERNEL, UVMF_INVLPG);
1892                 clear_page((void *)addr);
1893                 (*pt_end)--;
1894         }
1895 }
1896 /*
1897  * Set up the initial kernel pagetable.
1898  *
1899  * We can construct this by grafting the Xen provided pagetable into
1900  * head_64.S's preconstructed pagetables.  We copy the Xen L2's into
1901  * level2_ident_pgt, and level2_kernel_pgt.  This means that only the
1902  * kernel has a physical mapping to start with - but that's enough to
1903  * get __va working.  We need to fill in the rest of the physical
1904  * mapping once some sort of allocator has been set up.  NOTE: for
1905  * PVH, the page tables are native.
1906  */
1907 void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
1908 {
1909         pud_t *l3;
1910         pmd_t *l2;
1911         unsigned long addr[3];
1912         unsigned long pt_base, pt_end;
1913         unsigned i;
1914 
1915         /* max_pfn_mapped is the last pfn mapped in the initial memory
1916          * mappings. Considering that on Xen after the kernel mappings we
1917          * have the mappings of some pages that don't exist in pfn space, we
1918          * set max_pfn_mapped to the last real pfn mapped. */
1919         if (xen_start_info->mfn_list < __START_KERNEL_map)
1920                 max_pfn_mapped = xen_start_info->first_p2m_pfn;
1921         else
1922                 max_pfn_mapped = PFN_DOWN(__pa(xen_start_info->mfn_list));
1923 
1924         pt_base = PFN_DOWN(__pa(xen_start_info->pt_base));
1925         pt_end = pt_base + xen_start_info->nr_pt_frames;
1926 
1927         /* Zap identity mapping */
1928         init_level4_pgt[0] = __pgd(0);
1929 
1930         if (!xen_feature(XENFEAT_auto_translated_physmap)) {
1931                 /* Pre-constructed entries are in pfn, so convert to mfn */
1932                 /* L4[272] -> level3_ident_pgt
1933                  * L4[511] -> level3_kernel_pgt */
1934                 convert_pfn_mfn(init_level4_pgt);
1935 
1936                 /* L3_i[0] -> level2_ident_pgt */
1937                 convert_pfn_mfn(level3_ident_pgt);
1938                 /* L3_k[510] -> level2_kernel_pgt
1939                  * L3_k[511] -> level2_fixmap_pgt */
1940                 convert_pfn_mfn(level3_kernel_pgt);
1941 
1942                 /* L3_k[511][506] -> level1_fixmap_pgt */
1943                 convert_pfn_mfn(level2_fixmap_pgt);
1944         }
1945         /* We get [511][511] and have Xen's version of level2_kernel_pgt */
1946         l3 = m2v(pgd[pgd_index(__START_KERNEL_map)].pgd);
1947         l2 = m2v(l3[pud_index(__START_KERNEL_map)].pud);
1948 
1949         addr[0] = (unsigned long)pgd;
1950         addr[1] = (unsigned long)l3;
1951         addr[2] = (unsigned long)l2;
1952         /* Graft it onto L4[272][0]. Note that we creating an aliasing problem:
1953          * Both L4[272][0] and L4[511][510] have entries that point to the same
1954          * L2 (PMD) tables. Meaning that if you modify it in __va space
1955          * it will be also modified in the __ka space! (But if you just
1956          * modify the PMD table to point to other PTE's or none, then you
1957          * are OK - which is what cleanup_highmap does) */
1958         copy_page(level2_ident_pgt, l2);
1959         /* Graft it onto L4[511][510] */
1960         copy_page(level2_kernel_pgt, l2);
1961 
1962         /* Copy the initial P->M table mappings if necessary. */
1963         i = pgd_index(xen_start_info->mfn_list);
1964         if (i && i < pgd_index(__START_KERNEL_map))
1965                 init_level4_pgt[i] = ((pgd_t *)xen_start_info->pt_base)[i];
1966 
1967         if (!xen_feature(XENFEAT_auto_translated_physmap)) {
1968                 /* Make pagetable pieces RO */
1969                 set_page_prot(init_level4_pgt, PAGE_KERNEL_RO);
1970                 set_page_prot(level3_ident_pgt, PAGE_KERNEL_RO);
1971                 set_page_prot(level3_kernel_pgt, PAGE_KERNEL_RO);
1972                 set_page_prot(level3_user_vsyscall, PAGE_KERNEL_RO);
1973                 set_page_prot(level2_ident_pgt, PAGE_KERNEL_RO);
1974                 set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO);
1975                 set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO);
1976                 set_page_prot(level1_fixmap_pgt, PAGE_KERNEL_RO);
1977 
1978                 /* Pin down new L4 */
1979                 pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE,
1980                                   PFN_DOWN(__pa_symbol(init_level4_pgt)));
1981 
1982                 /* Unpin Xen-provided one */
1983                 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
1984 
1985                 /*
1986                  * At this stage there can be no user pgd, and no page
1987                  * structure to attach it to, so make sure we just set kernel
1988                  * pgd.
1989                  */
1990                 xen_mc_batch();
1991                 __xen_write_cr3(true, __pa(init_level4_pgt));
1992                 xen_mc_issue(PARAVIRT_LAZY_CPU);
1993         } else
1994                 native_write_cr3(__pa(init_level4_pgt));
1995 
1996         /* We can't that easily rip out L3 and L2, as the Xen pagetables are
1997          * set out this way: [L4], [L1], [L2], [L3], [L1], [L1] ...  for
1998          * the initial domain. For guests using the toolstack, they are in:
1999          * [L4], [L3], [L2], [L1], [L1], order .. So for dom0 we can only
2000          * rip out the [L4] (pgd), but for guests we shave off three pages.
2001          */
2002         for (i = 0; i < ARRAY_SIZE(addr); i++)
2003                 check_pt_base(&pt_base, &pt_end, addr[i]);
2004 
2005         /* Our (by three pages) smaller Xen pagetable that we are using */
2006         xen_pt_base = PFN_PHYS(pt_base);
2007         xen_pt_size = (pt_end - pt_base) * PAGE_SIZE;
2008         memblock_reserve(xen_pt_base, xen_pt_size);
2009 
2010         /* Revector the xen_start_info */
2011         xen_start_info = (struct start_info *)__va(__pa(xen_start_info));
2012 }
2013 
2014 /*
2015  * Read a value from a physical address.
2016  */
2017 static unsigned long __init xen_read_phys_ulong(phys_addr_t addr)
2018 {
2019         unsigned long *vaddr;
2020         unsigned long val;
2021 
2022         vaddr = early_memremap_ro(addr, sizeof(val));
2023         val = *vaddr;
2024         early_memunmap(vaddr, sizeof(val));
2025         return val;
2026 }
2027 
2028 /*
2029  * Translate a virtual address to a physical one without relying on mapped
2030  * page tables.
2031  */
2032 static phys_addr_t __init xen_early_virt_to_phys(unsigned long vaddr)
2033 {
2034         phys_addr_t pa;
2035         pgd_t pgd;
2036         pud_t pud;
2037         pmd_t pmd;
2038         pte_t pte;
2039 
2040         pa = read_cr3();
2041         pgd = native_make_pgd(xen_read_phys_ulong(pa + pgd_index(vaddr) *
2042                                                        sizeof(pgd)));
2043         if (!pgd_present(pgd))
2044                 return 0;
2045 
2046         pa = pgd_val(pgd) & PTE_PFN_MASK;
2047         pud = native_make_pud(xen_read_phys_ulong(pa + pud_index(vaddr) *
2048                                                        sizeof(pud)));
2049         if (!pud_present(pud))
2050                 return 0;
2051         pa = pud_pfn(pud) << PAGE_SHIFT;
2052         if (pud_large(pud))
2053                 return pa + (vaddr & ~PUD_MASK);
2054 
2055         pmd = native_make_pmd(xen_read_phys_ulong(pa + pmd_index(vaddr) *
2056                                                        sizeof(pmd)));
2057         if (!pmd_present(pmd))
2058                 return 0;
2059         pa = pmd_pfn(pmd) << PAGE_SHIFT;
2060         if (pmd_large(pmd))
2061                 return pa + (vaddr & ~PMD_MASK);
2062 
2063         pte = native_make_pte(xen_read_phys_ulong(pa + pte_index(vaddr) *
2064                                                        sizeof(pte)));
2065         if (!pte_present(pte))
2066                 return 0;
2067         pa = pte_pfn(pte) << PAGE_SHIFT;
2068 
2069         return pa | (vaddr & ~PAGE_MASK);
2070 }
2071 
2072 /*
2073  * Find a new area for the hypervisor supplied p2m list and relocate the p2m to
2074  * this area.
2075  */
2076 void __init xen_relocate_p2m(void)
2077 {
2078         phys_addr_t size, new_area, pt_phys, pmd_phys, pud_phys;
2079         unsigned long p2m_pfn, p2m_pfn_end, n_frames, pfn, pfn_end;
2080         int n_pte, n_pt, n_pmd, n_pud, idx_pte, idx_pt, idx_pmd, idx_pud;
2081         pte_t *pt;
2082         pmd_t *pmd;
2083         pud_t *pud;
2084         pgd_t *pgd;
2085         unsigned long *new_p2m;
2086 
2087         size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long));
2088         n_pte = roundup(size, PAGE_SIZE) >> PAGE_SHIFT;
2089         n_pt = roundup(size, PMD_SIZE) >> PMD_SHIFT;
2090         n_pmd = roundup(size, PUD_SIZE) >> PUD_SHIFT;
2091         n_pud = roundup(size, PGDIR_SIZE) >> PGDIR_SHIFT;
2092         n_frames = n_pte + n_pt + n_pmd + n_pud;
2093 
2094         new_area = xen_find_free_area(PFN_PHYS(n_frames));
2095         if (!new_area) {
2096                 xen_raw_console_write("Can't find new memory area for p2m needed due to E820 map conflict\n");
2097                 BUG();
2098         }
2099 
2100         /*
2101          * Setup the page tables for addressing the new p2m list.
2102          * We have asked the hypervisor to map the p2m list at the user address
2103          * PUD_SIZE. It may have done so, or it may have used a kernel space
2104          * address depending on the Xen version.
2105          * To avoid any possible virtual address collision, just use
2106          * 2 * PUD_SIZE for the new area.
2107          */
2108         pud_phys = new_area;
2109         pmd_phys = pud_phys + PFN_PHYS(n_pud);
2110         pt_phys = pmd_phys + PFN_PHYS(n_pmd);
2111         p2m_pfn = PFN_DOWN(pt_phys) + n_pt;
2112 
2113         pgd = __va(read_cr3());
2114         new_p2m = (unsigned long *)(2 * PGDIR_SIZE);
2115         for (idx_pud = 0; idx_pud < n_pud; idx_pud++) {
2116                 pud = early_memremap(pud_phys, PAGE_SIZE);
2117                 clear_page(pud);
2118                 for (idx_pmd = 0; idx_pmd < min(n_pmd, PTRS_PER_PUD);
2119                      idx_pmd++) {
2120                         pmd = early_memremap(pmd_phys, PAGE_SIZE);
2121                         clear_page(pmd);
2122                         for (idx_pt = 0; idx_pt < min(n_pt, PTRS_PER_PMD);
2123                              idx_pt++) {
2124                                 pt = early_memremap(pt_phys, PAGE_SIZE);
2125                                 clear_page(pt);
2126                                 for (idx_pte = 0;
2127                                      idx_pte < min(n_pte, PTRS_PER_PTE);
2128                                      idx_pte++) {
2129                                         set_pte(pt + idx_pte,
2130                                                 pfn_pte(p2m_pfn, PAGE_KERNEL));
2131                                         p2m_pfn++;
2132                                 }
2133                                 n_pte -= PTRS_PER_PTE;
2134                                 early_memunmap(pt, PAGE_SIZE);
2135                                 make_lowmem_page_readonly(__va(pt_phys));
2136                                 pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE,
2137                                                   PFN_DOWN(pt_phys));
2138                                 set_pmd(pmd + idx_pt,
2139                                         __pmd(_PAGE_TABLE | pt_phys));
2140                                 pt_phys += PAGE_SIZE;
2141                         }
2142                         n_pt -= PTRS_PER_PMD;
2143                         early_memunmap(pmd, PAGE_SIZE);
2144                         make_lowmem_page_readonly(__va(pmd_phys));
2145                         pin_pagetable_pfn(MMUEXT_PIN_L2_TABLE,
2146                                           PFN_DOWN(pmd_phys));
2147                         set_pud(pud + idx_pmd, __pud(_PAGE_TABLE | pmd_phys));
2148                         pmd_phys += PAGE_SIZE;
2149                 }
2150                 n_pmd -= PTRS_PER_PUD;
2151                 early_memunmap(pud, PAGE_SIZE);
2152                 make_lowmem_page_readonly(__va(pud_phys));
2153                 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, PFN_DOWN(pud_phys));
2154                 set_pgd(pgd + 2 + idx_pud, __pgd(_PAGE_TABLE | pud_phys));
2155                 pud_phys += PAGE_SIZE;
2156         }
2157 
2158         /* Now copy the old p2m info to the new area. */
2159         memcpy(new_p2m, xen_p2m_addr, size);
2160         xen_p2m_addr = new_p2m;
2161 
2162         /* Release the old p2m list and set new list info. */
2163         p2m_pfn = PFN_DOWN(xen_early_virt_to_phys(xen_start_info->mfn_list));
2164         BUG_ON(!p2m_pfn);
2165         p2m_pfn_end = p2m_pfn + PFN_DOWN(size);
2166 
2167         if (xen_start_info->mfn_list < __START_KERNEL_map) {
2168                 pfn = xen_start_info->first_p2m_pfn;
2169                 pfn_end = xen_start_info->first_p2m_pfn +
2170                           xen_start_info->nr_p2m_frames;
2171                 set_pgd(pgd + 1, __pgd(0));
2172         } else {
2173                 pfn = p2m_pfn;
2174                 pfn_end = p2m_pfn_end;
2175         }
2176 
2177         memblock_free(PFN_PHYS(pfn), PAGE_SIZE * (pfn_end - pfn));
2178         while (pfn < pfn_end) {
2179                 if (pfn == p2m_pfn) {
2180                         pfn = p2m_pfn_end;
2181                         continue;
2182                 }
2183                 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
2184                 pfn++;
2185         }
2186 
2187         xen_start_info->mfn_list = (unsigned long)xen_p2m_addr;
2188         xen_start_info->first_p2m_pfn =  PFN_DOWN(new_area);
2189         xen_start_info->nr_p2m_frames = n_frames;
2190 }
2191 
2192 #else   /* !CONFIG_X86_64 */
2193 static RESERVE_BRK_ARRAY(pmd_t, initial_kernel_pmd, PTRS_PER_PMD);
2194 static RESERVE_BRK_ARRAY(pmd_t, swapper_kernel_pmd, PTRS_PER_PMD);
2195 
2196 static void __init xen_write_cr3_init(unsigned long cr3)
2197 {
2198         unsigned long pfn = PFN_DOWN(__pa(swapper_pg_dir));
2199 
2200         BUG_ON(read_cr3() != __pa(initial_page_table));
2201         BUG_ON(cr3 != __pa(swapper_pg_dir));
2202 
2203         /*
2204          * We are switching to swapper_pg_dir for the first time (from
2205          * initial_page_table) and therefore need to mark that page
2206          * read-only and then pin it.
2207          *
2208          * Xen disallows sharing of kernel PMDs for PAE
2209          * guests. Therefore we must copy the kernel PMD from
2210          * initial_page_table into a new kernel PMD to be used in
2211          * swapper_pg_dir.
2212          */
2213         swapper_kernel_pmd =
2214                 extend_brk(sizeof(pmd_t) * PTRS_PER_PMD, PAGE_SIZE);
2215         copy_page(swapper_kernel_pmd, initial_kernel_pmd);
2216         swapper_pg_dir[KERNEL_PGD_BOUNDARY] =
2217                 __pgd(__pa(swapper_kernel_pmd) | _PAGE_PRESENT);
2218         set_page_prot(swapper_kernel_pmd, PAGE_KERNEL_RO);
2219 
2220         set_page_prot(swapper_pg_dir, PAGE_KERNEL_RO);
2221         xen_write_cr3(cr3);
2222         pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, pfn);
2223 
2224         pin_pagetable_pfn(MMUEXT_UNPIN_TABLE,
2225                           PFN_DOWN(__pa(initial_page_table)));
2226         set_page_prot(initial_page_table, PAGE_KERNEL);
2227         set_page_prot(initial_kernel_pmd, PAGE_KERNEL);
2228 
2229         pv_mmu_ops.write_cr3 = &xen_write_cr3;
2230 }
2231 
2232 /*
2233  * For 32 bit domains xen_start_info->pt_base is the pgd address which might be
2234  * not the first page table in the page table pool.
2235  * Iterate through the initial page tables to find the real page table base.
2236  */
2237 static phys_addr_t xen_find_pt_base(pmd_t *pmd)
2238 {
2239         phys_addr_t pt_base, paddr;
2240         unsigned pmdidx;
2241 
2242         pt_base = min(__pa(xen_start_info->pt_base), __pa(pmd));
2243 
2244         for (pmdidx = 0; pmdidx < PTRS_PER_PMD; pmdidx++)
2245                 if (pmd_present(pmd[pmdidx]) && !pmd_large(pmd[pmdidx])) {
2246                         paddr = m2p(pmd[pmdidx].pmd);
2247                         pt_base = min(pt_base, paddr);
2248                 }
2249 
2250         return pt_base;
2251 }
2252 
2253 void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
2254 {
2255         pmd_t *kernel_pmd;
2256 
2257         kernel_pmd = m2v(pgd[KERNEL_PGD_BOUNDARY].pgd);
2258 
2259         xen_pt_base = xen_find_pt_base(kernel_pmd);
2260         xen_pt_size = xen_start_info->nr_pt_frames * PAGE_SIZE;
2261 
2262         initial_kernel_pmd =
2263                 extend_brk(sizeof(pmd_t) * PTRS_PER_PMD, PAGE_SIZE);
2264 
2265         max_pfn_mapped = PFN_DOWN(xen_pt_base + xen_pt_size + 512 * 1024);
2266 
2267         copy_page(initial_kernel_pmd, kernel_pmd);
2268 
2269         xen_map_identity_early(initial_kernel_pmd, max_pfn);
2270 
2271         copy_page(initial_page_table, pgd);
2272         initial_page_table[KERNEL_PGD_BOUNDARY] =
2273                 __pgd(__pa(initial_kernel_pmd) | _PAGE_PRESENT);
2274 
2275         set_page_prot(initial_kernel_pmd, PAGE_KERNEL_RO);
2276         set_page_prot(initial_page_table, PAGE_KERNEL_RO);
2277         set_page_prot(empty_zero_page, PAGE_KERNEL_RO);
2278 
2279         pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
2280 
2281         pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE,
2282                           PFN_DOWN(__pa(initial_page_table)));
2283         xen_write_cr3(__pa(initial_page_table));
2284 
2285         memblock_reserve(xen_pt_base, xen_pt_size);
2286 }
2287 #endif  /* CONFIG_X86_64 */
2288 
2289 void __init xen_reserve_special_pages(void)
2290 {
2291         phys_addr_t paddr;
2292 
2293         memblock_reserve(__pa(xen_start_info), PAGE_SIZE);
2294         if (xen_start_info->store_mfn) {
2295                 paddr = PFN_PHYS(mfn_to_pfn(xen_start_info->store_mfn));
2296                 memblock_reserve(paddr, PAGE_SIZE);
2297         }
2298         if (!xen_initial_domain()) {
2299                 paddr = PFN_PHYS(mfn_to_pfn(xen_start_info->console.domU.mfn));
2300                 memblock_reserve(paddr, PAGE_SIZE);
2301         }
2302 }
2303 
2304 void __init xen_pt_check_e820(void)
2305 {
2306         if (xen_is_e820_reserved(xen_pt_base, xen_pt_size)) {
2307                 xen_raw_console_write("Xen hypervisor allocated page table memory conflicts with E820 map\n");
2308                 BUG();
2309         }
2310 }
2311 
2312 static unsigned char dummy_mapping[PAGE_SIZE] __page_aligned_bss;
2313 
2314 static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot)
2315 {
2316         pte_t pte;
2317 
2318         phys >>= PAGE_SHIFT;
2319 
2320         switch (idx) {
2321         case FIX_BTMAP_END ... FIX_BTMAP_BEGIN:
2322         case FIX_RO_IDT:
2323 #ifdef CONFIG_X86_32
2324         case FIX_WP_TEST:
2325 # ifdef CONFIG_HIGHMEM
2326         case FIX_KMAP_BEGIN ... FIX_KMAP_END:
2327 # endif
2328 #elif defined(CONFIG_X86_VSYSCALL_EMULATION)
2329         case VSYSCALL_PAGE:
2330 #endif
2331         case FIX_TEXT_POKE0:
2332         case FIX_TEXT_POKE1:
2333                 /* All local page mappings */
2334                 pte = pfn_pte(phys, prot);
2335                 break;
2336 
2337 #ifdef CONFIG_X86_LOCAL_APIC
2338         case FIX_APIC_BASE:     /* maps dummy local APIC */
2339                 pte = pfn_pte(PFN_DOWN(__pa(dummy_mapping)), PAGE_KERNEL);
2340                 break;
2341 #endif
2342 
2343 #ifdef CONFIG_X86_IO_APIC
2344         case FIX_IO_APIC_BASE_0 ... FIX_IO_APIC_BASE_END:
2345                 /*
2346                  * We just don't map the IO APIC - all access is via
2347                  * hypercalls.  Keep the address in the pte for reference.
2348                  */
2349                 pte = pfn_pte(PFN_DOWN(__pa(dummy_mapping)), PAGE_KERNEL);
2350                 break;
2351 #endif
2352 
2353         case FIX_PARAVIRT_BOOTMAP:
2354                 /* This is an MFN, but it isn't an IO mapping from the
2355                    IO domain */
2356                 pte = mfn_pte(phys, prot);
2357                 break;
2358 
2359         default:
2360                 /* By default, set_fixmap is used for hardware mappings */
2361                 pte = mfn_pte(phys, prot);
2362                 break;
2363         }
2364 
2365         __native_set_fixmap(idx, pte);
2366 
2367 #ifdef CONFIG_X86_VSYSCALL_EMULATION
2368         /* Replicate changes to map the vsyscall page into the user
2369            pagetable vsyscall mapping. */
2370         if (idx == VSYSCALL_PAGE) {
2371                 unsigned long vaddr = __fix_to_virt(idx);
2372                 set_pte_vaddr_pud(level3_user_vsyscall, vaddr, pte);
2373         }
2374 #endif
2375 }
2376 
2377 static void __init xen_post_allocator_init(void)
2378 {
2379         if (xen_feature(XENFEAT_auto_translated_physmap))
2380                 return;
2381 
2382         pv_mmu_ops.set_pte = xen_set_pte;
2383         pv_mmu_ops.set_pmd = xen_set_pmd;
2384         pv_mmu_ops.set_pud = xen_set_pud;
2385 #if CONFIG_PGTABLE_LEVELS == 4
2386         pv_mmu_ops.set_pgd = xen_set_pgd;
2387 #endif
2388 
2389         /* This will work as long as patching hasn't happened yet
2390            (which it hasn't) */
2391         pv_mmu_ops.alloc_pte = xen_alloc_pte;
2392         pv_mmu_ops.alloc_pmd = xen_alloc_pmd;
2393         pv_mmu_ops.release_pte = xen_release_pte;
2394         pv_mmu_ops.release_pmd = xen_release_pmd;
2395 #if CONFIG_PGTABLE_LEVELS == 4
2396         pv_mmu_ops.alloc_pud = xen_alloc_pud;
2397         pv_mmu_ops.release_pud = xen_release_pud;
2398 #endif
2399         pv_mmu_ops.make_pte = PV_CALLEE_SAVE(xen_make_pte);
2400 
2401 #ifdef CONFIG_X86_64
2402         pv_mmu_ops.write_cr3 = &xen_write_cr3;
2403         SetPagePinned(virt_to_page(level3_user_vsyscall));
2404 #endif
2405         xen_mark_init_mm_pinned();
2406 }
2407 
2408 static void xen_leave_lazy_mmu(void)
2409 {
2410         preempt_disable();
2411         xen_mc_flush();
2412         paravirt_leave_lazy_mmu();
2413         preempt_enable();
2414 }
2415 
2416 static const struct pv_mmu_ops xen_mmu_ops __initconst = {
2417         .read_cr2 = xen_read_cr2,
2418         .write_cr2 = xen_write_cr2,
2419 
2420         .read_cr3 = xen_read_cr3,
2421         .write_cr3 = xen_write_cr3_init,
2422 
2423         .flush_tlb_user = xen_flush_tlb,
2424         .flush_tlb_kernel = xen_flush_tlb,
2425         .flush_tlb_single = xen_flush_tlb_single,
2426         .flush_tlb_others = xen_flush_tlb_others,
2427 
2428         .pte_update = paravirt_nop,
2429 
2430         .pgd_alloc = xen_pgd_alloc,
2431         .pgd_free = xen_pgd_free,
2432 
2433         .alloc_pte = xen_alloc_pte_init,
2434         .release_pte = xen_release_pte_init,
2435         .alloc_pmd = xen_alloc_pmd_init,
2436         .release_pmd = xen_release_pmd_init,
2437 
2438         .set_pte = xen_set_pte_init,
2439         .set_pte_at = xen_set_pte_at,
2440         .set_pmd = xen_set_pmd_hyper,
2441 
2442         .ptep_modify_prot_start = __ptep_modify_prot_start,
2443         .ptep_modify_prot_commit = __ptep_modify_prot_commit,
2444 
2445         .pte_val = PV_CALLEE_SAVE(xen_pte_val),
2446         .pgd_val = PV_CALLEE_SAVE(xen_pgd_val),
2447 
2448         .make_pte = PV_CALLEE_SAVE(xen_make_pte_init),
2449         .make_pgd = PV_CALLEE_SAVE(xen_make_pgd),
2450 
2451 #ifdef CONFIG_X86_PAE
2452         .set_pte_atomic = xen_set_pte_atomic,
2453         .pte_clear = xen_pte_clear,
2454         .pmd_clear = xen_pmd_clear,
2455 #endif  /* CONFIG_X86_PAE */
2456         .set_pud = xen_set_pud_hyper,
2457 
2458         .make_pmd = PV_CALLEE_SAVE(xen_make_pmd),
2459         .pmd_val = PV_CALLEE_SAVE(xen_pmd_val),
2460 
2461 #if CONFIG_PGTABLE_LEVELS == 4
2462         .pud_val = PV_CALLEE_SAVE(xen_pud_val),
2463         .make_pud = PV_CALLEE_SAVE(xen_make_pud),
2464         .set_pgd = xen_set_pgd_hyper,
2465 
2466         .alloc_pud = xen_alloc_pmd_init,
2467         .release_pud = xen_release_pmd_init,
2468 #endif  /* CONFIG_PGTABLE_LEVELS == 4 */
2469 
2470         .activate_mm = xen_activate_mm,
2471         .dup_mmap = xen_dup_mmap,
2472         .exit_mmap = xen_exit_mmap,
2473 
2474         .lazy_mode = {
2475                 .enter = paravirt_enter_lazy_mmu,
2476                 .leave = xen_leave_lazy_mmu,
2477                 .flush = paravirt_flush_lazy_mmu,
2478         },
2479 
2480         .set_fixmap = xen_set_fixmap,
2481 };
2482 
2483 void __init xen_init_mmu_ops(void)
2484 {
2485         x86_init.paging.pagetable_init = xen_pagetable_init;
2486 
2487         if (xen_feature(XENFEAT_auto_translated_physmap))
2488                 return;
2489 
2490         pv_mmu_ops = xen_mmu_ops;
2491 
2492         memset(dummy_mapping, 0xff, PAGE_SIZE);
2493 }
2494 
2495 /* Protected by xen_reservation_lock. */
2496 #define MAX_CONTIG_ORDER 9 /* 2MB */
2497 static unsigned long discontig_frames[1<<MAX_CONTIG_ORDER];
2498 
2499 #define VOID_PTE (mfn_pte(0, __pgprot(0)))
2500 static void xen_zap_pfn_range(unsigned long vaddr, unsigned int order,
2501                                 unsigned long *in_frames,
2502                                 unsigned long *out_frames)
2503 {
2504         int i;
2505         struct multicall_space mcs;
2506 
2507         xen_mc_batch();
2508         for (i = 0; i < (1UL<<order); i++, vaddr += PAGE_SIZE) {
2509                 mcs = __xen_mc_entry(0);
2510 
2511                 if (in_frames)
2512                         in_frames[i] = virt_to_mfn(vaddr);
2513 
2514                 MULTI_update_va_mapping(mcs.mc, vaddr, VOID_PTE, 0);
2515                 __set_phys_to_machine(virt_to_pfn(vaddr), INVALID_P2M_ENTRY);
2516 
2517                 if (out_frames)
2518                         out_frames[i] = virt_to_pfn(vaddr);
2519         }
2520         xen_mc_issue(0);
2521 }
2522 
2523 /*
2524  * Update the pfn-to-mfn mappings for a virtual address range, either to
2525  * point to an array of mfns, or contiguously from a single starting
2526  * mfn.
2527  */
2528 static void xen_remap_exchanged_ptes(unsigned long vaddr, int order,
2529                                      unsigned long *mfns,
2530                                      unsigned long first_mfn)
2531 {
2532         unsigned i, limit;
2533         unsigned long mfn;
2534 
2535         xen_mc_batch();
2536 
2537         limit = 1u << order;
2538         for (i = 0; i < limit; i++, vaddr += PAGE_SIZE) {
2539                 struct multicall_space mcs;
2540                 unsigned flags;
2541 
2542                 mcs = __xen_mc_entry(0);
2543                 if (mfns)
2544                         mfn = mfns[i];
2545                 else
2546                         mfn = first_mfn + i;
2547 
2548                 if (i < (limit - 1))
2549                         flags = 0;
2550                 else {
2551                         if (order == 0)
2552                                 flags = UVMF_INVLPG | UVMF_ALL;
2553                         else
2554                                 flags = UVMF_TLB_FLUSH | UVMF_ALL;
2555                 }
2556 
2557                 MULTI_update_va_mapping(mcs.mc, vaddr,
2558                                 mfn_pte(mfn, PAGE_KERNEL), flags);
2559 
2560                 set_phys_to_machine(virt_to_pfn(vaddr), mfn);
2561         }
2562 
2563         xen_mc_issue(0);
2564 }
2565 
2566 /*
2567  * Perform the hypercall to exchange a region of our pfns to point to
2568  * memory with the required contiguous alignment.  Takes the pfns as
2569  * input, and populates mfns as output.
2570  *
2571  * Returns a success code indicating whether the hypervisor was able to
2572  * satisfy the request or not.
2573  */
2574 static int xen_exchange_memory(unsigned long extents_in, unsigned int order_in,
2575                                unsigned long *pfns_in,
2576                                unsigned long extents_out,
2577                                unsigned int order_out,
2578                                unsigned long *mfns_out,
2579                                unsigned int address_bits)
2580 {
2581         long rc;
2582         int success;
2583 
2584         struct xen_memory_exchange exchange = {
2585                 .in = {
2586                         .nr_extents   = extents_in,
2587                         .extent_order = order_in,
2588                         .extent_start = pfns_in,
2589                         .domid        = DOMID_SELF
2590                 },
2591                 .out = {
2592                         .nr_extents   = extents_out,
2593                         .extent_order = order_out,
2594                         .extent_start = mfns_out,
2595                         .address_bits = address_bits,
2596                         .domid        = DOMID_SELF
2597                 }
2598         };
2599 
2600         BUG_ON(extents_in << order_in != extents_out << order_out);
2601 
2602         rc = HYPERVISOR_memory_op(XENMEM_exchange, &exchange);
2603         success = (exchange.nr_exchanged == extents_in);
2604 
2605         BUG_ON(!success && ((exchange.nr_exchanged != 0) || (rc == 0)));
2606         BUG_ON(success && (rc != 0));
2607 
2608         return success;
2609 }
2610 
2611 int xen_create_contiguous_region(phys_addr_t pstart, unsigned int order,
2612                                  unsigned int address_bits,
2613                                  dma_addr_t *dma_handle)
2614 {
2615         unsigned long *in_frames = discontig_frames, out_frame;
2616         unsigned long  flags;
2617         int            success;
2618         unsigned long vstart = (unsigned long)phys_to_virt(pstart);
2619 
2620         /*
2621          * Currently an auto-translated guest will not perform I/O, nor will
2622          * it require PAE page directories below 4GB. Therefore any calls to
2623          * this function are redundant and can be ignored.
2624          */
2625 
2626         if (xen_feature(XENFEAT_auto_translated_physmap))
2627                 return 0;
2628 
2629         if (unlikely(order > MAX_CONTIG_ORDER))
2630                 return -ENOMEM;
2631 
2632         memset((void *) vstart, 0, PAGE_SIZE << order);
2633 
2634         spin_lock_irqsave(&xen_reservation_lock, flags);
2635 
2636         /* 1. Zap current PTEs, remembering MFNs. */
2637         xen_zap_pfn_range(vstart, order, in_frames, NULL);
2638 
2639         /* 2. Get a new contiguous memory extent. */
2640         out_frame = virt_to_pfn(vstart);
2641         success = xen_exchange_memory(1UL << order, 0, in_frames,
2642                                       1, order, &out_frame,
2643                                       address_bits);
2644 
2645         /* 3. Map the new extent in place of old pages. */
2646         if (success)
2647                 xen_remap_exchanged_ptes(vstart, order, NULL, out_frame);
2648         else
2649                 xen_remap_exchanged_ptes(vstart, order, in_frames, 0);
2650 
2651         spin_unlock_irqrestore(&xen_reservation_lock, flags);
2652 
2653         *dma_handle = virt_to_machine(vstart).maddr;
2654         return success ? 0 : -ENOMEM;
2655 }
2656 EXPORT_SYMBOL_GPL(xen_create_contiguous_region);
2657 
2658 void xen_destroy_contiguous_region(phys_addr_t pstart, unsigned int order)
2659 {
2660         unsigned long *out_frames = discontig_frames, in_frame;
2661         unsigned long  flags;
2662         int success;
2663         unsigned long vstart;
2664 
2665         if (xen_feature(XENFEAT_auto_translated_physmap))
2666                 return;
2667 
2668         if (unlikely(order > MAX_CONTIG_ORDER))
2669                 return;
2670 
2671         vstart = (unsigned long)phys_to_virt(pstart);
2672         memset((void *) vstart, 0, PAGE_SIZE << order);
2673 
2674         spin_lock_irqsave(&xen_reservation_lock, flags);
2675 
2676         /* 1. Find start MFN of contiguous extent. */
2677         in_frame = virt_to_mfn(vstart);
2678 
2679         /* 2. Zap current PTEs. */
2680         xen_zap_pfn_range(vstart, order, NULL, out_frames);
2681 
2682         /* 3. Do the exchange for non-contiguous MFNs. */
2683         success = xen_exchange_memory(1, order, &in_frame, 1UL << order,
2684                                         0, out_frames, 0);
2685 
2686         /* 4. Map new pages in place of old pages. */
2687         if (success)
2688                 xen_remap_exchanged_ptes(vstart, order, out_frames, 0);
2689         else
2690                 xen_remap_exchanged_ptes(vstart, order, NULL, in_frame);
2691 
2692         spin_unlock_irqrestore(&xen_reservation_lock, flags);
2693 }
2694 EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region);
2695 
2696 #ifdef CONFIG_XEN_PVHVM
2697 #ifdef CONFIG_PROC_VMCORE
2698 /*
2699  * This function is used in two contexts:
2700  * - the kdump kernel has to check whether a pfn of the crashed kernel
2701  *   was a ballooned page. vmcore is using this function to decide
2702  *   whether to access a pfn of the crashed kernel.
2703  * - the kexec kernel has to check whether a pfn was ballooned by the
2704  *   previous kernel. If the pfn is ballooned, handle it properly.
2705  * Returns 0 if the pfn is not backed by a RAM page, the caller may
2706  * handle the pfn special in this case.
2707  */
2708 static int xen_oldmem_pfn_is_ram(unsigned long pfn)
2709 {
2710         struct xen_hvm_get_mem_type a = {
2711                 .domid = DOMID_SELF,
2712                 .pfn = pfn,
2713         };
2714         int ram;
2715 
2716         if (HYPERVISOR_hvm_op(HVMOP_get_mem_type, &a))
2717                 return -ENXIO;
2718 
2719         switch (a.mem_type) {
2720                 case HVMMEM_mmio_dm:
2721                         ram = 0;
2722                         break;
2723                 case HVMMEM_ram_rw:
2724                 case HVMMEM_ram_ro:
2725                 default:
2726                         ram = 1;
2727                         break;
2728         }
2729 
2730         return ram;
2731 }
2732 #endif
2733 
2734 static void xen_hvm_exit_mmap(struct mm_struct *mm)
2735 {
2736         struct xen_hvm_pagetable_dying a;
2737         int rc;
2738 
2739         a.domid = DOMID_SELF;
2740         a.gpa = __pa(mm->pgd);
2741         rc = HYPERVISOR_hvm_op(HVMOP_pagetable_dying, &a);
2742         WARN_ON_ONCE(rc < 0);
2743 }
2744 
2745 static int is_pagetable_dying_supported(void)
2746 {
2747         struct xen_hvm_pagetable_dying a;
2748         int rc = 0;
2749 
2750         a.domid = DOMID_SELF;
2751         a.gpa = 0x00;
2752         rc = HYPERVISOR_hvm_op(HVMOP_pagetable_dying, &a);
2753         if (rc < 0) {
2754                 printk(KERN_DEBUG "HVMOP_pagetable_dying not supported\n");
2755                 return 0;
2756         }
2757         return 1;
2758 }
2759 
2760 void __init xen_hvm_init_mmu_ops(void)
2761 {
2762         if (is_pagetable_dying_supported())
2763                 pv_mmu_ops.exit_mmap = xen_hvm_exit_mmap;
2764 #ifdef CONFIG_PROC_VMCORE
2765         register_oldmem_pfn_is_ram(&xen_oldmem_pfn_is_ram);
2766 #endif
2767 }
2768 #endif
2769 
2770 #define REMAP_BATCH_SIZE 16
2771 
2772 struct remap_data {
2773         xen_pfn_t *mfn;
2774         bool contiguous;
2775         pgprot_t prot;
2776         struct mmu_update *mmu_update;
2777 };
2778 
2779 static int remap_area_mfn_pte_fn(pte_t *ptep, pgtable_t token,
2780                                  unsigned long addr, void *data)
2781 {
2782         struct remap_data *rmd = data;
2783         pte_t pte = pte_mkspecial(mfn_pte(*rmd->mfn, rmd->prot));
2784 
2785         /* If we have a contiguous range, just update the mfn itself,
2786            else update pointer to be "next mfn". */
2787         if (rmd->contiguous)
2788                 (*rmd->mfn)++;
2789         else
2790                 rmd->mfn++;
2791 
2792         rmd->mmu_update->ptr = virt_to_machine(ptep).maddr;
2793         rmd->mmu_update->val = pte_val_ma(pte);
2794         rmd->mmu_update++;
2795 
2796         return 0;
2797 }
2798 
2799 static int do_remap_gfn(struct vm_area_struct *vma,
2800                         unsigned long addr,
2801                         xen_pfn_t *gfn, int nr,
2802                         int *err_ptr, pgprot_t prot,
2803                         unsigned domid,
2804                         struct page **pages)
2805 {
2806         int err = 0;
2807         struct remap_data rmd;
2808         struct mmu_update mmu_update[REMAP_BATCH_SIZE];
2809         unsigned long range;
2810         int mapped = 0;
2811 
2812         BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_IO)) == (VM_PFNMAP | VM_IO)));
2813 
2814         if (xen_feature(XENFEAT_auto_translated_physmap)) {
2815 #ifdef CONFIG_XEN_PVH
2816                 /* We need to update the local page tables and the xen HAP */
2817                 return xen_xlate_remap_gfn_array(vma, addr, gfn, nr, err_ptr,
2818                                                  prot, domid, pages);
2819 #else
2820                 return -EINVAL;
2821 #endif
2822         }
2823 
2824         rmd.mfn = gfn;
2825         rmd.prot = prot;
2826         /* We use the err_ptr to indicate if there we are doing a contiguous
2827          * mapping or a discontigious mapping. */
2828         rmd.contiguous = !err_ptr;
2829 
2830         while (nr) {
2831                 int index = 0;
2832                 int done = 0;
2833                 int batch = min(REMAP_BATCH_SIZE, nr);
2834                 int batch_left = batch;
2835                 range = (unsigned long)batch << PAGE_SHIFT;
2836 
2837                 rmd.mmu_update = mmu_update;
2838                 err = apply_to_page_range(vma->vm_mm, addr, range,
2839                                           remap_area_mfn_pte_fn, &rmd);
2840                 if (err)
2841                         goto out;
2842 
2843                 /* We record the error for each page that gives an error, but
2844                  * continue mapping until the whole set is done */
2845                 do {
2846                         int i;
2847 
2848                         err = HYPERVISOR_mmu_update(&mmu_update[index],
2849                                                     batch_left, &done, domid);
2850 
2851                         /*
2852                          * @err_ptr may be the same buffer as @gfn, so
2853                          * only clear it after each chunk of @gfn is
2854                          * used.
2855                          */
2856                         if (err_ptr) {
2857                                 for (i = index; i < index + done; i++)
2858                                         err_ptr[i] = 0;
2859                         }
2860                         if (err < 0) {
2861                                 if (!err_ptr)
2862                                         goto out;
2863                                 err_ptr[i] = err;
2864                                 done++; /* Skip failed frame. */
2865                         } else
2866                                 mapped += done;
2867                         batch_left -= done;
2868                         index += done;
2869                 } while (batch_left);
2870 
2871                 nr -= batch;
2872                 addr += range;
2873                 if (err_ptr)
2874                         err_ptr += batch;
2875                 cond_resched();
2876         }
2877 out:
2878 
2879         xen_flush_tlb_all();
2880 
2881         return err < 0 ? err : mapped;
2882 }
2883 
2884 int xen_remap_domain_gfn_range(struct vm_area_struct *vma,
2885                                unsigned long addr,
2886                                xen_pfn_t gfn, int nr,
2887                                pgprot_t prot, unsigned domid,
2888                                struct page **pages)
2889 {
2890         return do_remap_gfn(vma, addr, &gfn, nr, NULL, prot, domid, pages);
2891 }
2892 EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_range);
2893 
2894 int xen_remap_domain_gfn_array(struct vm_area_struct *vma,
2895                                unsigned long addr,
2896                                xen_pfn_t *gfn, int nr,
2897                                int *err_ptr, pgprot_t prot,
2898                                unsigned domid, struct page **pages)
2899 {
2900         /* We BUG_ON because it's a programmer error to pass a NULL err_ptr,
2901          * and the consequences later is quite hard to detect what the actual
2902          * cause of "wrong memory was mapped in".
2903          */
2904         BUG_ON(err_ptr == NULL);
2905         return do_remap_gfn(vma, addr, gfn, nr, err_ptr, prot, domid, pages);
2906 }
2907 EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_array);
2908 
2909 
2910 /* Returns: 0 success */
2911 int xen_unmap_domain_gfn_range(struct vm_area_struct *vma,
2912                                int numpgs, struct page **pages)
2913 {
2914         if (!pages || !xen_feature(XENFEAT_auto_translated_physmap))
2915                 return 0;
2916 
2917 #ifdef CONFIG_XEN_PVH
2918         return xen_xlate_unmap_gfn_range(vma, numpgs, pages);
2919 #else
2920         return -EINVAL;
2921 #endif
2922 }
2923 EXPORT_SYMBOL_GPL(xen_unmap_domain_gfn_range);
2924 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp