~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/kvm/mmu.c

Version: ~ [ linux-5.14-rc3 ] ~ [ linux-5.13.5 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.53 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.135 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.198 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.240 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.276 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.276 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Kernel-based Virtual Machine driver for Linux
  3  *
  4  * This module enables machines with Intel VT-x extensions to run virtual
  5  * machines without emulation or binary translation.
  6  *
  7  * MMU support
  8  *
  9  * Copyright (C) 2006 Qumranet, Inc.
 10  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
 11  *
 12  * Authors:
 13  *   Yaniv Kamay  <yaniv@qumranet.com>
 14  *   Avi Kivity   <avi@qumranet.com>
 15  *
 16  * This work is licensed under the terms of the GNU GPL, version 2.  See
 17  * the COPYING file in the top-level directory.
 18  *
 19  */
 20 
 21 #include "irq.h"
 22 #include "mmu.h"
 23 #include "x86.h"
 24 #include "kvm_cache_regs.h"
 25 #include "cpuid.h"
 26 
 27 #include <linux/kvm_host.h>
 28 #include <linux/types.h>
 29 #include <linux/string.h>
 30 #include <linux/mm.h>
 31 #include <linux/highmem.h>
 32 #include <linux/module.h>
 33 #include <linux/swap.h>
 34 #include <linux/hugetlb.h>
 35 #include <linux/compiler.h>
 36 #include <linux/srcu.h>
 37 #include <linux/slab.h>
 38 #include <linux/uaccess.h>
 39 
 40 #include <asm/page.h>
 41 #include <asm/cmpxchg.h>
 42 #include <asm/io.h>
 43 #include <asm/vmx.h>
 44 
 45 /*
 46  * When setting this variable to true it enables Two-Dimensional-Paging
 47  * where the hardware walks 2 page tables:
 48  * 1. the guest-virtual to guest-physical
 49  * 2. while doing 1. it walks guest-physical to host-physical
 50  * If the hardware supports that we don't need to do shadow paging.
 51  */
 52 bool tdp_enabled = false;
 53 
 54 enum {
 55         AUDIT_PRE_PAGE_FAULT,
 56         AUDIT_POST_PAGE_FAULT,
 57         AUDIT_PRE_PTE_WRITE,
 58         AUDIT_POST_PTE_WRITE,
 59         AUDIT_PRE_SYNC,
 60         AUDIT_POST_SYNC
 61 };
 62 
 63 #undef MMU_DEBUG
 64 
 65 #ifdef MMU_DEBUG
 66 
 67 #define pgprintk(x...) do { if (dbg) printk(x); } while (0)
 68 #define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
 69 
 70 #else
 71 
 72 #define pgprintk(x...) do { } while (0)
 73 #define rmap_printk(x...) do { } while (0)
 74 
 75 #endif
 76 
 77 #ifdef MMU_DEBUG
 78 static bool dbg = 0;
 79 module_param(dbg, bool, 0644);
 80 #endif
 81 
 82 #ifndef MMU_DEBUG
 83 #define ASSERT(x) do { } while (0)
 84 #else
 85 #define ASSERT(x)                                                       \
 86         if (!(x)) {                                                     \
 87                 printk(KERN_WARNING "assertion failed %s:%d: %s\n",     \
 88                        __FILE__, __LINE__, #x);                         \
 89         }
 90 #endif
 91 
 92 #define PTE_PREFETCH_NUM                8
 93 
 94 #define PT_FIRST_AVAIL_BITS_SHIFT 10
 95 #define PT64_SECOND_AVAIL_BITS_SHIFT 52
 96 
 97 #define PT64_LEVEL_BITS 9
 98 
 99 #define PT64_LEVEL_SHIFT(level) \
100                 (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
101 
102 #define PT64_INDEX(address, level)\
103         (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
104 
105 
106 #define PT32_LEVEL_BITS 10
107 
108 #define PT32_LEVEL_SHIFT(level) \
109                 (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
110 
111 #define PT32_LVL_OFFSET_MASK(level) \
112         (PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
113                                                 * PT32_LEVEL_BITS))) - 1))
114 
115 #define PT32_INDEX(address, level)\
116         (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
117 
118 
119 #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
120 #define PT64_DIR_BASE_ADDR_MASK \
121         (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
122 #define PT64_LVL_ADDR_MASK(level) \
123         (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
124                                                 * PT64_LEVEL_BITS))) - 1))
125 #define PT64_LVL_OFFSET_MASK(level) \
126         (PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
127                                                 * PT64_LEVEL_BITS))) - 1))
128 
129 #define PT32_BASE_ADDR_MASK PAGE_MASK
130 #define PT32_DIR_BASE_ADDR_MASK \
131         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
132 #define PT32_LVL_ADDR_MASK(level) \
133         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
134                                             * PT32_LEVEL_BITS))) - 1))
135 
136 #define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | shadow_user_mask \
137                         | shadow_x_mask | shadow_nx_mask)
138 
139 #define ACC_EXEC_MASK    1
140 #define ACC_WRITE_MASK   PT_WRITABLE_MASK
141 #define ACC_USER_MASK    PT_USER_MASK
142 #define ACC_ALL          (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
143 
144 #include <trace/events/kvm.h>
145 
146 #define CREATE_TRACE_POINTS
147 #include "mmutrace.h"
148 
149 #define SPTE_HOST_WRITEABLE     (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
150 #define SPTE_MMU_WRITEABLE      (1ULL << (PT_FIRST_AVAIL_BITS_SHIFT + 1))
151 
152 #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
153 
154 /* make pte_list_desc fit well in cache line */
155 #define PTE_LIST_EXT 3
156 
157 struct pte_list_desc {
158         u64 *sptes[PTE_LIST_EXT];
159         struct pte_list_desc *more;
160 };
161 
162 struct kvm_shadow_walk_iterator {
163         u64 addr;
164         hpa_t shadow_addr;
165         u64 *sptep;
166         int level;
167         unsigned index;
168 };
169 
170 #define for_each_shadow_entry(_vcpu, _addr, _walker)    \
171         for (shadow_walk_init(&(_walker), _vcpu, _addr);        \
172              shadow_walk_okay(&(_walker));                      \
173              shadow_walk_next(&(_walker)))
174 
175 #define for_each_shadow_entry_lockless(_vcpu, _addr, _walker, spte)     \
176         for (shadow_walk_init(&(_walker), _vcpu, _addr);                \
177              shadow_walk_okay(&(_walker)) &&                            \
178                 ({ spte = mmu_spte_get_lockless(_walker.sptep); 1; });  \
179              __shadow_walk_next(&(_walker), spte))
180 
181 static struct kmem_cache *pte_list_desc_cache;
182 static struct kmem_cache *mmu_page_header_cache;
183 static struct percpu_counter kvm_total_used_mmu_pages;
184 
185 static u64 __read_mostly shadow_nx_mask;
186 static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
187 static u64 __read_mostly shadow_user_mask;
188 static u64 __read_mostly shadow_accessed_mask;
189 static u64 __read_mostly shadow_dirty_mask;
190 static u64 __read_mostly shadow_mmio_mask;
191 
192 static void mmu_spte_set(u64 *sptep, u64 spte);
193 static void mmu_free_roots(struct kvm_vcpu *vcpu);
194 
195 void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask)
196 {
197         shadow_mmio_mask = mmio_mask;
198 }
199 EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_mask);
200 
201 /*
202  * the low bit of the generation number is always presumed to be zero.
203  * This disables mmio caching during memslot updates.  The concept is
204  * similar to a seqcount but instead of retrying the access we just punt
205  * and ignore the cache.
206  *
207  * spte bits 3-11 are used as bits 1-9 of the generation number,
208  * the bits 52-61 are used as bits 10-19 of the generation number.
209  */
210 #define MMIO_SPTE_GEN_LOW_SHIFT         2
211 #define MMIO_SPTE_GEN_HIGH_SHIFT        52
212 
213 #define MMIO_GEN_SHIFT                  20
214 #define MMIO_GEN_LOW_SHIFT              10
215 #define MMIO_GEN_LOW_MASK               ((1 << MMIO_GEN_LOW_SHIFT) - 2)
216 #define MMIO_GEN_MASK                   ((1 << MMIO_GEN_SHIFT) - 1)
217 #define MMIO_MAX_GEN                    ((1 << MMIO_GEN_SHIFT) - 1)
218 
219 static u64 generation_mmio_spte_mask(unsigned int gen)
220 {
221         u64 mask;
222 
223         WARN_ON(gen > MMIO_MAX_GEN);
224 
225         mask = (gen & MMIO_GEN_LOW_MASK) << MMIO_SPTE_GEN_LOW_SHIFT;
226         mask |= ((u64)gen >> MMIO_GEN_LOW_SHIFT) << MMIO_SPTE_GEN_HIGH_SHIFT;
227         return mask;
228 }
229 
230 static unsigned int get_mmio_spte_generation(u64 spte)
231 {
232         unsigned int gen;
233 
234         spte &= ~shadow_mmio_mask;
235 
236         gen = (spte >> MMIO_SPTE_GEN_LOW_SHIFT) & MMIO_GEN_LOW_MASK;
237         gen |= (spte >> MMIO_SPTE_GEN_HIGH_SHIFT) << MMIO_GEN_LOW_SHIFT;
238         return gen;
239 }
240 
241 static unsigned int kvm_current_mmio_generation(struct kvm *kvm)
242 {
243         return kvm_memslots(kvm)->generation & MMIO_GEN_MASK;
244 }
245 
246 static void mark_mmio_spte(struct kvm *kvm, u64 *sptep, u64 gfn,
247                            unsigned access)
248 {
249         unsigned int gen = kvm_current_mmio_generation(kvm);
250         u64 mask = generation_mmio_spte_mask(gen);
251 
252         access &= ACC_WRITE_MASK | ACC_USER_MASK;
253         mask |= shadow_mmio_mask | access | gfn << PAGE_SHIFT;
254 
255         trace_mark_mmio_spte(sptep, gfn, access, gen);
256         mmu_spte_set(sptep, mask);
257 }
258 
259 static bool is_mmio_spte(u64 spte)
260 {
261         return (spte & shadow_mmio_mask) == shadow_mmio_mask;
262 }
263 
264 static gfn_t get_mmio_spte_gfn(u64 spte)
265 {
266         u64 mask = generation_mmio_spte_mask(MMIO_MAX_GEN) | shadow_mmio_mask;
267         return (spte & ~mask) >> PAGE_SHIFT;
268 }
269 
270 static unsigned get_mmio_spte_access(u64 spte)
271 {
272         u64 mask = generation_mmio_spte_mask(MMIO_MAX_GEN) | shadow_mmio_mask;
273         return (spte & ~mask) & ~PAGE_MASK;
274 }
275 
276 static bool set_mmio_spte(struct kvm *kvm, u64 *sptep, gfn_t gfn,
277                           pfn_t pfn, unsigned access)
278 {
279         if (unlikely(is_noslot_pfn(pfn))) {
280                 mark_mmio_spte(kvm, sptep, gfn, access);
281                 return true;
282         }
283 
284         return false;
285 }
286 
287 static bool check_mmio_spte(struct kvm *kvm, u64 spte)
288 {
289         unsigned int kvm_gen, spte_gen;
290 
291         kvm_gen = kvm_current_mmio_generation(kvm);
292         spte_gen = get_mmio_spte_generation(spte);
293 
294         trace_check_mmio_spte(spte, kvm_gen, spte_gen);
295         return likely(kvm_gen == spte_gen);
296 }
297 
298 void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
299                 u64 dirty_mask, u64 nx_mask, u64 x_mask)
300 {
301         shadow_user_mask = user_mask;
302         shadow_accessed_mask = accessed_mask;
303         shadow_dirty_mask = dirty_mask;
304         shadow_nx_mask = nx_mask;
305         shadow_x_mask = x_mask;
306 }
307 EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);
308 
309 static int is_cpuid_PSE36(void)
310 {
311         return 1;
312 }
313 
314 static int is_nx(struct kvm_vcpu *vcpu)
315 {
316         return vcpu->arch.efer & EFER_NX;
317 }
318 
319 static int is_shadow_present_pte(u64 pte)
320 {
321         return pte & PT_PRESENT_MASK && !is_mmio_spte(pte);
322 }
323 
324 static int is_large_pte(u64 pte)
325 {
326         return pte & PT_PAGE_SIZE_MASK;
327 }
328 
329 static int is_rmap_spte(u64 pte)
330 {
331         return is_shadow_present_pte(pte);
332 }
333 
334 static int is_last_spte(u64 pte, int level)
335 {
336         if (level == PT_PAGE_TABLE_LEVEL)
337                 return 1;
338         if (is_large_pte(pte))
339                 return 1;
340         return 0;
341 }
342 
343 static pfn_t spte_to_pfn(u64 pte)
344 {
345         return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
346 }
347 
348 static gfn_t pse36_gfn_delta(u32 gpte)
349 {
350         int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;
351 
352         return (gpte & PT32_DIR_PSE36_MASK) << shift;
353 }
354 
355 #ifdef CONFIG_X86_64
356 static void __set_spte(u64 *sptep, u64 spte)
357 {
358         *sptep = spte;
359 }
360 
361 static void __update_clear_spte_fast(u64 *sptep, u64 spte)
362 {
363         *sptep = spte;
364 }
365 
366 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
367 {
368         return xchg(sptep, spte);
369 }
370 
371 static u64 __get_spte_lockless(u64 *sptep)
372 {
373         return ACCESS_ONCE(*sptep);
374 }
375 #else
376 union split_spte {
377         struct {
378                 u32 spte_low;
379                 u32 spte_high;
380         };
381         u64 spte;
382 };
383 
384 static void count_spte_clear(u64 *sptep, u64 spte)
385 {
386         struct kvm_mmu_page *sp =  page_header(__pa(sptep));
387 
388         if (is_shadow_present_pte(spte))
389                 return;
390 
391         /* Ensure the spte is completely set before we increase the count */
392         smp_wmb();
393         sp->clear_spte_count++;
394 }
395 
396 static void __set_spte(u64 *sptep, u64 spte)
397 {
398         union split_spte *ssptep, sspte;
399 
400         ssptep = (union split_spte *)sptep;
401         sspte = (union split_spte)spte;
402 
403         ssptep->spte_high = sspte.spte_high;
404 
405         /*
406          * If we map the spte from nonpresent to present, We should store
407          * the high bits firstly, then set present bit, so cpu can not
408          * fetch this spte while we are setting the spte.
409          */
410         smp_wmb();
411 
412         ssptep->spte_low = sspte.spte_low;
413 }
414 
415 static void __update_clear_spte_fast(u64 *sptep, u64 spte)
416 {
417         union split_spte *ssptep, sspte;
418 
419         ssptep = (union split_spte *)sptep;
420         sspte = (union split_spte)spte;
421 
422         ssptep->spte_low = sspte.spte_low;
423 
424         /*
425          * If we map the spte from present to nonpresent, we should clear
426          * present bit firstly to avoid vcpu fetch the old high bits.
427          */
428         smp_wmb();
429 
430         ssptep->spte_high = sspte.spte_high;
431         count_spte_clear(sptep, spte);
432 }
433 
434 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
435 {
436         union split_spte *ssptep, sspte, orig;
437 
438         ssptep = (union split_spte *)sptep;
439         sspte = (union split_spte)spte;
440 
441         /* xchg acts as a barrier before the setting of the high bits */
442         orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low);
443         orig.spte_high = ssptep->spte_high;
444         ssptep->spte_high = sspte.spte_high;
445         count_spte_clear(sptep, spte);
446 
447         return orig.spte;
448 }
449 
450 /*
451  * The idea using the light way get the spte on x86_32 guest is from
452  * gup_get_pte(arch/x86/mm/gup.c).
453  *
454  * An spte tlb flush may be pending, because kvm_set_pte_rmapp
455  * coalesces them and we are running out of the MMU lock.  Therefore
456  * we need to protect against in-progress updates of the spte.
457  *
458  * Reading the spte while an update is in progress may get the old value
459  * for the high part of the spte.  The race is fine for a present->non-present
460  * change (because the high part of the spte is ignored for non-present spte),
461  * but for a present->present change we must reread the spte.
462  *
463  * All such changes are done in two steps (present->non-present and
464  * non-present->present), hence it is enough to count the number of
465  * present->non-present updates: if it changed while reading the spte,
466  * we might have hit the race.  This is done using clear_spte_count.
467  */
468 static u64 __get_spte_lockless(u64 *sptep)
469 {
470         struct kvm_mmu_page *sp =  page_header(__pa(sptep));
471         union split_spte spte, *orig = (union split_spte *)sptep;
472         int count;
473 
474 retry:
475         count = sp->clear_spte_count;
476         smp_rmb();
477 
478         spte.spte_low = orig->spte_low;
479         smp_rmb();
480 
481         spte.spte_high = orig->spte_high;
482         smp_rmb();
483 
484         if (unlikely(spte.spte_low != orig->spte_low ||
485               count != sp->clear_spte_count))
486                 goto retry;
487 
488         return spte.spte;
489 }
490 #endif
491 
492 static bool spte_is_locklessly_modifiable(u64 spte)
493 {
494         return (spte & (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE)) ==
495                 (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE);
496 }
497 
498 static bool spte_has_volatile_bits(u64 spte)
499 {
500         /*
501          * Always atomicly update spte if it can be updated
502          * out of mmu-lock, it can ensure dirty bit is not lost,
503          * also, it can help us to get a stable is_writable_pte()
504          * to ensure tlb flush is not missed.
505          */
506         if (spte_is_locklessly_modifiable(spte))
507                 return true;
508 
509         if (!shadow_accessed_mask)
510                 return false;
511 
512         if (!is_shadow_present_pte(spte))
513                 return false;
514 
515         if ((spte & shadow_accessed_mask) &&
516               (!is_writable_pte(spte) || (spte & shadow_dirty_mask)))
517                 return false;
518 
519         return true;
520 }
521 
522 static bool spte_is_bit_cleared(u64 old_spte, u64 new_spte, u64 bit_mask)
523 {
524         return (old_spte & bit_mask) && !(new_spte & bit_mask);
525 }
526 
527 /* Rules for using mmu_spte_set:
528  * Set the sptep from nonpresent to present.
529  * Note: the sptep being assigned *must* be either not present
530  * or in a state where the hardware will not attempt to update
531  * the spte.
532  */
533 static void mmu_spte_set(u64 *sptep, u64 new_spte)
534 {
535         WARN_ON(is_shadow_present_pte(*sptep));
536         __set_spte(sptep, new_spte);
537 }
538 
539 /* Rules for using mmu_spte_update:
540  * Update the state bits, it means the mapped pfn is not changged.
541  *
542  * Whenever we overwrite a writable spte with a read-only one we
543  * should flush remote TLBs. Otherwise rmap_write_protect
544  * will find a read-only spte, even though the writable spte
545  * might be cached on a CPU's TLB, the return value indicates this
546  * case.
547  */
548 static bool mmu_spte_update(u64 *sptep, u64 new_spte)
549 {
550         u64 old_spte = *sptep;
551         bool ret = false;
552 
553         WARN_ON(!is_rmap_spte(new_spte));
554 
555         if (!is_shadow_present_pte(old_spte)) {
556                 mmu_spte_set(sptep, new_spte);
557                 return ret;
558         }
559 
560         if (!spte_has_volatile_bits(old_spte))
561                 __update_clear_spte_fast(sptep, new_spte);
562         else
563                 old_spte = __update_clear_spte_slow(sptep, new_spte);
564 
565         /*
566          * For the spte updated out of mmu-lock is safe, since
567          * we always atomicly update it, see the comments in
568          * spte_has_volatile_bits().
569          */
570         if (spte_is_locklessly_modifiable(old_spte) &&
571               !is_writable_pte(new_spte))
572                 ret = true;
573 
574         if (!shadow_accessed_mask)
575                 return ret;
576 
577         if (spte_is_bit_cleared(old_spte, new_spte, shadow_accessed_mask))
578                 kvm_set_pfn_accessed(spte_to_pfn(old_spte));
579         if (spte_is_bit_cleared(old_spte, new_spte, shadow_dirty_mask))
580                 kvm_set_pfn_dirty(spte_to_pfn(old_spte));
581 
582         return ret;
583 }
584 
585 /*
586  * Rules for using mmu_spte_clear_track_bits:
587  * It sets the sptep from present to nonpresent, and track the
588  * state bits, it is used to clear the last level sptep.
589  */
590 static int mmu_spte_clear_track_bits(u64 *sptep)
591 {
592         pfn_t pfn;
593         u64 old_spte = *sptep;
594 
595         if (!spte_has_volatile_bits(old_spte))
596                 __update_clear_spte_fast(sptep, 0ull);
597         else
598                 old_spte = __update_clear_spte_slow(sptep, 0ull);
599 
600         if (!is_rmap_spte(old_spte))
601                 return 0;
602 
603         pfn = spte_to_pfn(old_spte);
604 
605         /*
606          * KVM does not hold the refcount of the page used by
607          * kvm mmu, before reclaiming the page, we should
608          * unmap it from mmu first.
609          */
610         WARN_ON(!kvm_is_reserved_pfn(pfn) && !page_count(pfn_to_page(pfn)));
611 
612         if (!shadow_accessed_mask || old_spte & shadow_accessed_mask)
613                 kvm_set_pfn_accessed(pfn);
614         if (!shadow_dirty_mask || (old_spte & shadow_dirty_mask))
615                 kvm_set_pfn_dirty(pfn);
616         return 1;
617 }
618 
619 /*
620  * Rules for using mmu_spte_clear_no_track:
621  * Directly clear spte without caring the state bits of sptep,
622  * it is used to set the upper level spte.
623  */
624 static void mmu_spte_clear_no_track(u64 *sptep)
625 {
626         __update_clear_spte_fast(sptep, 0ull);
627 }
628 
629 static u64 mmu_spte_get_lockless(u64 *sptep)
630 {
631         return __get_spte_lockless(sptep);
632 }
633 
634 static void walk_shadow_page_lockless_begin(struct kvm_vcpu *vcpu)
635 {
636         /*
637          * Prevent page table teardown by making any free-er wait during
638          * kvm_flush_remote_tlbs() IPI to all active vcpus.
639          */
640         local_irq_disable();
641         vcpu->mode = READING_SHADOW_PAGE_TABLES;
642         /*
643          * Make sure a following spte read is not reordered ahead of the write
644          * to vcpu->mode.
645          */
646         smp_mb();
647 }
648 
649 static void walk_shadow_page_lockless_end(struct kvm_vcpu *vcpu)
650 {
651         /*
652          * Make sure the write to vcpu->mode is not reordered in front of
653          * reads to sptes.  If it does, kvm_commit_zap_page() can see us
654          * OUTSIDE_GUEST_MODE and proceed to free the shadow page table.
655          */
656         smp_mb();
657         vcpu->mode = OUTSIDE_GUEST_MODE;
658         local_irq_enable();
659 }
660 
661 static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
662                                   struct kmem_cache *base_cache, int min)
663 {
664         void *obj;
665 
666         if (cache->nobjs >= min)
667                 return 0;
668         while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
669                 obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
670                 if (!obj)
671                         return -ENOMEM;
672                 cache->objects[cache->nobjs++] = obj;
673         }
674         return 0;
675 }
676 
677 static int mmu_memory_cache_free_objects(struct kvm_mmu_memory_cache *cache)
678 {
679         return cache->nobjs;
680 }
681 
682 static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc,
683                                   struct kmem_cache *cache)
684 {
685         while (mc->nobjs)
686                 kmem_cache_free(cache, mc->objects[--mc->nobjs]);
687 }
688 
689 static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
690                                        int min)
691 {
692         void *page;
693 
694         if (cache->nobjs >= min)
695                 return 0;
696         while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
697                 page = (void *)__get_free_page(GFP_KERNEL);
698                 if (!page)
699                         return -ENOMEM;
700                 cache->objects[cache->nobjs++] = page;
701         }
702         return 0;
703 }
704 
705 static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
706 {
707         while (mc->nobjs)
708                 free_page((unsigned long)mc->objects[--mc->nobjs]);
709 }
710 
711 static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
712 {
713         int r;
714 
715         r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
716                                    pte_list_desc_cache, 8 + PTE_PREFETCH_NUM);
717         if (r)
718                 goto out;
719         r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
720         if (r)
721                 goto out;
722         r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
723                                    mmu_page_header_cache, 4);
724 out:
725         return r;
726 }
727 
728 static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
729 {
730         mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
731                                 pte_list_desc_cache);
732         mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
733         mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache,
734                                 mmu_page_header_cache);
735 }
736 
737 static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
738 {
739         void *p;
740 
741         BUG_ON(!mc->nobjs);
742         p = mc->objects[--mc->nobjs];
743         return p;
744 }
745 
746 static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu)
747 {
748         return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache);
749 }
750 
751 static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc)
752 {
753         kmem_cache_free(pte_list_desc_cache, pte_list_desc);
754 }
755 
756 static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
757 {
758         if (!sp->role.direct)
759                 return sp->gfns[index];
760 
761         return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
762 }
763 
764 static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
765 {
766         if (sp->role.direct)
767                 BUG_ON(gfn != kvm_mmu_page_get_gfn(sp, index));
768         else
769                 sp->gfns[index] = gfn;
770 }
771 
772 /*
773  * Return the pointer to the large page information for a given gfn,
774  * handling slots that are not large page aligned.
775  */
776 static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
777                                               struct kvm_memory_slot *slot,
778                                               int level)
779 {
780         unsigned long idx;
781 
782         idx = gfn_to_index(gfn, slot->base_gfn, level);
783         return &slot->arch.lpage_info[level - 2][idx];
784 }
785 
786 static void account_shadowed(struct kvm *kvm, gfn_t gfn)
787 {
788         struct kvm_memory_slot *slot;
789         struct kvm_lpage_info *linfo;
790         int i;
791 
792         slot = gfn_to_memslot(kvm, gfn);
793         for (i = PT_DIRECTORY_LEVEL;
794              i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
795                 linfo = lpage_info_slot(gfn, slot, i);
796                 linfo->write_count += 1;
797         }
798         kvm->arch.indirect_shadow_pages++;
799 }
800 
801 static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
802 {
803         struct kvm_memory_slot *slot;
804         struct kvm_lpage_info *linfo;
805         int i;
806 
807         slot = gfn_to_memslot(kvm, gfn);
808         for (i = PT_DIRECTORY_LEVEL;
809              i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
810                 linfo = lpage_info_slot(gfn, slot, i);
811                 linfo->write_count -= 1;
812                 WARN_ON(linfo->write_count < 0);
813         }
814         kvm->arch.indirect_shadow_pages--;
815 }
816 
817 static int has_wrprotected_page(struct kvm *kvm,
818                                 gfn_t gfn,
819                                 int level)
820 {
821         struct kvm_memory_slot *slot;
822         struct kvm_lpage_info *linfo;
823 
824         slot = gfn_to_memslot(kvm, gfn);
825         if (slot) {
826                 linfo = lpage_info_slot(gfn, slot, level);
827                 return linfo->write_count;
828         }
829 
830         return 1;
831 }
832 
833 static int host_mapping_level(struct kvm *kvm, gfn_t gfn)
834 {
835         unsigned long page_size;
836         int i, ret = 0;
837 
838         page_size = kvm_host_page_size(kvm, gfn);
839 
840         for (i = PT_PAGE_TABLE_LEVEL;
841              i < (PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES); ++i) {
842                 if (page_size >= KVM_HPAGE_SIZE(i))
843                         ret = i;
844                 else
845                         break;
846         }
847 
848         return ret;
849 }
850 
851 static struct kvm_memory_slot *
852 gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t gfn,
853                             bool no_dirty_log)
854 {
855         struct kvm_memory_slot *slot;
856 
857         slot = gfn_to_memslot(vcpu->kvm, gfn);
858         if (!slot || slot->flags & KVM_MEMSLOT_INVALID ||
859               (no_dirty_log && slot->dirty_bitmap))
860                 slot = NULL;
861 
862         return slot;
863 }
864 
865 static bool mapping_level_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t large_gfn)
866 {
867         return !gfn_to_memslot_dirty_bitmap(vcpu, large_gfn, true);
868 }
869 
870 static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn)
871 {
872         int host_level, level, max_level;
873 
874         host_level = host_mapping_level(vcpu->kvm, large_gfn);
875 
876         if (host_level == PT_PAGE_TABLE_LEVEL)
877                 return host_level;
878 
879         max_level = min(kvm_x86_ops->get_lpage_level(), host_level);
880 
881         for (level = PT_DIRECTORY_LEVEL; level <= max_level; ++level)
882                 if (has_wrprotected_page(vcpu->kvm, large_gfn, level))
883                         break;
884 
885         return level - 1;
886 }
887 
888 /*
889  * Pte mapping structures:
890  *
891  * If pte_list bit zero is zero, then pte_list point to the spte.
892  *
893  * If pte_list bit zero is one, (then pte_list & ~1) points to a struct
894  * pte_list_desc containing more mappings.
895  *
896  * Returns the number of pte entries before the spte was added or zero if
897  * the spte was not added.
898  *
899  */
900 static int pte_list_add(struct kvm_vcpu *vcpu, u64 *spte,
901                         unsigned long *pte_list)
902 {
903         struct pte_list_desc *desc;
904         int i, count = 0;
905 
906         if (!*pte_list) {
907                 rmap_printk("pte_list_add: %p %llx 0->1\n", spte, *spte);
908                 *pte_list = (unsigned long)spte;
909         } else if (!(*pte_list & 1)) {
910                 rmap_printk("pte_list_add: %p %llx 1->many\n", spte, *spte);
911                 desc = mmu_alloc_pte_list_desc(vcpu);
912                 desc->sptes[0] = (u64 *)*pte_list;
913                 desc->sptes[1] = spte;
914                 *pte_list = (unsigned long)desc | 1;
915                 ++count;
916         } else {
917                 rmap_printk("pte_list_add: %p %llx many->many\n", spte, *spte);
918                 desc = (struct pte_list_desc *)(*pte_list & ~1ul);
919                 while (desc->sptes[PTE_LIST_EXT-1] && desc->more) {
920                         desc = desc->more;
921                         count += PTE_LIST_EXT;
922                 }
923                 if (desc->sptes[PTE_LIST_EXT-1]) {
924                         desc->more = mmu_alloc_pte_list_desc(vcpu);
925                         desc = desc->more;
926                 }
927                 for (i = 0; desc->sptes[i]; ++i)
928                         ++count;
929                 desc->sptes[i] = spte;
930         }
931         return count;
932 }
933 
934 static void
935 pte_list_desc_remove_entry(unsigned long *pte_list, struct pte_list_desc *desc,
936                            int i, struct pte_list_desc *prev_desc)
937 {
938         int j;
939 
940         for (j = PTE_LIST_EXT - 1; !desc->sptes[j] && j > i; --j)
941                 ;
942         desc->sptes[i] = desc->sptes[j];
943         desc->sptes[j] = NULL;
944         if (j != 0)
945                 return;
946         if (!prev_desc && !desc->more)
947                 *pte_list = (unsigned long)desc->sptes[0];
948         else
949                 if (prev_desc)
950                         prev_desc->more = desc->more;
951                 else
952                         *pte_list = (unsigned long)desc->more | 1;
953         mmu_free_pte_list_desc(desc);
954 }
955 
956 static void pte_list_remove(u64 *spte, unsigned long *pte_list)
957 {
958         struct pte_list_desc *desc;
959         struct pte_list_desc *prev_desc;
960         int i;
961 
962         if (!*pte_list) {
963                 printk(KERN_ERR "pte_list_remove: %p 0->BUG\n", spte);
964                 BUG();
965         } else if (!(*pte_list & 1)) {
966                 rmap_printk("pte_list_remove:  %p 1->0\n", spte);
967                 if ((u64 *)*pte_list != spte) {
968                         printk(KERN_ERR "pte_list_remove:  %p 1->BUG\n", spte);
969                         BUG();
970                 }
971                 *pte_list = 0;
972         } else {
973                 rmap_printk("pte_list_remove:  %p many->many\n", spte);
974                 desc = (struct pte_list_desc *)(*pte_list & ~1ul);
975                 prev_desc = NULL;
976                 while (desc) {
977                         for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i)
978                                 if (desc->sptes[i] == spte) {
979                                         pte_list_desc_remove_entry(pte_list,
980                                                                desc, i,
981                                                                prev_desc);
982                                         return;
983                                 }
984                         prev_desc = desc;
985                         desc = desc->more;
986                 }
987                 pr_err("pte_list_remove: %p many->many\n", spte);
988                 BUG();
989         }
990 }
991 
992 typedef void (*pte_list_walk_fn) (u64 *spte);
993 static void pte_list_walk(unsigned long *pte_list, pte_list_walk_fn fn)
994 {
995         struct pte_list_desc *desc;
996         int i;
997 
998         if (!*pte_list)
999                 return;
1000 
1001         if (!(*pte_list & 1))
1002                 return fn((u64 *)*pte_list);
1003 
1004         desc = (struct pte_list_desc *)(*pte_list & ~1ul);
1005         while (desc) {
1006                 for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i)
1007                         fn(desc->sptes[i]);
1008                 desc = desc->more;
1009         }
1010 }
1011 
1012 static unsigned long *__gfn_to_rmap(gfn_t gfn, int level,
1013                                     struct kvm_memory_slot *slot)
1014 {
1015         unsigned long idx;
1016 
1017         idx = gfn_to_index(gfn, slot->base_gfn, level);
1018         return &slot->arch.rmap[level - PT_PAGE_TABLE_LEVEL][idx];
1019 }
1020 
1021 /*
1022  * Take gfn and return the reverse mapping to it.
1023  */
1024 static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int level)
1025 {
1026         struct kvm_memory_slot *slot;
1027 
1028         slot = gfn_to_memslot(kvm, gfn);
1029         return __gfn_to_rmap(gfn, level, slot);
1030 }
1031 
1032 static bool rmap_can_add(struct kvm_vcpu *vcpu)
1033 {
1034         struct kvm_mmu_memory_cache *cache;
1035 
1036         cache = &vcpu->arch.mmu_pte_list_desc_cache;
1037         return mmu_memory_cache_free_objects(cache);
1038 }
1039 
1040 static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
1041 {
1042         struct kvm_mmu_page *sp;
1043         unsigned long *rmapp;
1044 
1045         sp = page_header(__pa(spte));
1046         kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
1047         rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
1048         return pte_list_add(vcpu, spte, rmapp);
1049 }
1050 
1051 static void rmap_remove(struct kvm *kvm, u64 *spte)
1052 {
1053         struct kvm_mmu_page *sp;
1054         gfn_t gfn;
1055         unsigned long *rmapp;
1056 
1057         sp = page_header(__pa(spte));
1058         gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
1059         rmapp = gfn_to_rmap(kvm, gfn, sp->role.level);
1060         pte_list_remove(spte, rmapp);
1061 }
1062 
1063 /*
1064  * Used by the following functions to iterate through the sptes linked by a
1065  * rmap.  All fields are private and not assumed to be used outside.
1066  */
1067 struct rmap_iterator {
1068         /* private fields */
1069         struct pte_list_desc *desc;     /* holds the sptep if not NULL */
1070         int pos;                        /* index of the sptep */
1071 };
1072 
1073 /*
1074  * Iteration must be started by this function.  This should also be used after
1075  * removing/dropping sptes from the rmap link because in such cases the
1076  * information in the itererator may not be valid.
1077  *
1078  * Returns sptep if found, NULL otherwise.
1079  */
1080 static u64 *rmap_get_first(unsigned long rmap, struct rmap_iterator *iter)
1081 {
1082         if (!rmap)
1083                 return NULL;
1084 
1085         if (!(rmap & 1)) {
1086                 iter->desc = NULL;
1087                 return (u64 *)rmap;
1088         }
1089 
1090         iter->desc = (struct pte_list_desc *)(rmap & ~1ul);
1091         iter->pos = 0;
1092         return iter->desc->sptes[iter->pos];
1093 }
1094 
1095 /*
1096  * Must be used with a valid iterator: e.g. after rmap_get_first().
1097  *
1098  * Returns sptep if found, NULL otherwise.
1099  */
1100 static u64 *rmap_get_next(struct rmap_iterator *iter)
1101 {
1102         if (iter->desc) {
1103                 if (iter->pos < PTE_LIST_EXT - 1) {
1104                         u64 *sptep;
1105 
1106                         ++iter->pos;
1107                         sptep = iter->desc->sptes[iter->pos];
1108                         if (sptep)
1109                                 return sptep;
1110                 }
1111 
1112                 iter->desc = iter->desc->more;
1113 
1114                 if (iter->desc) {
1115                         iter->pos = 0;
1116                         /* desc->sptes[0] cannot be NULL */
1117                         return iter->desc->sptes[iter->pos];
1118                 }
1119         }
1120 
1121         return NULL;
1122 }
1123 
1124 static void drop_spte(struct kvm *kvm, u64 *sptep)
1125 {
1126         if (mmu_spte_clear_track_bits(sptep))
1127                 rmap_remove(kvm, sptep);
1128 }
1129 
1130 
1131 static bool __drop_large_spte(struct kvm *kvm, u64 *sptep)
1132 {
1133         if (is_large_pte(*sptep)) {
1134                 WARN_ON(page_header(__pa(sptep))->role.level ==
1135                         PT_PAGE_TABLE_LEVEL);
1136                 drop_spte(kvm, sptep);
1137                 --kvm->stat.lpages;
1138                 return true;
1139         }
1140 
1141         return false;
1142 }
1143 
1144 static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
1145 {
1146         if (__drop_large_spte(vcpu->kvm, sptep))
1147                 kvm_flush_remote_tlbs(vcpu->kvm);
1148 }
1149 
1150 /*
1151  * Write-protect on the specified @sptep, @pt_protect indicates whether
1152  * spte write-protection is caused by protecting shadow page table.
1153  *
1154  * Note: write protection is difference between dirty logging and spte
1155  * protection:
1156  * - for dirty logging, the spte can be set to writable at anytime if
1157  *   its dirty bitmap is properly set.
1158  * - for spte protection, the spte can be writable only after unsync-ing
1159  *   shadow page.
1160  *
1161  * Return true if tlb need be flushed.
1162  */
1163 static bool spte_write_protect(struct kvm *kvm, u64 *sptep, bool pt_protect)
1164 {
1165         u64 spte = *sptep;
1166 
1167         if (!is_writable_pte(spte) &&
1168               !(pt_protect && spte_is_locklessly_modifiable(spte)))
1169                 return false;
1170 
1171         rmap_printk("rmap_write_protect: spte %p %llx\n", sptep, *sptep);
1172 
1173         if (pt_protect)
1174                 spte &= ~SPTE_MMU_WRITEABLE;
1175         spte = spte & ~PT_WRITABLE_MASK;
1176 
1177         return mmu_spte_update(sptep, spte);
1178 }
1179 
1180 static bool __rmap_write_protect(struct kvm *kvm, unsigned long *rmapp,
1181                                  bool pt_protect)
1182 {
1183         u64 *sptep;
1184         struct rmap_iterator iter;
1185         bool flush = false;
1186 
1187         for (sptep = rmap_get_first(*rmapp, &iter); sptep;) {
1188                 BUG_ON(!(*sptep & PT_PRESENT_MASK));
1189 
1190                 flush |= spte_write_protect(kvm, sptep, pt_protect);
1191                 sptep = rmap_get_next(&iter);
1192         }
1193 
1194         return flush;
1195 }
1196 
1197 /**
1198  * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages
1199  * @kvm: kvm instance
1200  * @slot: slot to protect
1201  * @gfn_offset: start of the BITS_PER_LONG pages we care about
1202  * @mask: indicates which pages we should protect
1203  *
1204  * Used when we do not need to care about huge page mappings: e.g. during dirty
1205  * logging we do not have any such mappings.
1206  */
1207 void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
1208                                      struct kvm_memory_slot *slot,
1209                                      gfn_t gfn_offset, unsigned long mask)
1210 {
1211         unsigned long *rmapp;
1212 
1213         while (mask) {
1214                 rmapp = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
1215                                       PT_PAGE_TABLE_LEVEL, slot);
1216                 __rmap_write_protect(kvm, rmapp, false);
1217 
1218                 /* clear the first set bit */
1219                 mask &= mask - 1;
1220         }
1221 }
1222 
1223 static bool rmap_write_protect(struct kvm *kvm, u64 gfn)
1224 {
1225         struct kvm_memory_slot *slot;
1226         unsigned long *rmapp;
1227         int i;
1228         bool write_protected = false;
1229 
1230         slot = gfn_to_memslot(kvm, gfn);
1231 
1232         for (i = PT_PAGE_TABLE_LEVEL;
1233              i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
1234                 rmapp = __gfn_to_rmap(gfn, i, slot);
1235                 write_protected |= __rmap_write_protect(kvm, rmapp, true);
1236         }
1237 
1238         return write_protected;
1239 }
1240 
1241 static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
1242                            struct kvm_memory_slot *slot, gfn_t gfn, int level,
1243                            unsigned long data)
1244 {
1245         u64 *sptep;
1246         struct rmap_iterator iter;
1247         int need_tlb_flush = 0;
1248 
1249         while ((sptep = rmap_get_first(*rmapp, &iter))) {
1250                 BUG_ON(!(*sptep & PT_PRESENT_MASK));
1251                 rmap_printk("kvm_rmap_unmap_hva: spte %p %llx gfn %llx (%d)\n",
1252                              sptep, *sptep, gfn, level);
1253 
1254                 drop_spte(kvm, sptep);
1255                 need_tlb_flush = 1;
1256         }
1257 
1258         return need_tlb_flush;
1259 }
1260 
1261 static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp,
1262                              struct kvm_memory_slot *slot, gfn_t gfn, int level,
1263                              unsigned long data)
1264 {
1265         u64 *sptep;
1266         struct rmap_iterator iter;
1267         int need_flush = 0;
1268         u64 new_spte;
1269         pte_t *ptep = (pte_t *)data;
1270         pfn_t new_pfn;
1271 
1272         WARN_ON(pte_huge(*ptep));
1273         new_pfn = pte_pfn(*ptep);
1274 
1275         for (sptep = rmap_get_first(*rmapp, &iter); sptep;) {
1276                 BUG_ON(!is_shadow_present_pte(*sptep));
1277                 rmap_printk("kvm_set_pte_rmapp: spte %p %llx gfn %llx (%d)\n",
1278                              sptep, *sptep, gfn, level);
1279 
1280                 need_flush = 1;
1281 
1282                 if (pte_write(*ptep)) {
1283                         drop_spte(kvm, sptep);
1284                         sptep = rmap_get_first(*rmapp, &iter);
1285                 } else {
1286                         new_spte = *sptep & ~PT64_BASE_ADDR_MASK;
1287                         new_spte |= (u64)new_pfn << PAGE_SHIFT;
1288 
1289                         new_spte &= ~PT_WRITABLE_MASK;
1290                         new_spte &= ~SPTE_HOST_WRITEABLE;
1291                         new_spte &= ~shadow_accessed_mask;
1292 
1293                         mmu_spte_clear_track_bits(sptep);
1294                         mmu_spte_set(sptep, new_spte);
1295                         sptep = rmap_get_next(&iter);
1296                 }
1297         }
1298 
1299         if (need_flush)
1300                 kvm_flush_remote_tlbs(kvm);
1301 
1302         return 0;
1303 }
1304 
1305 static int kvm_handle_hva_range(struct kvm *kvm,
1306                                 unsigned long start,
1307                                 unsigned long end,
1308                                 unsigned long data,
1309                                 int (*handler)(struct kvm *kvm,
1310                                                unsigned long *rmapp,
1311                                                struct kvm_memory_slot *slot,
1312                                                gfn_t gfn,
1313                                                int level,
1314                                                unsigned long data))
1315 {
1316         int j;
1317         int ret = 0;
1318         struct kvm_memslots *slots;
1319         struct kvm_memory_slot *memslot;
1320 
1321         slots = kvm_memslots(kvm);
1322 
1323         kvm_for_each_memslot(memslot, slots) {
1324                 unsigned long hva_start, hva_end;
1325                 gfn_t gfn_start, gfn_end;
1326 
1327                 hva_start = max(start, memslot->userspace_addr);
1328                 hva_end = min(end, memslot->userspace_addr +
1329                                         (memslot->npages << PAGE_SHIFT));
1330                 if (hva_start >= hva_end)
1331                         continue;
1332                 /*
1333                  * {gfn(page) | page intersects with [hva_start, hva_end)} =
1334                  * {gfn_start, gfn_start+1, ..., gfn_end-1}.
1335                  */
1336                 gfn_start = hva_to_gfn_memslot(hva_start, memslot);
1337                 gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
1338 
1339                 for (j = PT_PAGE_TABLE_LEVEL;
1340                      j < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++j) {
1341                         unsigned long idx, idx_end;
1342                         unsigned long *rmapp;
1343                         gfn_t gfn = gfn_start;
1344 
1345                         /*
1346                          * {idx(page_j) | page_j intersects with
1347                          *  [hva_start, hva_end)} = {idx, idx+1, ..., idx_end}.
1348                          */
1349                         idx = gfn_to_index(gfn_start, memslot->base_gfn, j);
1350                         idx_end = gfn_to_index(gfn_end - 1, memslot->base_gfn, j);
1351 
1352                         rmapp = __gfn_to_rmap(gfn_start, j, memslot);
1353 
1354                         for (; idx <= idx_end;
1355                                ++idx, gfn += (1UL << KVM_HPAGE_GFN_SHIFT(j)))
1356                                 ret |= handler(kvm, rmapp++, memslot,
1357                                                gfn, j, data);
1358                 }
1359         }
1360 
1361         return ret;
1362 }
1363 
1364 static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
1365                           unsigned long data,
1366                           int (*handler)(struct kvm *kvm, unsigned long *rmapp,
1367                                          struct kvm_memory_slot *slot,
1368                                          gfn_t gfn, int level,
1369                                          unsigned long data))
1370 {
1371         return kvm_handle_hva_range(kvm, hva, hva + 1, data, handler);
1372 }
1373 
1374 int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
1375 {
1376         return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp);
1377 }
1378 
1379 int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
1380 {
1381         return kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp);
1382 }
1383 
1384 void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
1385 {
1386         kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
1387 }
1388 
1389 static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
1390                          struct kvm_memory_slot *slot, gfn_t gfn, int level,
1391                          unsigned long data)
1392 {
1393         u64 *sptep;
1394         struct rmap_iterator uninitialized_var(iter);
1395         int young = 0;
1396 
1397         BUG_ON(!shadow_accessed_mask);
1398 
1399         for (sptep = rmap_get_first(*rmapp, &iter); sptep;
1400              sptep = rmap_get_next(&iter)) {
1401                 BUG_ON(!is_shadow_present_pte(*sptep));
1402 
1403                 if (*sptep & shadow_accessed_mask) {
1404                         young = 1;
1405                         clear_bit((ffs(shadow_accessed_mask) - 1),
1406                                  (unsigned long *)sptep);
1407                 }
1408         }
1409         trace_kvm_age_page(gfn, level, slot, young);
1410         return young;
1411 }
1412 
1413 static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
1414                               struct kvm_memory_slot *slot, gfn_t gfn,
1415                               int level, unsigned long data)
1416 {
1417         u64 *sptep;
1418         struct rmap_iterator iter;
1419         int young = 0;
1420 
1421         /*
1422          * If there's no access bit in the secondary pte set by the
1423          * hardware it's up to gup-fast/gup to set the access bit in
1424          * the primary pte or in the page structure.
1425          */
1426         if (!shadow_accessed_mask)
1427                 goto out;
1428 
1429         for (sptep = rmap_get_first(*rmapp, &iter); sptep;
1430              sptep = rmap_get_next(&iter)) {
1431                 BUG_ON(!is_shadow_present_pte(*sptep));
1432 
1433                 if (*sptep & shadow_accessed_mask) {
1434                         young = 1;
1435                         break;
1436                 }
1437         }
1438 out:
1439         return young;
1440 }
1441 
1442 #define RMAP_RECYCLE_THRESHOLD 1000
1443 
1444 static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
1445 {
1446         unsigned long *rmapp;
1447         struct kvm_mmu_page *sp;
1448 
1449         sp = page_header(__pa(spte));
1450 
1451         rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
1452 
1453         kvm_unmap_rmapp(vcpu->kvm, rmapp, NULL, gfn, sp->role.level, 0);
1454         kvm_flush_remote_tlbs(vcpu->kvm);
1455 }
1456 
1457 int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
1458 {
1459         /*
1460          * In case of absence of EPT Access and Dirty Bits supports,
1461          * emulate the accessed bit for EPT, by checking if this page has
1462          * an EPT mapping, and clearing it if it does. On the next access,
1463          * a new EPT mapping will be established.
1464          * This has some overhead, but not as much as the cost of swapping
1465          * out actively used pages or breaking up actively used hugepages.
1466          */
1467         if (!shadow_accessed_mask) {
1468                 /*
1469                  * We are holding the kvm->mmu_lock, and we are blowing up
1470                  * shadow PTEs. MMU notifier consumers need to be kept at bay.
1471                  * This is correct as long as we don't decouple the mmu_lock
1472                  * protected regions (like invalidate_range_start|end does).
1473                  */
1474                 kvm->mmu_notifier_seq++;
1475                 return kvm_handle_hva_range(kvm, start, end, 0,
1476                                             kvm_unmap_rmapp);
1477         }
1478 
1479         return kvm_handle_hva_range(kvm, start, end, 0, kvm_age_rmapp);
1480 }
1481 
1482 int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
1483 {
1484         return kvm_handle_hva(kvm, hva, 0, kvm_test_age_rmapp);
1485 }
1486 
1487 #ifdef MMU_DEBUG
1488 static int is_empty_shadow_page(u64 *spt)
1489 {
1490         u64 *pos;
1491         u64 *end;
1492 
1493         for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
1494                 if (is_shadow_present_pte(*pos)) {
1495                         printk(KERN_ERR "%s: %p %llx\n", __func__,
1496                                pos, *pos);
1497                         return 0;
1498                 }
1499         return 1;
1500 }
1501 #endif
1502 
1503 /*
1504  * This value is the sum of all of the kvm instances's
1505  * kvm->arch.n_used_mmu_pages values.  We need a global,
1506  * aggregate version in order to make the slab shrinker
1507  * faster
1508  */
1509 static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, int nr)
1510 {
1511         kvm->arch.n_used_mmu_pages += nr;
1512         percpu_counter_add(&kvm_total_used_mmu_pages, nr);
1513 }
1514 
1515 static void kvm_mmu_free_page(struct kvm_mmu_page *sp)
1516 {
1517         ASSERT(is_empty_shadow_page(sp->spt));
1518         hlist_del(&sp->hash_link);
1519         list_del(&sp->link);
1520         free_page((unsigned long)sp->spt);
1521         if (!sp->role.direct)
1522                 free_page((unsigned long)sp->gfns);
1523         kmem_cache_free(mmu_page_header_cache, sp);
1524 }
1525 
1526 static unsigned kvm_page_table_hashfn(gfn_t gfn)
1527 {
1528         return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
1529 }
1530 
1531 static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
1532                                     struct kvm_mmu_page *sp, u64 *parent_pte)
1533 {
1534         if (!parent_pte)
1535                 return;
1536 
1537         pte_list_add(vcpu, parent_pte, &sp->parent_ptes);
1538 }
1539 
1540 static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
1541                                        u64 *parent_pte)
1542 {
1543         pte_list_remove(parent_pte, &sp->parent_ptes);
1544 }
1545 
1546 static void drop_parent_pte(struct kvm_mmu_page *sp,
1547                             u64 *parent_pte)
1548 {
1549         mmu_page_remove_parent_pte(sp, parent_pte);
1550         mmu_spte_clear_no_track(parent_pte);
1551 }
1552 
1553 static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
1554                                                u64 *parent_pte, int direct)
1555 {
1556         struct kvm_mmu_page *sp;
1557 
1558         sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache);
1559         sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
1560         if (!direct)
1561                 sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
1562         set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
1563 
1564         /*
1565          * The active_mmu_pages list is the FIFO list, do not move the
1566          * page until it is zapped. kvm_zap_obsolete_pages depends on
1567          * this feature. See the comments in kvm_zap_obsolete_pages().
1568          */
1569         list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
1570         sp->parent_ptes = 0;
1571         mmu_page_add_parent_pte(vcpu, sp, parent_pte);
1572         kvm_mod_used_mmu_pages(vcpu->kvm, +1);
1573         return sp;
1574 }
1575 
1576 static void mark_unsync(u64 *spte);
1577 static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
1578 {
1579         pte_list_walk(&sp->parent_ptes, mark_unsync);
1580 }
1581 
1582 static void mark_unsync(u64 *spte)
1583 {
1584         struct kvm_mmu_page *sp;
1585         unsigned int index;
1586 
1587         sp = page_header(__pa(spte));
1588         index = spte - sp->spt;
1589         if (__test_and_set_bit(index, sp->unsync_child_bitmap))
1590                 return;
1591         if (sp->unsync_children++)
1592                 return;
1593         kvm_mmu_mark_parents_unsync(sp);
1594 }
1595 
1596 static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
1597                                struct kvm_mmu_page *sp)
1598 {
1599         return 1;
1600 }
1601 
1602 static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
1603 {
1604 }
1605 
1606 static void nonpaging_update_pte(struct kvm_vcpu *vcpu,
1607                                  struct kvm_mmu_page *sp, u64 *spte,
1608                                  const void *pte)
1609 {
1610         WARN_ON(1);
1611 }
1612 
1613 #define KVM_PAGE_ARRAY_NR 16
1614 
1615 struct kvm_mmu_pages {
1616         struct mmu_page_and_offset {
1617                 struct kvm_mmu_page *sp;
1618                 unsigned int idx;
1619         } page[KVM_PAGE_ARRAY_NR];
1620         unsigned int nr;
1621 };
1622 
1623 static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
1624                          int idx)
1625 {
1626         int i;
1627 
1628         if (sp->unsync)
1629                 for (i=0; i < pvec->nr; i++)
1630                         if (pvec->page[i].sp == sp)
1631                                 return 0;
1632 
1633         pvec->page[pvec->nr].sp = sp;
1634         pvec->page[pvec->nr].idx = idx;
1635         pvec->nr++;
1636         return (pvec->nr == KVM_PAGE_ARRAY_NR);
1637 }
1638 
1639 static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
1640                            struct kvm_mmu_pages *pvec)
1641 {
1642         int i, ret, nr_unsync_leaf = 0;
1643 
1644         for_each_set_bit(i, sp->unsync_child_bitmap, 512) {
1645                 struct kvm_mmu_page *child;
1646                 u64 ent = sp->spt[i];
1647 
1648                 if (!is_shadow_present_pte(ent) || is_large_pte(ent))
1649                         goto clear_child_bitmap;
1650 
1651                 child = page_header(ent & PT64_BASE_ADDR_MASK);
1652 
1653                 if (child->unsync_children) {
1654                         if (mmu_pages_add(pvec, child, i))
1655                                 return -ENOSPC;
1656 
1657                         ret = __mmu_unsync_walk(child, pvec);
1658                         if (!ret)
1659                                 goto clear_child_bitmap;
1660                         else if (ret > 0)
1661                                 nr_unsync_leaf += ret;
1662                         else
1663                                 return ret;
1664                 } else if (child->unsync) {
1665                         nr_unsync_leaf++;
1666                         if (mmu_pages_add(pvec, child, i))
1667                                 return -ENOSPC;
1668                 } else
1669                          goto clear_child_bitmap;
1670 
1671                 continue;
1672 
1673 clear_child_bitmap:
1674                 __clear_bit(i, sp->unsync_child_bitmap);
1675                 sp->unsync_children--;
1676                 WARN_ON((int)sp->unsync_children < 0);
1677         }
1678 
1679 
1680         return nr_unsync_leaf;
1681 }
1682 
1683 static int mmu_unsync_walk(struct kvm_mmu_page *sp,
1684                            struct kvm_mmu_pages *pvec)
1685 {
1686         if (!sp->unsync_children)
1687                 return 0;
1688 
1689         mmu_pages_add(pvec, sp, 0);
1690         return __mmu_unsync_walk(sp, pvec);
1691 }
1692 
1693 static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1694 {
1695         WARN_ON(!sp->unsync);
1696         trace_kvm_mmu_sync_page(sp);
1697         sp->unsync = 0;
1698         --kvm->stat.mmu_unsync;
1699 }
1700 
1701 static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
1702                                     struct list_head *invalid_list);
1703 static void kvm_mmu_commit_zap_page(struct kvm *kvm,
1704                                     struct list_head *invalid_list);
1705 
1706 /*
1707  * NOTE: we should pay more attention on the zapped-obsolete page
1708  * (is_obsolete_sp(sp) && sp->role.invalid) when you do hash list walk
1709  * since it has been deleted from active_mmu_pages but still can be found
1710  * at hast list.
1711  *
1712  * for_each_gfn_indirect_valid_sp has skipped that kind of page and
1713  * kvm_mmu_get_page(), the only user of for_each_gfn_sp(), has skipped
1714  * all the obsolete pages.
1715  */
1716 #define for_each_gfn_sp(_kvm, _sp, _gfn)                                \
1717         hlist_for_each_entry(_sp,                                       \
1718           &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)], hash_link) \
1719                 if ((_sp)->gfn != (_gfn)) {} else
1720 
1721 #define for_each_gfn_indirect_valid_sp(_kvm, _sp, _gfn)                 \
1722         for_each_gfn_sp(_kvm, _sp, _gfn)                                \
1723                 if ((_sp)->role.direct || (_sp)->role.invalid) {} else
1724 
1725 /* @sp->gfn should be write-protected at the call site */
1726 static int __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
1727                            struct list_head *invalid_list, bool clear_unsync)
1728 {
1729         if (sp->role.cr4_pae != !!is_pae(vcpu)) {
1730                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1731                 return 1;
1732         }
1733 
1734         if (clear_unsync)
1735                 kvm_unlink_unsync_page(vcpu->kvm, sp);
1736 
1737         if (vcpu->arch.mmu.sync_page(vcpu, sp)) {
1738                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1739                 return 1;
1740         }
1741 
1742         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1743         return 0;
1744 }
1745 
1746 static int kvm_sync_page_transient(struct kvm_vcpu *vcpu,
1747                                    struct kvm_mmu_page *sp)
1748 {
1749         LIST_HEAD(invalid_list);
1750         int ret;
1751 
1752         ret = __kvm_sync_page(vcpu, sp, &invalid_list, false);
1753         if (ret)
1754                 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
1755 
1756         return ret;
1757 }
1758 
1759 #ifdef CONFIG_KVM_MMU_AUDIT
1760 #include "mmu_audit.c"
1761 #else
1762 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) { }
1763 static void mmu_audit_disable(void) { }
1764 #endif
1765 
1766 static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
1767                          struct list_head *invalid_list)
1768 {
1769         return __kvm_sync_page(vcpu, sp, invalid_list, true);
1770 }
1771 
1772 /* @gfn should be write-protected at the call site */
1773 static void kvm_sync_pages(struct kvm_vcpu *vcpu,  gfn_t gfn)
1774 {
1775         struct kvm_mmu_page *s;
1776         LIST_HEAD(invalid_list);
1777         bool flush = false;
1778 
1779         for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
1780                 if (!s->unsync)
1781                         continue;
1782 
1783                 WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
1784                 kvm_unlink_unsync_page(vcpu->kvm, s);
1785                 if ((s->role.cr4_pae != !!is_pae(vcpu)) ||
1786                         (vcpu->arch.mmu.sync_page(vcpu, s))) {
1787                         kvm_mmu_prepare_zap_page(vcpu->kvm, s, &invalid_list);
1788                         continue;
1789                 }
1790                 flush = true;
1791         }
1792 
1793         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
1794         if (flush)
1795                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1796 }
1797 
1798 struct mmu_page_path {
1799         struct kvm_mmu_page *parent[PT64_ROOT_LEVEL-1];
1800         unsigned int idx[PT64_ROOT_LEVEL-1];
1801 };
1802 
1803 #define for_each_sp(pvec, sp, parents, i)                       \
1804                 for (i = mmu_pages_next(&pvec, &parents, -1),   \
1805                         sp = pvec.page[i].sp;                   \
1806                         i < pvec.nr && ({ sp = pvec.page[i].sp; 1;});   \
1807                         i = mmu_pages_next(&pvec, &parents, i))
1808 
1809 static int mmu_pages_next(struct kvm_mmu_pages *pvec,
1810                           struct mmu_page_path *parents,
1811                           int i)
1812 {
1813         int n;
1814 
1815         for (n = i+1; n < pvec->nr; n++) {
1816                 struct kvm_mmu_page *sp = pvec->page[n].sp;
1817 
1818                 if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
1819                         parents->idx[0] = pvec->page[n].idx;
1820                         return n;
1821                 }
1822 
1823                 parents->parent[sp->role.level-2] = sp;
1824                 parents->idx[sp->role.level-1] = pvec->page[n].idx;
1825         }
1826 
1827         return n;
1828 }
1829 
1830 static void mmu_pages_clear_parents(struct mmu_page_path *parents)
1831 {
1832         struct kvm_mmu_page *sp;
1833         unsigned int level = 0;
1834 
1835         do {
1836                 unsigned int idx = parents->idx[level];
1837 
1838                 sp = parents->parent[level];
1839                 if (!sp)
1840                         return;
1841 
1842                 --sp->unsync_children;
1843                 WARN_ON((int)sp->unsync_children < 0);
1844                 __clear_bit(idx, sp->unsync_child_bitmap);
1845                 level++;
1846         } while (level < PT64_ROOT_LEVEL-1 && !sp->unsync_children);
1847 }
1848 
1849 static void kvm_mmu_pages_init(struct kvm_mmu_page *parent,
1850                                struct mmu_page_path *parents,
1851                                struct kvm_mmu_pages *pvec)
1852 {
1853         parents->parent[parent->role.level-1] = NULL;
1854         pvec->nr = 0;
1855 }
1856 
1857 static void mmu_sync_children(struct kvm_vcpu *vcpu,
1858                               struct kvm_mmu_page *parent)
1859 {
1860         int i;
1861         struct kvm_mmu_page *sp;
1862         struct mmu_page_path parents;
1863         struct kvm_mmu_pages pages;
1864         LIST_HEAD(invalid_list);
1865 
1866         kvm_mmu_pages_init(parent, &parents, &pages);
1867         while (mmu_unsync_walk(parent, &pages)) {
1868                 bool protected = false;
1869 
1870                 for_each_sp(pages, sp, parents, i)
1871                         protected |= rmap_write_protect(vcpu->kvm, sp->gfn);
1872 
1873                 if (protected)
1874                         kvm_flush_remote_tlbs(vcpu->kvm);
1875 
1876                 for_each_sp(pages, sp, parents, i) {
1877                         kvm_sync_page(vcpu, sp, &invalid_list);
1878                         mmu_pages_clear_parents(&parents);
1879                 }
1880                 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
1881                 cond_resched_lock(&vcpu->kvm->mmu_lock);
1882                 kvm_mmu_pages_init(parent, &parents, &pages);
1883         }
1884 }
1885 
1886 static void init_shadow_page_table(struct kvm_mmu_page *sp)
1887 {
1888         int i;
1889 
1890         for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
1891                 sp->spt[i] = 0ull;
1892 }
1893 
1894 static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp)
1895 {
1896         sp->write_flooding_count = 0;
1897 }
1898 
1899 static void clear_sp_write_flooding_count(u64 *spte)
1900 {
1901         struct kvm_mmu_page *sp =  page_header(__pa(spte));
1902 
1903         __clear_sp_write_flooding_count(sp);
1904 }
1905 
1906 static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp)
1907 {
1908         return unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen);
1909 }
1910 
1911 static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
1912                                              gfn_t gfn,
1913                                              gva_t gaddr,
1914                                              unsigned level,
1915                                              int direct,
1916                                              unsigned access,
1917                                              u64 *parent_pte)
1918 {
1919         union kvm_mmu_page_role role;
1920         unsigned quadrant;
1921         struct kvm_mmu_page *sp;
1922         bool need_sync = false;
1923 
1924         role = vcpu->arch.mmu.base_role;
1925         role.level = level;
1926         role.direct = direct;
1927         if (role.direct)
1928                 role.cr4_pae = 0;
1929         role.access = access;
1930         if (!vcpu->arch.mmu.direct_map
1931             && vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
1932                 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
1933                 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
1934                 role.quadrant = quadrant;
1935         }
1936         for_each_gfn_sp(vcpu->kvm, sp, gfn) {
1937                 if (is_obsolete_sp(vcpu->kvm, sp))
1938                         continue;
1939 
1940                 if (!need_sync && sp->unsync)
1941                         need_sync = true;
1942 
1943                 if (sp->role.word != role.word)
1944                         continue;
1945 
1946                 if (sp->unsync && kvm_sync_page_transient(vcpu, sp))
1947                         break;
1948 
1949                 mmu_page_add_parent_pte(vcpu, sp, parent_pte);
1950                 if (sp->unsync_children) {
1951                         kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
1952                         kvm_mmu_mark_parents_unsync(sp);
1953                 } else if (sp->unsync)
1954                         kvm_mmu_mark_parents_unsync(sp);
1955 
1956                 __clear_sp_write_flooding_count(sp);
1957                 trace_kvm_mmu_get_page(sp, false);
1958                 return sp;
1959         }
1960         ++vcpu->kvm->stat.mmu_cache_miss;
1961         sp = kvm_mmu_alloc_page(vcpu, parent_pte, direct);
1962         if (!sp)
1963                 return sp;
1964         sp->gfn = gfn;
1965         sp->role = role;
1966         hlist_add_head(&sp->hash_link,
1967                 &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]);
1968         if (!direct) {
1969                 if (rmap_write_protect(vcpu->kvm, gfn))
1970                         kvm_flush_remote_tlbs(vcpu->kvm);
1971                 if (level > PT_PAGE_TABLE_LEVEL && need_sync)
1972                         kvm_sync_pages(vcpu, gfn);
1973 
1974                 account_shadowed(vcpu->kvm, gfn);
1975         }
1976         sp->mmu_valid_gen = vcpu->kvm->arch.mmu_valid_gen;
1977         init_shadow_page_table(sp);
1978         trace_kvm_mmu_get_page(sp, true);
1979         return sp;
1980 }
1981 
1982 static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
1983                              struct kvm_vcpu *vcpu, u64 addr)
1984 {
1985         iterator->addr = addr;
1986         iterator->shadow_addr = vcpu->arch.mmu.root_hpa;
1987         iterator->level = vcpu->arch.mmu.shadow_root_level;
1988 
1989         if (iterator->level == PT64_ROOT_LEVEL &&
1990             vcpu->arch.mmu.root_level < PT64_ROOT_LEVEL &&
1991             !vcpu->arch.mmu.direct_map)
1992                 --iterator->level;
1993 
1994         if (iterator->level == PT32E_ROOT_LEVEL) {
1995                 iterator->shadow_addr
1996                         = vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
1997                 iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
1998                 --iterator->level;
1999                 if (!iterator->shadow_addr)
2000                         iterator->level = 0;
2001         }
2002 }
2003 
2004 static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
2005 {
2006         if (iterator->level < PT_PAGE_TABLE_LEVEL)
2007                 return false;
2008 
2009         iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
2010         iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
2011         return true;
2012 }
2013 
2014 static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator,
2015                                u64 spte)
2016 {
2017         if (is_last_spte(spte, iterator->level)) {
2018                 iterator->level = 0;
2019                 return;
2020         }
2021 
2022         iterator->shadow_addr = spte & PT64_BASE_ADDR_MASK;
2023         --iterator->level;
2024 }
2025 
2026 static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
2027 {
2028         return __shadow_walk_next(iterator, *iterator->sptep);
2029 }
2030 
2031 static void link_shadow_page(u64 *sptep, struct kvm_mmu_page *sp, bool accessed)
2032 {
2033         u64 spte;
2034 
2035         BUILD_BUG_ON(VMX_EPT_READABLE_MASK != PT_PRESENT_MASK ||
2036                         VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK);
2037 
2038         spte = __pa(sp->spt) | PT_PRESENT_MASK | PT_WRITABLE_MASK |
2039                shadow_user_mask | shadow_x_mask;
2040 
2041         if (accessed)
2042                 spte |= shadow_accessed_mask;
2043 
2044         mmu_spte_set(sptep, spte);
2045 }
2046 
2047 static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2048                                    unsigned direct_access)
2049 {
2050         if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
2051                 struct kvm_mmu_page *child;
2052 
2053                 /*
2054                  * For the direct sp, if the guest pte's dirty bit
2055                  * changed form clean to dirty, it will corrupt the
2056                  * sp's access: allow writable in the read-only sp,
2057                  * so we should update the spte at this point to get
2058                  * a new sp with the correct access.
2059                  */
2060                 child = page_header(*sptep & PT64_BASE_ADDR_MASK);
2061                 if (child->role.access == direct_access)
2062                         return;
2063 
2064                 drop_parent_pte(child, sptep);
2065                 kvm_flush_remote_tlbs(vcpu->kvm);
2066         }
2067 }
2068 
2069 static bool mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
2070                              u64 *spte)
2071 {
2072         u64 pte;
2073         struct kvm_mmu_page *child;
2074 
2075         pte = *spte;
2076         if (is_shadow_present_pte(pte)) {
2077                 if (is_last_spte(pte, sp->role.level)) {
2078                         drop_spte(kvm, spte);
2079                         if (is_large_pte(pte))
2080                                 --kvm->stat.lpages;
2081                 } else {
2082                         child = page_header(pte & PT64_BASE_ADDR_MASK);
2083                         drop_parent_pte(child, spte);
2084                 }
2085                 return true;
2086         }
2087 
2088         if (is_mmio_spte(pte))
2089                 mmu_spte_clear_no_track(spte);
2090 
2091         return false;
2092 }
2093 
2094 static void kvm_mmu_page_unlink_children(struct kvm *kvm,
2095                                          struct kvm_mmu_page *sp)
2096 {
2097         unsigned i;
2098 
2099         for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
2100                 mmu_page_zap_pte(kvm, sp, sp->spt + i);
2101 }
2102 
2103 static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
2104 {
2105         mmu_page_remove_parent_pte(sp, parent_pte);
2106 }
2107 
2108 static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
2109 {
2110         u64 *sptep;
2111         struct rmap_iterator iter;
2112 
2113         while ((sptep = rmap_get_first(sp->parent_ptes, &iter)))
2114                 drop_parent_pte(sp, sptep);
2115 }
2116 
2117 static int mmu_zap_unsync_children(struct kvm *kvm,
2118                                    struct kvm_mmu_page *parent,
2119                                    struct list_head *invalid_list)
2120 {
2121         int i, zapped = 0;
2122         struct mmu_page_path parents;
2123         struct kvm_mmu_pages pages;
2124 
2125         if (parent->role.level == PT_PAGE_TABLE_LEVEL)
2126                 return 0;
2127 
2128         kvm_mmu_pages_init(parent, &parents, &pages);
2129         while (mmu_unsync_walk(parent, &pages)) {
2130                 struct kvm_mmu_page *sp;
2131 
2132                 for_each_sp(pages, sp, parents, i) {
2133                         kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
2134                         mmu_pages_clear_parents(&parents);
2135                         zapped++;
2136                 }
2137                 kvm_mmu_pages_init(parent, &parents, &pages);
2138         }
2139 
2140         return zapped;
2141 }
2142 
2143 static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
2144                                     struct list_head *invalid_list)
2145 {
2146         int ret;
2147 
2148         trace_kvm_mmu_prepare_zap_page(sp);
2149         ++kvm->stat.mmu_shadow_zapped;
2150         ret = mmu_zap_unsync_children(kvm, sp, invalid_list);
2151         kvm_mmu_page_unlink_children(kvm, sp);
2152         kvm_mmu_unlink_parents(kvm, sp);
2153 
2154         if (!sp->role.invalid && !sp->role.direct)
2155                 unaccount_shadowed(kvm, sp->gfn);
2156 
2157         if (sp->unsync)
2158                 kvm_unlink_unsync_page(kvm, sp);
2159         if (!sp->root_count) {
2160                 /* Count self */
2161                 ret++;
2162                 list_move(&sp->link, invalid_list);
2163                 kvm_mod_used_mmu_pages(kvm, -1);
2164         } else {
2165                 list_move(&sp->link, &kvm->arch.active_mmu_pages);
2166 
2167                 /*
2168                  * The obsolete pages can not be used on any vcpus.
2169                  * See the comments in kvm_mmu_invalidate_zap_all_pages().
2170                  */
2171                 if (!sp->role.invalid && !is_obsolete_sp(kvm, sp))
2172                         kvm_reload_remote_mmus(kvm);
2173         }
2174 
2175         sp->role.invalid = 1;
2176         return ret;
2177 }
2178 
2179 static void kvm_mmu_commit_zap_page(struct kvm *kvm,
2180                                     struct list_head *invalid_list)
2181 {
2182         struct kvm_mmu_page *sp, *nsp;
2183 
2184         if (list_empty(invalid_list))
2185                 return;
2186 
2187         /*
2188          * wmb: make sure everyone sees our modifications to the page tables
2189          * rmb: make sure we see changes to vcpu->mode
2190          */
2191         smp_mb();
2192 
2193         /*
2194          * Wait for all vcpus to exit guest mode and/or lockless shadow
2195          * page table walks.
2196          */
2197         kvm_flush_remote_tlbs(kvm);
2198 
2199         list_for_each_entry_safe(sp, nsp, invalid_list, link) {
2200                 WARN_ON(!sp->role.invalid || sp->root_count);
2201                 kvm_mmu_free_page(sp);
2202         }
2203 }
2204 
2205 static bool prepare_zap_oldest_mmu_page(struct kvm *kvm,
2206                                         struct list_head *invalid_list)
2207 {
2208         struct kvm_mmu_page *sp;
2209 
2210         if (list_empty(&kvm->arch.active_mmu_pages))
2211                 return false;
2212 
2213         sp = list_entry(kvm->arch.active_mmu_pages.prev,
2214                         struct kvm_mmu_page, link);
2215         kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
2216 
2217         return true;
2218 }
2219 
2220 /*
2221  * Changing the number of mmu pages allocated to the vm
2222  * Note: if goal_nr_mmu_pages is too small, you will get dead lock
2223  */
2224 void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int goal_nr_mmu_pages)
2225 {
2226         LIST_HEAD(invalid_list);
2227 
2228         spin_lock(&kvm->mmu_lock);
2229 
2230         if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
2231                 /* Need to free some mmu pages to achieve the goal. */
2232                 while (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages)
2233                         if (!prepare_zap_oldest_mmu_page(kvm, &invalid_list))
2234                                 break;
2235 
2236                 kvm_mmu_commit_zap_page(kvm, &invalid_list);
2237                 goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
2238         }
2239 
2240         kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
2241 
2242         spin_unlock(&kvm->mmu_lock);
2243 }
2244 
2245 int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
2246 {
2247         struct kvm_mmu_page *sp;
2248         LIST_HEAD(invalid_list);
2249         int r;
2250 
2251         pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
2252         r = 0;
2253         spin_lock(&kvm->mmu_lock);
2254         for_each_gfn_indirect_valid_sp(kvm, sp, gfn) {
2255                 pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
2256                          sp->role.word);
2257                 r = 1;
2258                 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
2259         }
2260         kvm_mmu_commit_zap_page(kvm, &invalid_list);
2261         spin_unlock(&kvm->mmu_lock);
2262 
2263         return r;
2264 }
2265 EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page);
2266 
2267 /*
2268  * The function is based on mtrr_type_lookup() in
2269  * arch/x86/kernel/cpu/mtrr/generic.c
2270  */
2271 static int get_mtrr_type(struct mtrr_state_type *mtrr_state,
2272                          u64 start, u64 end)
2273 {
2274         int i;
2275         u64 base, mask;
2276         u8 prev_match, curr_match;
2277         int num_var_ranges = KVM_NR_VAR_MTRR;
2278 
2279         if (!mtrr_state->enabled)
2280                 return 0xFF;
2281 
2282         /* Make end inclusive end, instead of exclusive */
2283         end--;
2284 
2285         /* Look in fixed ranges. Just return the type as per start */
2286         if (mtrr_state->have_fixed && (start < 0x100000)) {
2287                 int idx;
2288 
2289                 if (start < 0x80000) {
2290                         idx = 0;
2291                         idx += (start >> 16);
2292                         return mtrr_state->fixed_ranges[idx];
2293                 } else if (start < 0xC0000) {
2294                         idx = 1 * 8;
2295                         idx += ((start - 0x80000) >> 14);
2296                         return mtrr_state->fixed_ranges[idx];
2297                 } else if (start < 0x1000000) {
2298                         idx = 3 * 8;
2299                         idx += ((start - 0xC0000) >> 12);
2300                         return mtrr_state->fixed_ranges[idx];
2301                 }
2302         }
2303 
2304         /*
2305          * Look in variable ranges
2306          * Look of multiple ranges matching this address and pick type
2307          * as per MTRR precedence
2308          */
2309         if (!(mtrr_state->enabled & 2))
2310                 return mtrr_state->def_type;
2311 
2312         prev_match = 0xFF;
2313         for (i = 0; i < num_var_ranges; ++i) {
2314                 unsigned short start_state, end_state;
2315 
2316                 if (!(mtrr_state->var_ranges[i].mask_lo & (1 << 11)))
2317                         continue;
2318 
2319                 base = (((u64)mtrr_state->var_ranges[i].base_hi) << 32) +
2320                        (mtrr_state->var_ranges[i].base_lo & PAGE_MASK);
2321                 mask = (((u64)mtrr_state->var_ranges[i].mask_hi) << 32) +
2322                        (mtrr_state->var_ranges[i].mask_lo & PAGE_MASK);
2323 
2324                 start_state = ((start & mask) == (base & mask));
2325                 end_state = ((end & mask) == (base & mask));
2326                 if (start_state != end_state)
2327                         return 0xFE;
2328 
2329                 if ((start & mask) != (base & mask))
2330                         continue;
2331 
2332                 curr_match = mtrr_state->var_ranges[i].base_lo & 0xff;
2333                 if (prev_match == 0xFF) {
2334                         prev_match = curr_match;
2335                         continue;
2336                 }
2337 
2338                 if (prev_match == MTRR_TYPE_UNCACHABLE ||
2339                     curr_match == MTRR_TYPE_UNCACHABLE)
2340                         return MTRR_TYPE_UNCACHABLE;
2341 
2342                 if ((prev_match == MTRR_TYPE_WRBACK &&
2343                      curr_match == MTRR_TYPE_WRTHROUGH) ||
2344                     (prev_match == MTRR_TYPE_WRTHROUGH &&
2345                      curr_match == MTRR_TYPE_WRBACK)) {
2346                         prev_match = MTRR_TYPE_WRTHROUGH;
2347                         curr_match = MTRR_TYPE_WRTHROUGH;
2348                 }
2349 
2350                 if (prev_match != curr_match)
2351                         return MTRR_TYPE_UNCACHABLE;
2352         }
2353 
2354         if (prev_match != 0xFF)
2355                 return prev_match;
2356 
2357         return mtrr_state->def_type;
2358 }
2359 
2360 u8 kvm_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn)
2361 {
2362         u8 mtrr;
2363 
2364         mtrr = get_mtrr_type(&vcpu->arch.mtrr_state, gfn << PAGE_SHIFT,
2365                              (gfn << PAGE_SHIFT) + PAGE_SIZE);
2366         if (mtrr == 0xfe || mtrr == 0xff)
2367                 mtrr = MTRR_TYPE_WRBACK;
2368         return mtrr;
2369 }
2370 EXPORT_SYMBOL_GPL(kvm_get_guest_memory_type);
2371 
2372 static void __kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
2373 {
2374         trace_kvm_mmu_unsync_page(sp);
2375         ++vcpu->kvm->stat.mmu_unsync;
2376         sp->unsync = 1;
2377 
2378         kvm_mmu_mark_parents_unsync(sp);
2379 }
2380 
2381 static void kvm_unsync_pages(struct kvm_vcpu *vcpu,  gfn_t gfn)
2382 {
2383         struct kvm_mmu_page *s;
2384 
2385         for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
2386                 if (s->unsync)
2387                         continue;
2388                 WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
2389                 __kvm_unsync_page(vcpu, s);
2390         }
2391 }
2392 
2393 static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
2394                                   bool can_unsync)
2395 {
2396         struct kvm_mmu_page *s;
2397         bool need_unsync = false;
2398 
2399         for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
2400                 if (!can_unsync)
2401                         return 1;
2402 
2403                 if (s->role.level != PT_PAGE_TABLE_LEVEL)
2404                         return 1;
2405 
2406                 if (!s->unsync)
2407                         need_unsync = true;
2408         }
2409         if (need_unsync)
2410                 kvm_unsync_pages(vcpu, gfn);
2411         return 0;
2412 }
2413 
2414 static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2415                     unsigned pte_access, int level,
2416                     gfn_t gfn, pfn_t pfn, bool speculative,
2417                     bool can_unsync, bool host_writable)
2418 {
2419         u64 spte;
2420         int ret = 0;
2421 
2422         if (set_mmio_spte(vcpu->kvm, sptep, gfn, pfn, pte_access))
2423                 return 0;
2424 
2425         spte = PT_PRESENT_MASK;
2426         if (!speculative)
2427                 spte |= shadow_accessed_mask;
2428 
2429         if (pte_access & ACC_EXEC_MASK)
2430                 spte |= shadow_x_mask;
2431         else
2432                 spte |= shadow_nx_mask;
2433 
2434         if (pte_access & ACC_USER_MASK)
2435                 spte |= shadow_user_mask;
2436 
2437         if (level > PT_PAGE_TABLE_LEVEL)
2438                 spte |= PT_PAGE_SIZE_MASK;
2439         if (tdp_enabled)
2440                 spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
2441                         kvm_is_reserved_pfn(pfn));
2442 
2443         if (host_writable)
2444                 spte |= SPTE_HOST_WRITEABLE;
2445         else
2446                 pte_access &= ~ACC_WRITE_MASK;
2447 
2448         spte |= (u64)pfn << PAGE_SHIFT;
2449 
2450         if (pte_access & ACC_WRITE_MASK) {
2451 
2452                 /*
2453                  * Other vcpu creates new sp in the window between
2454                  * mapping_level() and acquiring mmu-lock. We can
2455                  * allow guest to retry the access, the mapping can
2456                  * be fixed if guest refault.
2457                  */
2458                 if (level > PT_PAGE_TABLE_LEVEL &&
2459                     has_wrprotected_page(vcpu->kvm, gfn, level))
2460                         goto done;
2461 
2462                 spte |= PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE;
2463 
2464                 /*
2465                  * Optimization: for pte sync, if spte was writable the hash
2466                  * lookup is unnecessary (and expensive). Write protection
2467                  * is responsibility of mmu_get_page / kvm_sync_page.
2468                  * Same reasoning can be applied to dirty page accounting.
2469                  */
2470                 if (!can_unsync && is_writable_pte(*sptep))
2471                         goto set_pte;
2472 
2473                 if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
2474                         pgprintk("%s: found shadow page for %llx, marking ro\n",
2475                                  __func__, gfn);
2476                         ret = 1;
2477                         pte_access &= ~ACC_WRITE_MASK;
2478                         spte &= ~(PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE);
2479                 }
2480         }
2481 
2482         if (pte_access & ACC_WRITE_MASK)
2483                 mark_page_dirty(vcpu->kvm, gfn);
2484 
2485 set_pte:
2486         if (mmu_spte_update(sptep, spte))
2487                 kvm_flush_remote_tlbs(vcpu->kvm);
2488 done:
2489         return ret;
2490 }
2491 
2492 static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2493                          unsigned pte_access, int write_fault, int *emulate,
2494                          int level, gfn_t gfn, pfn_t pfn, bool speculative,
2495                          bool host_writable)
2496 {
2497         int was_rmapped = 0;
2498         int rmap_count;
2499 
2500         pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__,
2501                  *sptep, write_fault, gfn);
2502 
2503         if (is_rmap_spte(*sptep)) {
2504                 /*
2505                  * If we overwrite a PTE page pointer with a 2MB PMD, unlink
2506                  * the parent of the now unreachable PTE.
2507                  */
2508                 if (level > PT_PAGE_TABLE_LEVEL &&
2509                     !is_large_pte(*sptep)) {
2510                         struct kvm_mmu_page *child;
2511                         u64 pte = *sptep;
2512 
2513                         child = page_header(pte & PT64_BASE_ADDR_MASK);
2514                         drop_parent_pte(child, sptep);
2515                         kvm_flush_remote_tlbs(vcpu->kvm);
2516                 } else if (pfn != spte_to_pfn(*sptep)) {
2517                         pgprintk("hfn old %llx new %llx\n",
2518                                  spte_to_pfn(*sptep), pfn);
2519                         drop_spte(vcpu->kvm, sptep);
2520                         kvm_flush_remote_tlbs(vcpu->kvm);
2521                 } else
2522                         was_rmapped = 1;
2523         }
2524 
2525         if (set_spte(vcpu, sptep, pte_access, level, gfn, pfn, speculative,
2526               true, host_writable)) {
2527                 if (write_fault)
2528                         *emulate = 1;
2529                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2530         }
2531 
2532         if (unlikely(is_mmio_spte(*sptep) && emulate))
2533                 *emulate = 1;
2534 
2535         pgprintk("%s: setting spte %llx\n", __func__, *sptep);
2536         pgprintk("instantiating %s PTE (%s) at %llx (%llx) addr %p\n",
2537                  is_large_pte(*sptep)? "2MB" : "4kB",
2538                  *sptep & PT_PRESENT_MASK ?"RW":"R", gfn,
2539                  *sptep, sptep);
2540         if (!was_rmapped && is_large_pte(*sptep))
2541                 ++vcpu->kvm->stat.lpages;
2542 
2543         if (is_shadow_present_pte(*sptep)) {
2544                 if (!was_rmapped) {
2545                         rmap_count = rmap_add(vcpu, sptep, gfn);
2546                         if (rmap_count > RMAP_RECYCLE_THRESHOLD)
2547                                 rmap_recycle(vcpu, sptep, gfn);
2548                 }
2549         }
2550 
2551         kvm_release_pfn_clean(pfn);
2552 }
2553 
2554 static pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
2555                                      bool no_dirty_log)
2556 {
2557         struct kvm_memory_slot *slot;
2558 
2559         slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log);
2560         if (!slot)
2561                 return KVM_PFN_ERR_FAULT;
2562 
2563         return gfn_to_pfn_memslot_atomic(slot, gfn);
2564 }
2565 
2566 static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
2567                                     struct kvm_mmu_page *sp,
2568                                     u64 *start, u64 *end)
2569 {
2570         struct page *pages[PTE_PREFETCH_NUM];
2571         unsigned access = sp->role.access;
2572         int i, ret;
2573         gfn_t gfn;
2574 
2575         gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
2576         if (!gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK))
2577                 return -1;
2578 
2579         ret = gfn_to_page_many_atomic(vcpu->kvm, gfn, pages, end - start);
2580         if (ret <= 0)
2581                 return -1;
2582 
2583         for (i = 0; i < ret; i++, gfn++, start++)
2584                 mmu_set_spte(vcpu, start, access, 0, NULL,
2585                              sp->role.level, gfn, page_to_pfn(pages[i]),
2586                              true, true);
2587 
2588         return 0;
2589 }
2590 
2591 static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
2592                                   struct kvm_mmu_page *sp, u64 *sptep)
2593 {
2594         u64 *spte, *start = NULL;
2595         int i;
2596 
2597         WARN_ON(!sp->role.direct);
2598 
2599         i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
2600         spte = sp->spt + i;
2601 
2602         for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
2603                 if (is_shadow_present_pte(*spte) || spte == sptep) {
2604                         if (!start)
2605                                 continue;
2606                         if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
2607                                 break;
2608                         start = NULL;
2609                 } else if (!start)
2610                         start = spte;
2611         }
2612 }
2613 
2614 static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
2615 {
2616         struct kvm_mmu_page *sp;
2617 
2618         /*
2619          * Since it's no accessed bit on EPT, it's no way to
2620          * distinguish between actually accessed translations
2621          * and prefetched, so disable pte prefetch if EPT is
2622          * enabled.
2623          */
2624         if (!shadow_accessed_mask)
2625                 return;
2626 
2627         sp = page_header(__pa(sptep));
2628         if (sp->role.level > PT_PAGE_TABLE_LEVEL)
2629                 return;
2630 
2631         __direct_pte_prefetch(vcpu, sp, sptep);
2632 }
2633 
2634 static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
2635                         int map_writable, int level, gfn_t gfn, pfn_t pfn,
2636                         bool prefault)
2637 {
2638         struct kvm_shadow_walk_iterator iterator;
2639         struct kvm_mmu_page *sp;
2640         int emulate = 0;
2641         gfn_t pseudo_gfn;
2642 
2643         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
2644                 return 0;
2645 
2646         for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
2647                 if (iterator.level == level) {
2648                         mmu_set_spte(vcpu, iterator.sptep, ACC_ALL,
2649                                      write, &emulate, level, gfn, pfn,
2650                                      prefault, map_writable);
2651                         direct_pte_prefetch(vcpu, iterator.sptep);
2652                         ++vcpu->stat.pf_fixed;
2653                         break;
2654                 }
2655 
2656                 drop_large_spte(vcpu, iterator.sptep);
2657                 if (!is_shadow_present_pte(*iterator.sptep)) {
2658                         u64 base_addr = iterator.addr;
2659 
2660                         base_addr &= PT64_LVL_ADDR_MASK(iterator.level);
2661                         pseudo_gfn = base_addr >> PAGE_SHIFT;
2662                         sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
2663                                               iterator.level - 1,
2664                                               1, ACC_ALL, iterator.sptep);
2665 
2666                         link_shadow_page(iterator.sptep, sp, true);
2667                 }
2668         }
2669         return emulate;
2670 }
2671 
2672 static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk)
2673 {
2674         siginfo_t info;
2675 
2676         info.si_signo   = SIGBUS;
2677         info.si_errno   = 0;
2678         info.si_code    = BUS_MCEERR_AR;
2679         info.si_addr    = (void __user *)address;
2680         info.si_addr_lsb = PAGE_SHIFT;
2681 
2682         send_sig_info(SIGBUS, &info, tsk);
2683 }
2684 
2685 static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, pfn_t pfn)
2686 {
2687         /*
2688          * Do not cache the mmio info caused by writing the readonly gfn
2689          * into the spte otherwise read access on readonly gfn also can
2690          * caused mmio page fault and treat it as mmio access.
2691          * Return 1 to tell kvm to emulate it.
2692          */
2693         if (pfn == KVM_PFN_ERR_RO_FAULT)
2694                 return 1;
2695 
2696         if (pfn == KVM_PFN_ERR_HWPOISON) {
2697                 kvm_send_hwpoison_signal(gfn_to_hva(vcpu->kvm, gfn), current);
2698                 return 0;
2699         }
2700 
2701         return -EFAULT;
2702 }
2703 
2704 static void transparent_hugepage_adjust(struct kvm_vcpu *vcpu,
2705                                         gfn_t *gfnp, pfn_t *pfnp, int *levelp)
2706 {
2707         pfn_t pfn = *pfnp;
2708         gfn_t gfn = *gfnp;
2709         int level = *levelp;
2710 
2711         /*
2712          * Check if it's a transparent hugepage. If this would be an
2713          * hugetlbfs page, level wouldn't be set to
2714          * PT_PAGE_TABLE_LEVEL and there would be no adjustment done
2715          * here.
2716          */
2717         if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn) &&
2718             level == PT_PAGE_TABLE_LEVEL &&
2719             PageTransCompound(pfn_to_page(pfn)) &&
2720             !has_wrprotected_page(vcpu->kvm, gfn, PT_DIRECTORY_LEVEL)) {
2721                 unsigned long mask;
2722                 /*
2723                  * mmu_notifier_retry was successful and we hold the
2724                  * mmu_lock here, so the pmd can't become splitting
2725                  * from under us, and in turn
2726                  * __split_huge_page_refcount() can't run from under
2727                  * us and we can safely transfer the refcount from
2728                  * PG_tail to PG_head as we switch the pfn to tail to
2729                  * head.
2730                  */
2731                 *levelp = level = PT_DIRECTORY_LEVEL;
2732                 mask = KVM_PAGES_PER_HPAGE(level) - 1;
2733                 VM_BUG_ON((gfn & mask) != (pfn & mask));
2734                 if (pfn & mask) {
2735                         gfn &= ~mask;
2736                         *gfnp = gfn;
2737                         kvm_release_pfn_clean(pfn);
2738                         pfn &= ~mask;
2739                         kvm_get_pfn(pfn);
2740                         *pfnp = pfn;
2741                 }
2742         }
2743 }
2744 
2745 static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn,
2746                                 pfn_t pfn, unsigned access, int *ret_val)
2747 {
2748         bool ret = true;
2749 
2750         /* The pfn is invalid, report the error! */
2751         if (unlikely(is_error_pfn(pfn))) {
2752                 *ret_val = kvm_handle_bad_page(vcpu, gfn, pfn);
2753                 goto exit;
2754         }
2755 
2756         if (unlikely(is_noslot_pfn(pfn)))
2757                 vcpu_cache_mmio_info(vcpu, gva, gfn, access);
2758 
2759         ret = false;
2760 exit:
2761         return ret;
2762 }
2763 
2764 static bool page_fault_can_be_fast(u32 error_code)
2765 {
2766         /*
2767          * Do not fix the mmio spte with invalid generation number which
2768          * need to be updated by slow page fault path.
2769          */
2770         if (unlikely(error_code & PFERR_RSVD_MASK))
2771                 return false;
2772 
2773         /*
2774          * #PF can be fast only if the shadow page table is present and it
2775          * is caused by write-protect, that means we just need change the
2776          * W bit of the spte which can be done out of mmu-lock.
2777          */
2778         if (!(error_code & PFERR_PRESENT_MASK) ||
2779               !(error_code & PFERR_WRITE_MASK))
2780                 return false;
2781 
2782         return true;
2783 }
2784 
2785 static bool
2786 fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
2787                         u64 *sptep, u64 spte)
2788 {
2789         gfn_t gfn;
2790 
2791         WARN_ON(!sp->role.direct);
2792 
2793         /*
2794          * The gfn of direct spte is stable since it is calculated
2795          * by sp->gfn.
2796          */
2797         gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt);
2798 
2799         if (cmpxchg64(sptep, spte, spte | PT_WRITABLE_MASK) == spte)
2800                 mark_page_dirty(vcpu->kvm, gfn);
2801 
2802         return true;
2803 }
2804 
2805 /*
2806  * Return value:
2807  * - true: let the vcpu to access on the same address again.
2808  * - false: let the real page fault path to fix it.
2809  */
2810 static bool fast_page_fault(struct kvm_vcpu *vcpu, gva_t gva, int level,
2811                             u32 error_code)
2812 {
2813         struct kvm_shadow_walk_iterator iterator;
2814         struct kvm_mmu_page *sp;
2815         bool ret = false;
2816         u64 spte = 0ull;
2817 
2818         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
2819                 return false;
2820 
2821         if (!page_fault_can_be_fast(error_code))
2822                 return false;
2823 
2824         walk_shadow_page_lockless_begin(vcpu);
2825         for_each_shadow_entry_lockless(vcpu, gva, iterator, spte)
2826                 if (!is_shadow_present_pte(spte) || iterator.level < level)
2827                         break;
2828 
2829         /*
2830          * If the mapping has been changed, let the vcpu fault on the
2831          * same address again.
2832          */
2833         if (!is_rmap_spte(spte)) {
2834                 ret = true;
2835                 goto exit;
2836         }
2837 
2838         sp = page_header(__pa(iterator.sptep));
2839         if (!is_last_spte(spte, sp->role.level))
2840                 goto exit;
2841 
2842         /*
2843          * Check if it is a spurious fault caused by TLB lazily flushed.
2844          *
2845          * Need not check the access of upper level table entries since
2846          * they are always ACC_ALL.
2847          */
2848          if (is_writable_pte(spte)) {
2849                 ret = true;
2850                 goto exit;
2851         }
2852 
2853         /*
2854          * Currently, to simplify the code, only the spte write-protected
2855          * by dirty-log can be fast fixed.
2856          */
2857         if (!spte_is_locklessly_modifiable(spte))
2858                 goto exit;
2859 
2860         /*
2861          * Do not fix write-permission on the large spte since we only dirty
2862          * the first page into the dirty-bitmap in fast_pf_fix_direct_spte()
2863          * that means other pages are missed if its slot is dirty-logged.
2864          *
2865          * Instead, we let the slow page fault path create a normal spte to
2866          * fix the access.
2867          *
2868          * See the comments in kvm_arch_commit_memory_region().
2869          */
2870         if (sp->role.level > PT_PAGE_TABLE_LEVEL)
2871                 goto exit;
2872 
2873         /*
2874          * Currently, fast page fault only works for direct mapping since
2875          * the gfn is not stable for indirect shadow page.
2876          * See Documentation/virtual/kvm/locking.txt to get more detail.
2877          */
2878         ret = fast_pf_fix_direct_spte(vcpu, sp, iterator.sptep, spte);
2879 exit:
2880         trace_fast_page_fault(vcpu, gva, error_code, iterator.sptep,
2881                               spte, ret);
2882         walk_shadow_page_lockless_end(vcpu);
2883 
2884         return ret;
2885 }
2886 
2887 static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
2888                          gva_t gva, pfn_t *pfn, bool write, bool *writable);
2889 static void make_mmu_pages_available(struct kvm_vcpu *vcpu);
2890 
2891 static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, u32 error_code,
2892                          gfn_t gfn, bool prefault)
2893 {
2894         int r;
2895         int level;
2896         int force_pt_level;
2897         pfn_t pfn;
2898         unsigned long mmu_seq;
2899         bool map_writable, write = error_code & PFERR_WRITE_MASK;
2900 
2901         force_pt_level = mapping_level_dirty_bitmap(vcpu, gfn);
2902         if (likely(!force_pt_level)) {
2903                 level = mapping_level(vcpu, gfn);
2904                 /*
2905                  * This path builds a PAE pagetable - so we can map
2906                  * 2mb pages at maximum. Therefore check if the level
2907                  * is larger than that.
2908                  */
2909                 if (level > PT_DIRECTORY_LEVEL)
2910                         level = PT_DIRECTORY_LEVEL;
2911 
2912                 gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
2913         } else
2914                 level = PT_PAGE_TABLE_LEVEL;
2915 
2916         if (fast_page_fault(vcpu, v, level, error_code))
2917                 return 0;
2918 
2919         mmu_seq = vcpu->kvm->mmu_notifier_seq;
2920         smp_rmb();
2921 
2922         if (try_async_pf(vcpu, prefault, gfn, v, &pfn, write, &map_writable))
2923                 return 0;
2924 
2925         if (handle_abnormal_pfn(vcpu, v, gfn, pfn, ACC_ALL, &r))
2926                 return r;
2927 
2928         spin_lock(&vcpu->kvm->mmu_lock);
2929         if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
2930                 goto out_unlock;
2931         make_mmu_pages_available(vcpu);
2932         if (likely(!force_pt_level))
2933                 transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
2934         r = __direct_map(vcpu, v, write, map_writable, level, gfn, pfn,
2935                          prefault);
2936         spin_unlock(&vcpu->kvm->mmu_lock);
2937 
2938 
2939         return r;
2940 
2941 out_unlock:
2942         spin_unlock(&vcpu->kvm->mmu_lock);
2943         kvm_release_pfn_clean(pfn);
2944         return 0;
2945 }
2946 
2947 
2948 static void mmu_free_roots(struct kvm_vcpu *vcpu)
2949 {
2950         int i;
2951         struct kvm_mmu_page *sp;
2952         LIST_HEAD(invalid_list);
2953 
2954         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
2955                 return;
2956 
2957         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL &&
2958             (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL ||
2959              vcpu->arch.mmu.direct_map)) {
2960                 hpa_t root = vcpu->arch.mmu.root_hpa;
2961 
2962                 spin_lock(&vcpu->kvm->mmu_lock);
2963                 sp = page_header(root);
2964                 --sp->root_count;
2965                 if (!sp->root_count && sp->role.invalid) {
2966                         kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
2967                         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
2968                 }
2969                 spin_unlock(&vcpu->kvm->mmu_lock);
2970                 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
2971                 return;
2972         }
2973 
2974         spin_lock(&vcpu->kvm->mmu_lock);
2975         for (i = 0; i < 4; ++i) {
2976                 hpa_t root = vcpu->arch.mmu.pae_root[i];
2977 
2978                 if (root) {
2979                         root &= PT64_BASE_ADDR_MASK;
2980                         sp = page_header(root);
2981                         --sp->root_count;
2982                         if (!sp->root_count && sp->role.invalid)
2983                                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
2984                                                          &invalid_list);
2985                 }
2986                 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
2987         }
2988         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
2989         spin_unlock(&vcpu->kvm->mmu_lock);
2990         vcpu->arch.mmu.root_hpa = INVALID_PAGE;
2991 }
2992 
2993 static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
2994 {
2995         int ret = 0;
2996 
2997         if (!kvm_is_visible_gfn(vcpu->kvm, root_gfn)) {
2998                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
2999                 ret = 1;
3000         }
3001 
3002         return ret;
3003 }
3004 
3005 static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
3006 {
3007         struct kvm_mmu_page *sp;
3008         unsigned i;
3009 
3010         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
3011                 spin_lock(&vcpu->kvm->mmu_lock);
3012                 make_mmu_pages_available(vcpu);
3013                 sp = kvm_mmu_get_page(vcpu, 0, 0, PT64_ROOT_LEVEL,
3014                                       1, ACC_ALL, NULL);
3015                 ++sp->root_count;
3016                 spin_unlock(&vcpu->kvm->mmu_lock);
3017                 vcpu->arch.mmu.root_hpa = __pa(sp->spt);
3018         } else if (vcpu->arch.mmu.shadow_root_level == PT32E_ROOT_LEVEL) {
3019                 for (i = 0; i < 4; ++i) {
3020                         hpa_t root = vcpu->arch.mmu.pae_root[i];
3021 
3022                         ASSERT(!VALID_PAGE(root));
3023                         spin_lock(&vcpu->kvm->mmu_lock);
3024                         make_mmu_pages_available(vcpu);
3025                         sp = kvm_mmu_get_page(vcpu, i << (30 - PAGE_SHIFT),
3026                                               i << 30,
3027                                               PT32_ROOT_LEVEL, 1, ACC_ALL,
3028                                               NULL);
3029                         root = __pa(sp->spt);
3030                         ++sp->root_count;
3031                         spin_unlock(&vcpu->kvm->mmu_lock);
3032                         vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
3033                 }
3034                 vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
3035         } else
3036                 BUG();
3037 
3038         return 0;
3039 }
3040 
3041 static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
3042 {
3043         struct kvm_mmu_page *sp;
3044         u64 pdptr, pm_mask;
3045         gfn_t root_gfn;
3046         int i;
3047 
3048         root_gfn = vcpu->arch.mmu.get_cr3(vcpu) >> PAGE_SHIFT;
3049 
3050         if (mmu_check_root(vcpu, root_gfn))
3051                 return 1;
3052 
3053         /*
3054          * Do we shadow a long mode page table? If so we need to
3055          * write-protect the guests page table root.
3056          */
3057         if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
3058                 hpa_t root = vcpu->arch.mmu.root_hpa;
3059 
3060                 ASSERT(!VALID_PAGE(root));
3061 
3062                 spin_lock(&vcpu->kvm->mmu_lock);
3063                 make_mmu_pages_available(vcpu);
3064                 sp = kvm_mmu_get_page(vcpu, root_gfn, 0, PT64_ROOT_LEVEL,
3065                                       0, ACC_ALL, NULL);
3066                 root = __pa(sp->spt);
3067                 ++sp->root_count;
3068                 spin_unlock(&vcpu->kvm->mmu_lock);
3069                 vcpu->arch.mmu.root_hpa = root;
3070                 return 0;
3071         }
3072 
3073         /*
3074          * We shadow a 32 bit page table. This may be a legacy 2-level
3075          * or a PAE 3-level page table. In either case we need to be aware that
3076          * the shadow page table may be a PAE or a long mode page table.
3077          */
3078         pm_mask = PT_PRESENT_MASK;
3079         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL)
3080                 pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;
3081 
3082         for (i = 0; i < 4; ++i) {
3083                 hpa_t root = vcpu->arch.mmu.pae_root[i];
3084 
3085                 ASSERT(!VALID_PAGE(root));
3086                 if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
3087                         pdptr = vcpu->arch.mmu.get_pdptr(vcpu, i);
3088                         if (!is_present_gpte(pdptr)) {
3089                                 vcpu->arch.mmu.pae_root[i] = 0;
3090                                 continue;
3091                         }
3092                         root_gfn = pdptr >> PAGE_SHIFT;
3093                         if (mmu_check_root(vcpu, root_gfn))
3094                                 return 1;
3095                 }
3096                 spin_lock(&vcpu->kvm->mmu_lock);
3097                 make_mmu_pages_available(vcpu);
3098                 sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
3099                                       PT32_ROOT_LEVEL, 0,
3100                                       ACC_ALL, NULL);
3101                 root = __pa(sp->spt);
3102                 ++sp->root_count;
3103                 spin_unlock(&vcpu->kvm->mmu_lock);
3104 
3105                 vcpu->arch.mmu.pae_root[i] = root | pm_mask;
3106         }
3107         vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
3108 
3109         /*
3110          * If we shadow a 32 bit page table with a long mode page
3111          * table we enter this path.
3112          */
3113         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
3114                 if (vcpu->arch.mmu.lm_root == NULL) {
3115                         /*
3116                          * The additional page necessary for this is only
3117                          * allocated on demand.
3118                          */
3119 
3120                         u64 *lm_root;
3121 
3122                         lm_root = (void*)get_zeroed_page(GFP_KERNEL);
3123                         if (lm_root == NULL)
3124                                 return 1;
3125 
3126                         lm_root[0] = __pa(vcpu->arch.mmu.pae_root) | pm_mask;
3127 
3128                         vcpu->arch.mmu.lm_root = lm_root;
3129                 }
3130 
3131                 vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.lm_root);
3132         }
3133 
3134         return 0;
3135 }
3136 
3137 static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
3138 {
3139         if (vcpu->arch.mmu.direct_map)
3140                 return mmu_alloc_direct_roots(vcpu);
3141         else
3142                 return mmu_alloc_shadow_roots(vcpu);
3143 }
3144 
3145 static void mmu_sync_roots(struct kvm_vcpu *vcpu)
3146 {
3147         int i;
3148         struct kvm_mmu_page *sp;
3149 
3150         if (vcpu->arch.mmu.direct_map)
3151                 return;
3152 
3153         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
3154                 return;
3155 
3156         vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
3157         kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
3158         if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
3159                 hpa_t root = vcpu->arch.mmu.root_hpa;
3160                 sp = page_header(root);
3161                 mmu_sync_children(vcpu, sp);
3162                 kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3163                 return;
3164         }
3165         for (i = 0; i < 4; ++i) {
3166                 hpa_t root = vcpu->arch.mmu.pae_root[i];
3167 
3168                 if (root && VALID_PAGE(root)) {
3169                         root &= PT64_BASE_ADDR_MASK;
3170                         sp = page_header(root);
3171                         mmu_sync_children(vcpu, sp);
3172                 }
3173         }
3174         kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3175 }
3176 
3177 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
3178 {
3179         spin_lock(&vcpu->kvm->mmu_lock);
3180         mmu_sync_roots(vcpu);
3181         spin_unlock(&vcpu->kvm->mmu_lock);
3182 }
3183 EXPORT_SYMBOL_GPL(kvm_mmu_sync_roots);
3184 
3185 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr,
3186                                   u32 access, struct x86_exception *exception)
3187 {
3188         if (exception)
3189                 exception->error_code = 0;
3190         return vaddr;
3191 }
3192 
3193 static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gva_t vaddr,
3194                                          u32 access,
3195                                          struct x86_exception *exception)
3196 {
3197         if (exception)
3198                 exception->error_code = 0;
3199         return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access, exception);
3200 }
3201 
3202 static bool quickly_check_mmio_pf(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3203 {
3204         if (direct)
3205                 return vcpu_match_mmio_gpa(vcpu, addr);
3206 
3207         return vcpu_match_mmio_gva(vcpu, addr);
3208 }
3209 
3210 static u64 walk_shadow_page_get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr)
3211 {
3212         struct kvm_shadow_walk_iterator iterator;
3213         u64 spte = 0ull;
3214 
3215         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
3216                 return spte;
3217 
3218         walk_shadow_page_lockless_begin(vcpu);
3219         for_each_shadow_entry_lockless(vcpu, addr, iterator, spte)
3220                 if (!is_shadow_present_pte(spte))
3221                         break;
3222         walk_shadow_page_lockless_end(vcpu);
3223 
3224         return spte;
3225 }
3226 
3227 int handle_mmio_page_fault_common(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3228 {
3229         u64 spte;
3230 
3231         if (quickly_check_mmio_pf(vcpu, addr, direct))
3232                 return RET_MMIO_PF_EMULATE;
3233 
3234         spte = walk_shadow_page_get_mmio_spte(vcpu, addr);
3235 
3236         if (is_mmio_spte(spte)) {
3237                 gfn_t gfn = get_mmio_spte_gfn(spte);
3238                 unsigned access = get_mmio_spte_access(spte);
3239 
3240                 if (!check_mmio_spte(vcpu->kvm, spte))
3241                         return RET_MMIO_PF_INVALID;
3242 
3243                 if (direct)
3244                         addr = 0;
3245 
3246                 trace_handle_mmio_page_fault(addr, gfn, access);
3247                 vcpu_cache_mmio_info(vcpu, addr, gfn, access);
3248                 return RET_MMIO_PF_EMULATE;
3249         }
3250 
3251         /*
3252          * If the page table is zapped by other cpus, let CPU fault again on
3253          * the address.
3254          */
3255         return RET_MMIO_PF_RETRY;
3256 }
3257 EXPORT_SYMBOL_GPL(handle_mmio_page_fault_common);
3258 
3259 static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr,
3260                                   u32 error_code, bool direct)
3261 {
3262         int ret;
3263 
3264         ret = handle_mmio_page_fault_common(vcpu, addr, direct);
3265         WARN_ON(ret == RET_MMIO_PF_BUG);
3266         return ret;
3267 }
3268 
3269 static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
3270                                 u32 error_code, bool prefault)
3271 {
3272         gfn_t gfn;
3273         int r;
3274 
3275         pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
3276 
3277         if (unlikely(error_code & PFERR_RSVD_MASK)) {
3278                 r = handle_mmio_page_fault(vcpu, gva, error_code, true);
3279 
3280                 if (likely(r != RET_MMIO_PF_INVALID))
3281                         return r;
3282         }
3283 
3284         r = mmu_topup_memory_caches(vcpu);
3285         if (r)
3286                 return r;
3287 
3288         ASSERT(vcpu);
3289         ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
3290 
3291         gfn = gva >> PAGE_SHIFT;
3292 
3293         return nonpaging_map(vcpu, gva & PAGE_MASK,
3294                              error_code, gfn, prefault);
3295 }
3296 
3297 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn)
3298 {
3299         struct kvm_arch_async_pf arch;
3300 
3301         arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
3302         arch.gfn = gfn;
3303         arch.direct_map = vcpu->arch.mmu.direct_map;
3304         arch.cr3 = vcpu->arch.mmu.get_cr3(vcpu);
3305 
3306         return kvm_setup_async_pf(vcpu, gva, gfn_to_hva(vcpu->kvm, gfn), &arch);
3307 }
3308 
3309 static bool can_do_async_pf(struct kvm_vcpu *vcpu)
3310 {
3311         if (unlikely(!irqchip_in_kernel(vcpu->kvm) ||
3312                      kvm_event_needs_reinjection(vcpu)))
3313                 return false;
3314 
3315         return kvm_x86_ops->interrupt_allowed(vcpu);
3316 }
3317 
3318 static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
3319                          gva_t gva, pfn_t *pfn, bool write, bool *writable)
3320 {
3321         bool async;
3322 
3323         *pfn = gfn_to_pfn_async(vcpu->kvm, gfn, &async, write, writable);
3324 
3325         if (!async)
3326                 return false; /* *pfn has correct page already */
3327 
3328         if (!prefault && can_do_async_pf(vcpu)) {
3329                 trace_kvm_try_async_get_page(gva, gfn);
3330                 if (kvm_find_async_pf_gfn(vcpu, gfn)) {
3331                         trace_kvm_async_pf_doublefault(gva, gfn);
3332                         kvm_make_request(KVM_REQ_APF_HALT, vcpu);
3333                         return true;
3334                 } else if (kvm_arch_setup_async_pf(vcpu, gva, gfn))
3335                         return true;
3336         }
3337 
3338         *pfn = gfn_to_pfn_prot(vcpu->kvm, gfn, write, writable);
3339 
3340         return false;
3341 }
3342 
3343 static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code,
3344                           bool prefault)
3345 {
3346         pfn_t pfn;
3347         int r;
3348         int level;
3349         int force_pt_level;
3350         gfn_t gfn = gpa >> PAGE_SHIFT;
3351         unsigned long mmu_seq;
3352         int write = error_code & PFERR_WRITE_MASK;
3353         bool map_writable;
3354 
3355         ASSERT(vcpu);
3356         ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
3357 
3358         if (unlikely(error_code & PFERR_RSVD_MASK)) {
3359                 r = handle_mmio_page_fault(vcpu, gpa, error_code, true);
3360 
3361                 if (likely(r != RET_MMIO_PF_INVALID))
3362                         return r;
3363         }
3364 
3365         r = mmu_topup_memory_caches(vcpu);
3366         if (r)
3367                 return r;
3368 
3369         force_pt_level = mapping_level_dirty_bitmap(vcpu, gfn);
3370         if (likely(!force_pt_level)) {
3371                 level = mapping_level(vcpu, gfn);
3372                 gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
3373         } else
3374                 level = PT_PAGE_TABLE_LEVEL;
3375 
3376         if (fast_page_fault(vcpu, gpa, level, error_code))
3377                 return 0;
3378 
3379         mmu_seq = vcpu->kvm->mmu_notifier_seq;
3380         smp_rmb();
3381 
3382         if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable))
3383                 return 0;
3384 
3385         if (handle_abnormal_pfn(vcpu, 0, gfn, pfn, ACC_ALL, &r))
3386                 return r;
3387 
3388         spin_lock(&vcpu->kvm->mmu_lock);
3389         if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
3390                 goto out_unlock;
3391         make_mmu_pages_available(vcpu);
3392         if (likely(!force_pt_level))
3393                 transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
3394         r = __direct_map(vcpu, gpa, write, map_writable,
3395                          level, gfn, pfn, prefault);
3396         spin_unlock(&vcpu->kvm->mmu_lock);
3397 
3398         return r;
3399 
3400 out_unlock:
3401         spin_unlock(&vcpu->kvm->mmu_lock);
3402         kvm_release_pfn_clean(pfn);
3403         return 0;
3404 }
3405 
3406 static void nonpaging_init_context(struct kvm_vcpu *vcpu,
3407                                    struct kvm_mmu *context)
3408 {
3409         context->page_fault = nonpaging_page_fault;
3410         context->gva_to_gpa = nonpaging_gva_to_gpa;
3411         context->sync_page = nonpaging_sync_page;
3412         context->invlpg = nonpaging_invlpg;
3413         context->update_pte = nonpaging_update_pte;
3414         context->root_level = 0;
3415         context->shadow_root_level = PT32E_ROOT_LEVEL;
3416         context->root_hpa = INVALID_PAGE;
3417         context->direct_map = true;
3418         context->nx = false;
3419 }
3420 
3421 void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu)
3422 {
3423         mmu_free_roots(vcpu);
3424 }
3425 
3426 static unsigned long get_cr3(struct kvm_vcpu *vcpu)
3427 {
3428         return kvm_read_cr3(vcpu);
3429 }
3430 
3431 static void inject_page_fault(struct kvm_vcpu *vcpu,
3432                               struct x86_exception *fault)
3433 {
3434         vcpu->arch.mmu.inject_page_fault(vcpu, fault);
3435 }
3436 
3437 static bool sync_mmio_spte(struct kvm *kvm, u64 *sptep, gfn_t gfn,
3438                            unsigned access, int *nr_present)
3439 {
3440         if (unlikely(is_mmio_spte(*sptep))) {
3441                 if (gfn != get_mmio_spte_gfn(*sptep)) {
3442                         mmu_spte_clear_no_track(sptep);
3443                         return true;
3444                 }
3445 
3446                 (*nr_present)++;
3447                 mark_mmio_spte(kvm, sptep, gfn, access);
3448                 return true;
3449         }
3450 
3451         return false;
3452 }
3453 
3454 static inline bool is_last_gpte(struct kvm_mmu *mmu, unsigned level, unsigned gpte)
3455 {
3456         unsigned index;
3457 
3458         index = level - 1;
3459         index |= (gpte & PT_PAGE_SIZE_MASK) >> (PT_PAGE_SIZE_SHIFT - 2);
3460         return mmu->last_pte_bitmap & (1 << index);
3461 }
3462 
3463 #define PTTYPE_EPT 18 /* arbitrary */
3464 #define PTTYPE PTTYPE_EPT
3465 #include "paging_tmpl.h"
3466 #undef PTTYPE
3467 
3468 #define PTTYPE 64
3469 #include "paging_tmpl.h"
3470 #undef PTTYPE
3471 
3472 #define PTTYPE 32
3473 #include "paging_tmpl.h"
3474 #undef PTTYPE
3475 
3476 static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
3477                                   struct kvm_mmu *context)
3478 {
3479         int maxphyaddr = cpuid_maxphyaddr(vcpu);
3480         u64 exb_bit_rsvd = 0;
3481         u64 gbpages_bit_rsvd = 0;
3482         u64 nonleaf_bit8_rsvd = 0;
3483 
3484         context->bad_mt_xwr = 0;
3485 
3486         if (!context->nx)
3487                 exb_bit_rsvd = rsvd_bits(63, 63);
3488         if (!guest_cpuid_has_gbpages(vcpu))
3489                 gbpages_bit_rsvd = rsvd_bits(7, 7);
3490 
3491         /*
3492          * Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for
3493          * leaf entries) on AMD CPUs only.
3494          */
3495         if (guest_cpuid_is_amd(vcpu))
3496                 nonleaf_bit8_rsvd = rsvd_bits(8, 8);
3497 
3498         switch (context->root_level) {
3499         case PT32_ROOT_LEVEL:
3500                 /* no rsvd bits for 2 level 4K page table entries */
3501                 context->rsvd_bits_mask[0][1] = 0;
3502                 context->rsvd_bits_mask[0][0] = 0;
3503                 context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
3504 
3505                 if (!is_pse(vcpu)) {
3506                         context->rsvd_bits_mask[1][1] = 0;
3507                         break;
3508                 }
3509 
3510                 if (is_cpuid_PSE36())
3511                         /* 36bits PSE 4MB page */
3512                         context->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
3513                 else
3514                         /* 32 bits PSE 4MB page */
3515                         context->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
3516                 break;
3517         case PT32E_ROOT_LEVEL:
3518                 context->rsvd_bits_mask[0][2] =
3519                         rsvd_bits(maxphyaddr, 63) |
3520                         rsvd_bits(5, 8) | rsvd_bits(1, 2);      /* PDPTE */
3521                 context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
3522                         rsvd_bits(maxphyaddr, 62);      /* PDE */
3523                 context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
3524                         rsvd_bits(maxphyaddr, 62);      /* PTE */
3525                 context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
3526                         rsvd_bits(maxphyaddr, 62) |
3527                         rsvd_bits(13, 20);              /* large page */
3528                 context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
3529                 break;
3530         case PT64_ROOT_LEVEL:
3531                 context->rsvd_bits_mask[0][3] = exb_bit_rsvd |
3532                         nonleaf_bit8_rsvd | rsvd_bits(7, 7) | rsvd_bits(maxphyaddr, 51);
3533                 context->rsvd_bits_mask[0][2] = exb_bit_rsvd |
3534                         nonleaf_bit8_rsvd | gbpages_bit_rsvd | rsvd_bits(maxphyaddr, 51);
3535                 context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
3536                         rsvd_bits(maxphyaddr, 51);
3537                 context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
3538                         rsvd_bits(maxphyaddr, 51);
3539                 context->rsvd_bits_mask[1][3] = context->rsvd_bits_mask[0][3];
3540                 context->rsvd_bits_mask[1][2] = exb_bit_rsvd |
3541                         gbpages_bit_rsvd | rsvd_bits(maxphyaddr, 51) |
3542                         rsvd_bits(13, 29);
3543                 context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
3544                         rsvd_bits(maxphyaddr, 51) |
3545                         rsvd_bits(13, 20);              /* large page */
3546                 context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
3547                 break;
3548         }
3549 }
3550 
3551 static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu,
3552                 struct kvm_mmu *context, bool execonly)
3553 {
3554         int maxphyaddr = cpuid_maxphyaddr(vcpu);
3555         int pte;
3556 
3557         context->rsvd_bits_mask[0][3] =
3558                 rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 7);
3559         context->rsvd_bits_mask[0][2] =
3560                 rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 6);
3561         context->rsvd_bits_mask[0][1] =
3562                 rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 6);
3563         context->rsvd_bits_mask[0][0] = rsvd_bits(maxphyaddr, 51);
3564 
3565         /* large page */
3566         context->rsvd_bits_mask[1][3] = context->rsvd_bits_mask[0][3];
3567         context->rsvd_bits_mask[1][2] =
3568                 rsvd_bits(maxphyaddr, 51) | rsvd_bits(12, 29);
3569         context->rsvd_bits_mask[1][1] =
3570                 rsvd_bits(maxphyaddr, 51) | rsvd_bits(12, 20);
3571         context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
3572 
3573         for (pte = 0; pte < 64; pte++) {
3574                 int rwx_bits = pte & 7;
3575                 int mt = pte >> 3;
3576                 if (mt == 0x2 || mt == 0x3 || mt == 0x7 ||
3577                                 rwx_bits == 0x2 || rwx_bits == 0x6 ||
3578                                 (rwx_bits == 0x4 && !execonly))
3579                         context->bad_mt_xwr |= (1ull << pte);
3580         }
3581 }
3582 
3583 static void update_permission_bitmask(struct kvm_vcpu *vcpu,
3584                                       struct kvm_mmu *mmu, bool ept)
3585 {
3586         unsigned bit, byte, pfec;
3587         u8 map;
3588         bool fault, x, w, u, wf, uf, ff, smapf, cr4_smap, cr4_smep, smap = 0;
3589 
3590         cr4_smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
3591         cr4_smap = kvm_read_cr4_bits(vcpu, X86_CR4_SMAP);
3592         for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {
3593                 pfec = byte << 1;
3594                 map = 0;
3595                 wf = pfec & PFERR_WRITE_MASK;
3596                 uf = pfec & PFERR_USER_MASK;
3597                 ff = pfec & PFERR_FETCH_MASK;
3598                 /*
3599                  * PFERR_RSVD_MASK bit is set in PFEC if the access is not
3600                  * subject to SMAP restrictions, and cleared otherwise. The
3601                  * bit is only meaningful if the SMAP bit is set in CR4.
3602                  */
3603                 smapf = !(pfec & PFERR_RSVD_MASK);
3604                 for (bit = 0; bit < 8; ++bit) {
3605                         x = bit & ACC_EXEC_MASK;
3606                         w = bit & ACC_WRITE_MASK;
3607                         u = bit & ACC_USER_MASK;
3608 
3609                         if (!ept) {
3610                                 /* Not really needed: !nx will cause pte.nx to fault */
3611                                 x |= !mmu->nx;
3612                                 /* Allow supervisor writes if !cr0.wp */
3613                                 w |= !is_write_protection(vcpu) && !uf;
3614                                 /* Disallow supervisor fetches of user code if cr4.smep */
3615                                 x &= !(cr4_smep && u && !uf);
3616 
3617                                 /*
3618                                  * SMAP:kernel-mode data accesses from user-mode
3619                                  * mappings should fault. A fault is considered
3620                                  * as a SMAP violation if all of the following
3621                                  * conditions are ture:
3622                                  *   - X86_CR4_SMAP is set in CR4
3623                                  *   - An user page is accessed
3624                                  *   - Page fault in kernel mode
3625                                  *   - if CPL = 3 or X86_EFLAGS_AC is clear
3626                                  *
3627                                  *   Here, we cover the first three conditions.
3628                                  *   The fourth is computed dynamically in
3629                                  *   permission_fault() and is in smapf.
3630                                  *
3631                                  *   Also, SMAP does not affect instruction
3632                                  *   fetches, add the !ff check here to make it
3633                                  *   clearer.
3634                                  */
3635                                 smap = cr4_smap && u && !uf && !ff;
3636                         } else
3637                                 /* Not really needed: no U/S accesses on ept  */
3638                                 u = 1;
3639 
3640                         fault = (ff && !x) || (uf && !u) || (wf && !w) ||
3641                                 (smapf && smap);
3642                         map |= fault << bit;
3643                 }
3644                 mmu->permissions[byte] = map;
3645         }
3646 }
3647 
3648 static void update_last_pte_bitmap(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
3649 {
3650         u8 map;
3651         unsigned level, root_level = mmu->root_level;
3652         const unsigned ps_set_index = 1 << 2;  /* bit 2 of index: ps */
3653 
3654         if (root_level == PT32E_ROOT_LEVEL)
3655                 --root_level;
3656         /* PT_PAGE_TABLE_LEVEL always terminates */
3657         map = 1 | (1 << ps_set_index);
3658         for (level = PT_DIRECTORY_LEVEL; level <= root_level; ++level) {
3659                 if (level <= PT_PDPE_LEVEL
3660                     && (mmu->root_level >= PT32E_ROOT_LEVEL || is_pse(vcpu)))
3661                         map |= 1 << (ps_set_index | (level - 1));
3662         }
3663         mmu->last_pte_bitmap = map;
3664 }
3665 
3666 static void paging64_init_context_common(struct kvm_vcpu *vcpu,
3667                                          struct kvm_mmu *context,
3668                                          int level)
3669 {
3670         context->nx = is_nx(vcpu);
3671         context->root_level = level;
3672 
3673         reset_rsvds_bits_mask(vcpu, context);
3674         update_permission_bitmask(vcpu, context, false);
3675         update_last_pte_bitmap(vcpu, context);
3676 
3677         ASSERT(is_pae(vcpu));
3678         context->page_fault = paging64_page_fault;
3679         context->gva_to_gpa = paging64_gva_to_gpa;
3680         context->sync_page = paging64_sync_page;
3681         context->invlpg = paging64_invlpg;
3682         context->update_pte = paging64_update_pte;
3683         context->shadow_root_level = level;
3684         context->root_hpa = INVALID_PAGE;
3685         context->direct_map = false;
3686 }
3687 
3688 static void paging64_init_context(struct kvm_vcpu *vcpu,
3689                                   struct kvm_mmu *context)
3690 {
3691         paging64_init_context_common(vcpu, context, PT64_ROOT_LEVEL);
3692 }
3693 
3694 static void paging32_init_context(struct kvm_vcpu *vcpu,
3695                                   struct kvm_mmu *context)
3696 {
3697         context->nx = false;
3698         context->root_level = PT32_ROOT_LEVEL;
3699 
3700         reset_rsvds_bits_mask(vcpu, context);
3701         update_permission_bitmask(vcpu, context, false);
3702         update_last_pte_bitmap(vcpu, context);
3703 
3704         context->page_fault = paging32_page_fault;
3705         context->gva_to_gpa = paging32_gva_to_gpa;
3706         context->sync_page = paging32_sync_page;
3707         context->invlpg = paging32_invlpg;
3708         context->update_pte = paging32_update_pte;
3709         context->shadow_root_level = PT32E_ROOT_LEVEL;
3710         context->root_hpa = INVALID_PAGE;
3711         context->direct_map = false;
3712 }
3713 
3714 static void paging32E_init_context(struct kvm_vcpu *vcpu,
3715                                    struct kvm_mmu *context)
3716 {
3717         paging64_init_context_common(vcpu, context, PT32E_ROOT_LEVEL);
3718 }
3719 
3720 static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
3721 {
3722         struct kvm_mmu *context = vcpu->arch.walk_mmu;
3723 
3724         context->base_role.word = 0;
3725         context->page_fault = tdp_page_fault;
3726         context->sync_page = nonpaging_sync_page;
3727         context->invlpg = nonpaging_invlpg;
3728         context->update_pte = nonpaging_update_pte;
3729         context->shadow_root_level = kvm_x86_ops->get_tdp_level();
3730         context->root_hpa = INVALID_PAGE;
3731         context->direct_map = true;
3732         context->set_cr3 = kvm_x86_ops->set_tdp_cr3;
3733         context->get_cr3 = get_cr3;
3734         context->get_pdptr = kvm_pdptr_read;
3735         context->inject_page_fault = kvm_inject_page_fault;
3736 
3737         if (!is_paging(vcpu)) {
3738                 context->nx = false;
3739                 context->gva_to_gpa = nonpaging_gva_to_gpa;
3740                 context->root_level = 0;
3741         } else if (is_long_mode(vcpu)) {
3742                 context->nx = is_nx(vcpu);
3743                 context->root_level = PT64_ROOT_LEVEL;
3744                 reset_rsvds_bits_mask(vcpu, context);
3745                 context->gva_to_gpa = paging64_gva_to_gpa;
3746         } else if (is_pae(vcpu)) {
3747                 context->nx = is_nx(vcpu);
3748                 context->root_level = PT32E_ROOT_LEVEL;
3749                 reset_rsvds_bits_mask(vcpu, context);
3750                 context->gva_to_gpa = paging64_gva_to_gpa;
3751         } else {
3752                 context->nx = false;
3753                 context->root_level = PT32_ROOT_LEVEL;
3754                 reset_rsvds_bits_mask(vcpu, context);
3755                 context->gva_to_gpa = paging32_gva_to_gpa;
3756         }
3757 
3758         update_permission_bitmask(vcpu, context, false);
3759         update_last_pte_bitmap(vcpu, context);
3760 }
3761 
3762 void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *context)
3763 {
3764         bool smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
3765         bool smap = kvm_read_cr4_bits(vcpu, X86_CR4_SMAP);
3766         ASSERT(vcpu);
3767         ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
3768 
3769         if (!is_paging(vcpu))
3770                 nonpaging_init_context(vcpu, context);
3771         else if (is_long_mode(vcpu))
3772                 paging64_init_context(vcpu, context);
3773         else if (is_pae(vcpu))
3774                 paging32E_init_context(vcpu, context);
3775         else
3776                 paging32_init_context(vcpu, context);
3777 
3778         vcpu->arch.mmu.base_role.nxe = is_nx(vcpu);
3779         vcpu->arch.mmu.base_role.cr4_pae = !!is_pae(vcpu);
3780         vcpu->arch.mmu.base_role.cr0_wp  = is_write_protection(vcpu);
3781         vcpu->arch.mmu.base_role.smep_andnot_wp
3782                 = smep && !is_write_protection(vcpu);
3783         context->base_role.smap_andnot_wp
3784                 = smap && !is_write_protection(vcpu);
3785 }
3786 EXPORT_SYMBOL_GPL(kvm_init_shadow_mmu);
3787 
3788 void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *context,
3789                 bool execonly)
3790 {
3791         ASSERT(vcpu);
3792         ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
3793 
3794         context->shadow_root_level = kvm_x86_ops->get_tdp_level();
3795 
3796         context->nx = true;
3797         context->page_fault = ept_page_fault;
3798         context->gva_to_gpa = ept_gva_to_gpa;
3799         context->sync_page = ept_sync_page;
3800         context->invlpg = ept_invlpg;
3801         context->update_pte = ept_update_pte;
3802         context->root_level = context->shadow_root_level;
3803         context->root_hpa = INVALID_PAGE;
3804         context->direct_map = false;
3805 
3806         update_permission_bitmask(vcpu, context, true);
3807         reset_rsvds_bits_mask_ept(vcpu, context, execonly);
3808 }
3809 EXPORT_SYMBOL_GPL(kvm_init_shadow_ept_mmu);
3810 
3811 static void init_kvm_softmmu(struct kvm_vcpu *vcpu)
3812 {
3813         kvm_init_shadow_mmu(vcpu, vcpu->arch.walk_mmu);
3814         vcpu->arch.walk_mmu->set_cr3           = kvm_x86_ops->set_cr3;
3815         vcpu->arch.walk_mmu->get_cr3           = get_cr3;
3816         vcpu->arch.walk_mmu->get_pdptr         = kvm_pdptr_read;
3817         vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault;
3818 }
3819 
3820 static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
3821 {
3822         struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;
3823 
3824         g_context->get_cr3           = get_cr3;
3825         g_context->get_pdptr         = kvm_pdptr_read;
3826         g_context->inject_page_fault = kvm_inject_page_fault;
3827 
3828         /*
3829          * Note that arch.mmu.gva_to_gpa translates l2_gva to l1_gpa. The
3830          * translation of l2_gpa to l1_gpa addresses is done using the
3831          * arch.nested_mmu.gva_to_gpa function. Basically the gva_to_gpa
3832          * functions between mmu and nested_mmu are swapped.
3833          */
3834         if (!is_paging(vcpu)) {
3835                 g_context->nx = false;
3836                 g_context->root_level = 0;
3837                 g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested;
3838         } else if (is_long_mode(vcpu)) {
3839                 g_context->nx = is_nx(vcpu);
3840                 g_context->root_level = PT64_ROOT_LEVEL;
3841                 reset_rsvds_bits_mask(vcpu, g_context);
3842                 g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
3843         } else if (is_pae(vcpu)) {
3844                 g_context->nx = is_nx(vcpu);
3845                 g_context->root_level = PT32E_ROOT_LEVEL;
3846                 reset_rsvds_bits_mask(vcpu, g_context);
3847                 g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
3848         } else {
3849                 g_context->nx = false;
3850                 g_context->root_level = PT32_ROOT_LEVEL;
3851                 reset_rsvds_bits_mask(vcpu, g_context);
3852                 g_context->gva_to_gpa = paging32_gva_to_gpa_nested;
3853         }
3854 
3855         update_permission_bitmask(vcpu, g_context, false);
3856         update_last_pte_bitmap(vcpu, g_context);
3857 }
3858 
3859 static void init_kvm_mmu(struct kvm_vcpu *vcpu)
3860 {
3861         if (mmu_is_nested(vcpu))
3862                 return init_kvm_nested_mmu(vcpu);
3863         else if (tdp_enabled)
3864                 return init_kvm_tdp_mmu(vcpu);
3865         else
3866                 return init_kvm_softmmu(vcpu);
3867 }
3868 
3869 void kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
3870 {
3871         ASSERT(vcpu);
3872 
3873         kvm_mmu_unload(vcpu);
3874         init_kvm_mmu(vcpu);
3875 }
3876 EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
3877 
3878 int kvm_mmu_load(struct kvm_vcpu *vcpu)
3879 {
3880         int r;
3881 
3882         r = mmu_topup_memory_caches(vcpu);
3883         if (r)
3884                 goto out;
3885         r = mmu_alloc_roots(vcpu);
3886         kvm_mmu_sync_roots(vcpu);
3887         if (r)
3888                 goto out;
3889         /* set_cr3() should ensure TLB has been flushed */
3890         vcpu->arch.mmu.set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
3891 out:
3892         return r;
3893 }
3894 EXPORT_SYMBOL_GPL(kvm_mmu_load);
3895 
3896 void kvm_mmu_unload(struct kvm_vcpu *vcpu)
3897 {
3898         mmu_free_roots(vcpu);
3899         WARN_ON(VALID_PAGE(vcpu->arch.mmu.root_hpa));
3900 }
3901 EXPORT_SYMBOL_GPL(kvm_mmu_unload);
3902 
3903 static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
3904                                   struct kvm_mmu_page *sp, u64 *spte,
3905                                   const void *new)
3906 {
3907         if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
3908                 ++vcpu->kvm->stat.mmu_pde_zapped;
3909                 return;
3910         }
3911 
3912         ++vcpu->kvm->stat.mmu_pte_updated;
3913         vcpu->arch.mmu.update_pte(vcpu, sp, spte, new);
3914 }
3915 
3916 static bool need_remote_flush(u64 old, u64 new)
3917 {
3918         if (!is_shadow_present_pte(old))
3919                 return false;
3920         if (!is_shadow_present_pte(new))
3921                 return true;
3922         if ((old ^ new) & PT64_BASE_ADDR_MASK)
3923                 return true;
3924         old ^= shadow_nx_mask;
3925         new ^= shadow_nx_mask;
3926         return (old & ~new & PT64_PERM_MASK) != 0;
3927 }
3928 
3929 static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, bool zap_page,
3930                                     bool remote_flush, bool local_flush)
3931 {
3932         if (zap_page)
3933                 return;
3934 
3935         if (remote_flush)
3936                 kvm_flush_remote_tlbs(vcpu->kvm);
3937         else if (local_flush)
3938                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3939 }
3940 
3941 static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa,
3942                                     const u8 *new, int *bytes)
3943 {
3944         u64 gentry;
3945         int r;
3946 
3947         /*
3948          * Assume that the pte write on a page table of the same type
3949          * as the current vcpu paging mode since we update the sptes only
3950          * when they have the same mode.
3951          */
3952         if (is_pae(vcpu) && *bytes == 4) {
3953                 /* Handle a 32-bit guest writing two halves of a 64-bit gpte */
3954                 *gpa &= ~(gpa_t)7;
3955                 *bytes = 8;
3956                 r = kvm_read_guest(vcpu->kvm, *gpa, &gentry, 8);
3957                 if (r)
3958                         gentry = 0;
3959                 new = (const u8 *)&gentry;
3960         }
3961 
3962         switch (*bytes) {
3963         case 4:
3964                 gentry = *(const u32 *)new;
3965                 break;
3966         case 8:
3967                 gentry = *(const u64 *)new;
3968                 break;
3969         default:
3970                 gentry = 0;
3971                 break;
3972         }
3973 
3974         return gentry;
3975 }
3976 
3977 /*
3978  * If we're seeing too many writes to a page, it may no longer be a page table,
3979  * or we may be forking, in which case it is better to unmap the page.
3980  */
3981 static bool detect_write_flooding(struct kvm_mmu_page *sp)
3982 {
3983         /*
3984          * Skip write-flooding detected for the sp whose level is 1, because
3985          * it can become unsync, then the guest page is not write-protected.
3986          */
3987         if (sp->role.level == PT_PAGE_TABLE_LEVEL)
3988                 return false;
3989 
3990         return ++sp->write_flooding_count >= 3;
3991 }
3992 
3993 /*
3994  * Misaligned accesses are too much trouble to fix up; also, they usually
3995  * indicate a page is not used as a page table.
3996  */
3997 static bool detect_write_misaligned(struct kvm_mmu_page *sp, gpa_t gpa,
3998                                     int bytes)
3999 {
4000         unsigned offset, pte_size, misaligned;
4001 
4002         pgprintk("misaligned: gpa %llx bytes %d role %x\n",
4003                  gpa, bytes, sp->role.word);
4004 
4005         offset = offset_in_page(gpa);
4006         pte_size = sp->role.cr4_pae ? 8 : 4;
4007 
4008         /*
4009          * Sometimes, the OS only writes the last one bytes to update status
4010          * bits, for example, in linux, andb instruction is used in clear_bit().
4011          */
4012         if (!(offset & (pte_size - 1)) && bytes == 1)
4013                 return false;
4014 
4015         misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
4016         misaligned |= bytes < 4;
4017 
4018         return misaligned;
4019 }
4020 
4021 static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte)
4022 {
4023         unsigned page_offset, quadrant;
4024         u64 *spte;
4025         int level;
4026 
4027         page_offset = offset_in_page(gpa);
4028         level = sp->role.level;
4029         *nspte = 1;
4030         if (!sp->role.cr4_pae) {
4031                 page_offset <<= 1;      /* 32->64 */
4032                 /*
4033                  * A 32-bit pde maps 4MB while the shadow pdes map
4034                  * only 2MB.  So we need to double the offset again
4035                  * and zap two pdes instead of one.
4036                  */
4037                 if (level == PT32_ROOT_LEVEL) {
4038                         page_offset &= ~7; /* kill rounding error */
4039                         page_offset <<= 1;
4040                         *nspte = 2;
4041                 }
4042                 quadrant = page_offset >> PAGE_SHIFT;
4043                 page_offset &= ~PAGE_MASK;
4044                 if (quadrant != sp->role.quadrant)
4045                         return NULL;
4046         }
4047 
4048         spte = &sp->spt[page_offset / sizeof(*spte)];
4049         return spte;
4050 }
4051 
4052 void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
4053                        const u8 *new, int bytes)
4054 {
4055         gfn_t gfn = gpa >> PAGE_SHIFT;
4056         struct kvm_mmu_page *sp;
4057         LIST_HEAD(invalid_list);
4058         u64 entry, gentry, *spte;
4059         int npte;
4060         bool remote_flush, local_flush, zap_page;
4061         union kvm_mmu_page_role mask = { };
4062 
4063         mask.cr0_wp = 1;
4064         mask.cr4_pae = 1;
4065         mask.nxe = 1;
4066         mask.smep_andnot_wp = 1;
4067         mask.smap_andnot_wp = 1;
4068 
4069         /*
4070          * If we don't have indirect shadow pages, it means no page is
4071          * write-protected, so we can exit simply.
4072          */
4073         if (!ACCESS_ONCE(vcpu->kvm->arch.indirect_shadow_pages))
4074                 return;
4075 
4076         zap_page = remote_flush = local_flush = false;
4077 
4078         pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
4079 
4080         gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, new, &bytes);
4081 
4082         /*
4083          * No need to care whether allocation memory is successful
4084          * or not since pte prefetch is skiped if it does not have
4085          * enough objects in the cache.
4086          */
4087         mmu_topup_memory_caches(vcpu);
4088 
4089         spin_lock(&vcpu->kvm->mmu_lock);
4090         ++vcpu->kvm->stat.mmu_pte_write;
4091         kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE);
4092 
4093         for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) {
4094                 if (detect_write_misaligned(sp, gpa, bytes) ||
4095                       detect_write_flooding(sp)) {
4096                         zap_page |= !!kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
4097                                                      &invalid_list);
4098                         ++vcpu->kvm->stat.mmu_flooded;
4099                         continue;
4100                 }
4101 
4102                 spte = get_written_sptes(sp, gpa, &npte);
4103                 if (!spte)
4104                         continue;
4105 
4106                 local_flush = true;
4107                 while (npte--) {
4108                         entry = *spte;
4109                         mmu_page_zap_pte(vcpu->kvm, sp, spte);
4110                         if (gentry &&
4111                               !((sp->role.word ^ vcpu->arch.mmu.base_role.word)
4112                               & mask.word) && rmap_can_add(vcpu))
4113                                 mmu_pte_write_new_pte(vcpu, sp, spte, &gentry);
4114                         if (need_remote_flush(entry, *spte))
4115                                 remote_flush = true;
4116                         ++spte;
4117                 }
4118         }
4119         mmu_pte_write_flush_tlb(vcpu, zap_page, remote_flush, local_flush);
4120         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
4121         kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE);
4122         spin_unlock(&vcpu->kvm->mmu_lock);
4123 }
4124 
4125 int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
4126 {
4127         gpa_t gpa;
4128         int r;
4129 
4130         if (vcpu->arch.mmu.direct_map)
4131                 return 0;
4132 
4133         gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);
4134 
4135         r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
4136 
4137         return r;
4138 }
4139 EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
4140 
4141 static void make_mmu_pages_available(struct kvm_vcpu *vcpu)
4142 {
4143         LIST_HEAD(invalid_list);
4144 
4145         if (likely(kvm_mmu_available_pages(vcpu->kvm) >= KVM_MIN_FREE_MMU_PAGES))
4146                 return;
4147 
4148         while (kvm_mmu_available_pages(vcpu->kvm) < KVM_REFILL_PAGES) {
4149                 if (!prepare_zap_oldest_mmu_page(vcpu->kvm, &invalid_list))
4150                         break;
4151 
4152                 ++vcpu->kvm->stat.mmu_recycled;
4153         }
4154         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
4155 }
4156 
4157 static bool is_mmio_page_fault(struct kvm_vcpu *vcpu, gva_t addr)
4158 {
4159         if (vcpu->arch.mmu.direct_map || mmu_is_nested(vcpu))
4160                 return vcpu_match_mmio_gpa(vcpu, addr);
4161 
4162         return vcpu_match_mmio_gva(vcpu, addr);
4163 }
4164 
4165 int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code,
4166                        void *insn, int insn_len)
4167 {
4168         int r, emulation_type = EMULTYPE_RETRY;
4169         enum emulation_result er;
4170 
4171         r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code, false);
4172         if (r < 0)
4173                 goto out;
4174 
4175         if (!r) {
4176                 r = 1;
4177                 goto out;
4178         }
4179 
4180         if (is_mmio_page_fault(vcpu, cr2))
4181                 emulation_type = 0;
4182 
4183         er = x86_emulate_instruction(vcpu, cr2, emulation_type, insn, insn_len);
4184 
4185         switch (er) {
4186         case EMULATE_DONE:
4187                 return 1;
4188         case EMULATE_USER_EXIT:
4189                 ++vcpu->stat.mmio_exits;
4190                 /* fall through */
4191         case EMULATE_FAIL:
4192                 return 0;
4193         default:
4194                 BUG();
4195         }
4196 out:
4197         return r;
4198 }
4199 EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
4200 
4201 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
4202 {
4203         vcpu->arch.mmu.invlpg(vcpu, gva);
4204         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
4205         ++vcpu->stat.invlpg;
4206 }
4207 EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
4208 
4209 void kvm_enable_tdp(void)
4210 {
4211         tdp_enabled = true;
4212 }
4213 EXPORT_SYMBOL_GPL(kvm_enable_tdp);
4214 
4215 void kvm_disable_tdp(void)
4216 {
4217         tdp_enabled = false;
4218 }
4219 EXPORT_SYMBOL_GPL(kvm_disable_tdp);
4220 
4221 static void free_mmu_pages(struct kvm_vcpu *vcpu)
4222 {
4223         free_page((unsigned long)vcpu->arch.mmu.pae_root);
4224         if (vcpu->arch.mmu.lm_root != NULL)
4225                 free_page((unsigned long)vcpu->arch.mmu.lm_root);
4226 }
4227 
4228 static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
4229 {
4230         struct page *page;
4231         int i;
4232 
4233         ASSERT(vcpu);
4234 
4235         /*
4236          * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
4237          * Therefore we need to allocate shadow page tables in the first
4238          * 4GB of memory, which happens to fit the DMA32 zone.
4239          */
4240         page = alloc_page(GFP_KERNEL | __GFP_DMA32);
4241         if (!page)
4242                 return -ENOMEM;
4243 
4244         vcpu->arch.mmu.pae_root = page_address(page);
4245         for (i = 0; i < 4; ++i)
4246                 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
4247 
4248         return 0;
4249 }
4250 
4251 int kvm_mmu_create(struct kvm_vcpu *vcpu)
4252 {
4253         ASSERT(vcpu);
4254 
4255         vcpu->arch.walk_mmu = &vcpu->arch.mmu;
4256         vcpu->arch.mmu.root_hpa = INVALID_PAGE;
4257         vcpu->arch.mmu.translate_gpa = translate_gpa;
4258         vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;
4259 
4260         return alloc_mmu_pages(vcpu);
4261 }
4262 
4263 void kvm_mmu_setup(struct kvm_vcpu *vcpu)
4264 {
4265         ASSERT(vcpu);
4266         ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
4267 
4268         init_kvm_mmu(vcpu);
4269 }
4270 
4271 void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
4272 {
4273         struct kvm_memory_slot *memslot;
4274         gfn_t last_gfn;
4275         int i;
4276 
4277         memslot = id_to_memslot(kvm->memslots, slot);
4278         last_gfn = memslot->base_gfn + memslot->npages - 1;
4279 
4280         spin_lock(&kvm->mmu_lock);
4281 
4282         for (i = PT_PAGE_TABLE_LEVEL;
4283              i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
4284                 unsigned long *rmapp;
4285                 unsigned long last_index, index;
4286 
4287                 rmapp = memslot->arch.rmap[i - PT_PAGE_TABLE_LEVEL];
4288                 last_index = gfn_to_index(last_gfn, memslot->base_gfn, i);
4289 
4290                 for (index = 0; index <= last_index; ++index, ++rmapp) {
4291                         if (*rmapp)
4292                                 __rmap_write_protect(kvm, rmapp, false);
4293 
4294                         if (need_resched() || spin_needbreak(&kvm->mmu_lock))
4295                                 cond_resched_lock(&kvm->mmu_lock);
4296                 }
4297         }
4298 
4299         spin_unlock(&kvm->mmu_lock);
4300 
4301         /*
4302          * kvm_mmu_slot_remove_write_access() and kvm_vm_ioctl_get_dirty_log()
4303          * which do tlb flush out of mmu-lock should be serialized by
4304          * kvm->slots_lock otherwise tlb flush would be missed.
4305          */
4306         lockdep_assert_held(&kvm->slots_lock);
4307 
4308         /*
4309          * We can flush all the TLBs out of the mmu lock without TLB
4310          * corruption since we just change the spte from writable to
4311          * readonly so that we only need to care the case of changing
4312          * spte from present to present (changing the spte from present
4313          * to nonpresent will flush all the TLBs immediately), in other
4314          * words, the only case we care is mmu_spte_update() where we
4315          * haved checked SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE
4316          * instead of PT_WRITABLE_MASK, that means it does not depend
4317          * on PT_WRITABLE_MASK anymore.
4318          */
4319         kvm_flush_remote_tlbs(kvm);
4320 }
4321 
4322 #define BATCH_ZAP_PAGES 10
4323 static void kvm_zap_obsolete_pages(struct kvm *kvm)
4324 {
4325         struct kvm_mmu_page *sp, *node;
4326         int batch = 0;
4327 
4328 restart:
4329         list_for_each_entry_safe_reverse(sp, node,
4330               &kvm->arch.active_mmu_pages, link) {
4331                 int ret;
4332 
4333                 /*
4334                  * No obsolete page exists before new created page since
4335                  * active_mmu_pages is the FIFO list.
4336                  */
4337                 if (!is_obsolete_sp(kvm, sp))
4338                         break;
4339 
4340                 /*
4341                  * Since we are reversely walking the list and the invalid
4342                  * list will be moved to the head, skip the invalid page
4343                  * can help us to avoid the infinity list walking.
4344                  */
4345                 if (sp->role.invalid)
4346                         continue;
4347 
4348                 /*
4349                  * Need not flush tlb since we only zap the sp with invalid
4350                  * generation number.
4351                  */
4352                 if (batch >= BATCH_ZAP_PAGES &&
4353                       cond_resched_lock(&kvm->mmu_lock)) {
4354                         batch = 0;
4355                         goto restart;
4356                 }
4357 
4358                 ret = kvm_mmu_prepare_zap_page(kvm, sp,
4359                                 &kvm->arch.zapped_obsolete_pages);
4360                 batch += ret;
4361 
4362                 if (ret)
4363                         goto restart;
4364         }
4365 
4366         /*
4367          * Should flush tlb before free page tables since lockless-walking
4368          * may use the pages.
4369          */
4370         kvm_mmu_commit_zap_page(kvm, &kvm->arch.zapped_obsolete_pages);
4371 }
4372 
4373 /*
4374  * Fast invalidate all shadow pages and use lock-break technique
4375  * to zap obsolete pages.
4376  *
4377  * It's required when memslot is being deleted or VM is being
4378  * destroyed, in these cases, we should ensure that KVM MMU does
4379  * not use any resource of the being-deleted slot or all slots
4380  * after calling the function.
4381  */
4382 void kvm_mmu_invalidate_zap_all_pages(struct kvm *kvm)
4383 {
4384         spin_lock(&kvm->mmu_lock);
4385         trace_kvm_mmu_invalidate_zap_all_pages(kvm);
4386         kvm->arch.mmu_valid_gen++;
4387 
4388         /*
4389          * Notify all vcpus to reload its shadow page table
4390          * and flush TLB. Then all vcpus will switch to new
4391          * shadow page table with the new mmu_valid_gen.
4392          *
4393          * Note: we should do this under the protection of
4394          * mmu-lock, otherwise, vcpu would purge shadow page
4395          * but miss tlb flush.
4396          */
4397         kvm_reload_remote_mmus(kvm);
4398 
4399         kvm_zap_obsolete_pages(kvm);
4400         spin_unlock(&kvm->mmu_lock);
4401 }
4402 
4403 static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm)
4404 {
4405         return unlikely(!list_empty_careful(&kvm->arch.zapped_obsolete_pages));
4406 }
4407 
4408 void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm)
4409 {
4410         /*
4411          * The very rare case: if the generation-number is round,
4412          * zap all shadow pages.
4413          */
4414         if (unlikely(kvm_current_mmio_generation(kvm) == 0)) {
4415                 printk_ratelimited(KERN_DEBUG "kvm: zapping shadow pages for mmio generation wraparound\n");
4416                 kvm_mmu_invalidate_zap_all_pages(kvm);
4417         }
4418 }
4419 
4420 static unsigned long
4421 mmu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
4422 {
4423         struct kvm *kvm;
4424         int nr_to_scan = sc->nr_to_scan;
4425         unsigned long freed = 0;
4426 
4427         spin_lock(&kvm_lock);
4428 
4429         list_for_each_entry(kvm, &vm_list, vm_list) {
4430                 int idx;
4431                 LIST_HEAD(invalid_list);
4432 
4433                 /*
4434                  * Never scan more than sc->nr_to_scan VM instances.
4435                  * Will not hit this condition practically since we do not try
4436                  * to shrink more than one VM and it is very unlikely to see
4437                  * !n_used_mmu_pages so many times.
4438                  */
4439                 if (!nr_to_scan--)
4440                         break;
4441                 /*
4442                  * n_used_mmu_pages is accessed without holding kvm->mmu_lock
4443                  * here. We may skip a VM instance errorneosly, but we do not
4444                  * want to shrink a VM that only started to populate its MMU
4445                  * anyway.
4446                  */
4447                 if (!kvm->arch.n_used_mmu_pages &&
4448                       !kvm_has_zapped_obsolete_pages(kvm))
4449                         continue;
4450 
4451                 idx = srcu_read_lock(&kvm->srcu);
4452                 spin_lock(&kvm->mmu_lock);
4453 
4454                 if (kvm_has_zapped_obsolete_pages(kvm)) {
4455                         kvm_mmu_commit_zap_page(kvm,
4456                               &kvm->arch.zapped_obsolete_pages);
4457                         goto unlock;
4458                 }
4459 
4460                 if (prepare_zap_oldest_mmu_page(kvm, &invalid_list))
4461                         freed++;
4462                 kvm_mmu_commit_zap_page(kvm, &invalid_list);
4463 
4464 unlock:
4465                 spin_unlock(&kvm->mmu_lock);
4466                 srcu_read_unlock(&kvm->srcu, idx);
4467 
4468                 /*
4469                  * unfair on small ones
4470                  * per-vm shrinkers cry out
4471                  * sadness comes quickly
4472                  */
4473                 list_move_tail(&kvm->vm_list, &vm_list);
4474                 break;
4475         }
4476 
4477         spin_unlock(&kvm_lock);
4478         return freed;
4479 }
4480 
4481 static unsigned long
4482 mmu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
4483 {
4484         return percpu_counter_read_positive(&kvm_total_used_mmu_pages);
4485 }
4486 
4487 static struct shrinker mmu_shrinker = {
4488         .count_objects = mmu_shrink_count,
4489         .scan_objects = mmu_shrink_scan,
4490         .seeks = DEFAULT_SEEKS * 10,
4491 };
4492 
4493 static void mmu_destroy_caches(void)
4494 {
4495         if (pte_list_desc_cache)
4496                 kmem_cache_destroy(pte_list_desc_cache);
4497         if (mmu_page_header_cache)
4498                 kmem_cache_destroy(mmu_page_header_cache);
4499 }
4500 
4501 int kvm_mmu_module_init(void)
4502 {
4503         pte_list_desc_cache = kmem_cache_create("pte_list_desc",
4504                                             sizeof(struct pte_list_desc),
4505                                             0, 0, NULL);
4506         if (!pte_list_desc_cache)
4507                 goto nomem;
4508 
4509         mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
4510                                                   sizeof(struct kvm_mmu_page),
4511                                                   0, 0, NULL);
4512         if (!mmu_page_header_cache)
4513                 goto nomem;
4514 
4515         if (percpu_counter_init(&kvm_total_used_mmu_pages, 0, GFP_KERNEL))
4516                 goto nomem;
4517 
4518         register_shrinker(&mmu_shrinker);
4519 
4520         return 0;
4521 
4522 nomem:
4523         mmu_destroy_caches();
4524         return -ENOMEM;
4525 }
4526 
4527 /*
4528  * Caculate mmu pages needed for kvm.
4529  */
4530 unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
4531 {
4532         unsigned int nr_mmu_pages;
4533         unsigned int  nr_pages = 0;
4534         struct kvm_memslots *slots;
4535         struct kvm_memory_slot *memslot;
4536 
4537         slots = kvm_memslots(kvm);
4538 
4539         kvm_for_each_memslot(memslot, slots)
4540                 nr_pages += memslot->npages;
4541 
4542         nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
4543         nr_mmu_pages = max(nr_mmu_pages,
4544                         (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
4545 
4546         return nr_mmu_pages;
4547 }
4548 
4549 int kvm_mmu_get_spte_hierarchy(struct kvm_vcpu *vcpu, u64 addr, u64 sptes[4])
4550 {
4551         struct kvm_shadow_walk_iterator iterator;
4552         u64 spte;
4553         int nr_sptes = 0;
4554 
4555         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
4556                 return nr_sptes;
4557 
4558         walk_shadow_page_lockless_begin(vcpu);
4559         for_each_shadow_entry_lockless(vcpu, addr, iterator, spte) {
4560                 sptes[iterator.level-1] = spte;
4561                 nr_sptes++;
4562                 if (!is_shadow_present_pte(spte))
4563                         break;
4564         }
4565         walk_shadow_page_lockless_end(vcpu);
4566 
4567         return nr_sptes;
4568 }
4569 EXPORT_SYMBOL_GPL(kvm_mmu_get_spte_hierarchy);
4570 
4571 void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
4572 {
4573         ASSERT(vcpu);
4574 
4575         kvm_mmu_unload(vcpu);
4576         free_mmu_pages(vcpu);
4577         mmu_free_memory_caches(vcpu);
4578 }
4579 
4580 void kvm_mmu_module_exit(void)
4581 {
4582         mmu_destroy_caches();
4583         percpu_counter_destroy(&kvm_total_used_mmu_pages);
4584         unregister_shrinker(&mmu_shrinker);
4585         mmu_audit_disable();
4586 }
4587 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp