~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/kvm/mmu.c

Version: ~ [ linux-5.4-rc3 ] ~ [ linux-5.3.6 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.79 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.149 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.196 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.196 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.75 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Kernel-based Virtual Machine driver for Linux
  3  *
  4  * This module enables machines with Intel VT-x extensions to run virtual
  5  * machines without emulation or binary translation.
  6  *
  7  * MMU support
  8  *
  9  * Copyright (C) 2006 Qumranet, Inc.
 10  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
 11  *
 12  * Authors:
 13  *   Yaniv Kamay  <yaniv@qumranet.com>
 14  *   Avi Kivity   <avi@qumranet.com>
 15  *
 16  * This work is licensed under the terms of the GNU GPL, version 2.  See
 17  * the COPYING file in the top-level directory.
 18  *
 19  */
 20 
 21 #include "irq.h"
 22 #include "mmu.h"
 23 #include "x86.h"
 24 #include "kvm_cache_regs.h"
 25 #include "cpuid.h"
 26 
 27 #include <linux/kvm_host.h>
 28 #include <linux/types.h>
 29 #include <linux/string.h>
 30 #include <linux/mm.h>
 31 #include <linux/highmem.h>
 32 #include <linux/moduleparam.h>
 33 #include <linux/export.h>
 34 #include <linux/swap.h>
 35 #include <linux/hugetlb.h>
 36 #include <linux/compiler.h>
 37 #include <linux/srcu.h>
 38 #include <linux/slab.h>
 39 #include <linux/uaccess.h>
 40 
 41 #include <asm/page.h>
 42 #include <asm/cmpxchg.h>
 43 #include <asm/io.h>
 44 #include <asm/vmx.h>
 45 #include <asm/kvm_page_track.h>
 46 
 47 /*
 48  * When setting this variable to true it enables Two-Dimensional-Paging
 49  * where the hardware walks 2 page tables:
 50  * 1. the guest-virtual to guest-physical
 51  * 2. while doing 1. it walks guest-physical to host-physical
 52  * If the hardware supports that we don't need to do shadow paging.
 53  */
 54 bool tdp_enabled = false;
 55 
 56 enum {
 57         AUDIT_PRE_PAGE_FAULT,
 58         AUDIT_POST_PAGE_FAULT,
 59         AUDIT_PRE_PTE_WRITE,
 60         AUDIT_POST_PTE_WRITE,
 61         AUDIT_PRE_SYNC,
 62         AUDIT_POST_SYNC
 63 };
 64 
 65 #undef MMU_DEBUG
 66 
 67 #ifdef MMU_DEBUG
 68 static bool dbg = 0;
 69 module_param(dbg, bool, 0644);
 70 
 71 #define pgprintk(x...) do { if (dbg) printk(x); } while (0)
 72 #define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
 73 #define MMU_WARN_ON(x) WARN_ON(x)
 74 #else
 75 #define pgprintk(x...) do { } while (0)
 76 #define rmap_printk(x...) do { } while (0)
 77 #define MMU_WARN_ON(x) do { } while (0)
 78 #endif
 79 
 80 #define PTE_PREFETCH_NUM                8
 81 
 82 #define PT_FIRST_AVAIL_BITS_SHIFT 10
 83 #define PT64_SECOND_AVAIL_BITS_SHIFT 52
 84 
 85 #define PT64_LEVEL_BITS 9
 86 
 87 #define PT64_LEVEL_SHIFT(level) \
 88                 (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
 89 
 90 #define PT64_INDEX(address, level)\
 91         (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
 92 
 93 
 94 #define PT32_LEVEL_BITS 10
 95 
 96 #define PT32_LEVEL_SHIFT(level) \
 97                 (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
 98 
 99 #define PT32_LVL_OFFSET_MASK(level) \
100         (PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
101                                                 * PT32_LEVEL_BITS))) - 1))
102 
103 #define PT32_INDEX(address, level)\
104         (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
105 
106 
107 #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
108 #define PT64_DIR_BASE_ADDR_MASK \
109         (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
110 #define PT64_LVL_ADDR_MASK(level) \
111         (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
112                                                 * PT64_LEVEL_BITS))) - 1))
113 #define PT64_LVL_OFFSET_MASK(level) \
114         (PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
115                                                 * PT64_LEVEL_BITS))) - 1))
116 
117 #define PT32_BASE_ADDR_MASK PAGE_MASK
118 #define PT32_DIR_BASE_ADDR_MASK \
119         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
120 #define PT32_LVL_ADDR_MASK(level) \
121         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
122                                             * PT32_LEVEL_BITS))) - 1))
123 
124 #define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | shadow_user_mask \
125                         | shadow_x_mask | shadow_nx_mask)
126 
127 #define ACC_EXEC_MASK    1
128 #define ACC_WRITE_MASK   PT_WRITABLE_MASK
129 #define ACC_USER_MASK    PT_USER_MASK
130 #define ACC_ALL          (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
131 
132 #include <trace/events/kvm.h>
133 
134 #define CREATE_TRACE_POINTS
135 #include "mmutrace.h"
136 
137 #define SPTE_HOST_WRITEABLE     (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
138 #define SPTE_MMU_WRITEABLE      (1ULL << (PT_FIRST_AVAIL_BITS_SHIFT + 1))
139 
140 #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
141 
142 /* make pte_list_desc fit well in cache line */
143 #define PTE_LIST_EXT 3
144 
145 struct pte_list_desc {
146         u64 *sptes[PTE_LIST_EXT];
147         struct pte_list_desc *more;
148 };
149 
150 struct kvm_shadow_walk_iterator {
151         u64 addr;
152         hpa_t shadow_addr;
153         u64 *sptep;
154         int level;
155         unsigned index;
156 };
157 
158 #define for_each_shadow_entry(_vcpu, _addr, _walker)    \
159         for (shadow_walk_init(&(_walker), _vcpu, _addr);        \
160              shadow_walk_okay(&(_walker));                      \
161              shadow_walk_next(&(_walker)))
162 
163 #define for_each_shadow_entry_lockless(_vcpu, _addr, _walker, spte)     \
164         for (shadow_walk_init(&(_walker), _vcpu, _addr);                \
165              shadow_walk_okay(&(_walker)) &&                            \
166                 ({ spte = mmu_spte_get_lockless(_walker.sptep); 1; });  \
167              __shadow_walk_next(&(_walker), spte))
168 
169 static struct kmem_cache *pte_list_desc_cache;
170 static struct kmem_cache *mmu_page_header_cache;
171 static struct percpu_counter kvm_total_used_mmu_pages;
172 
173 static u64 __read_mostly shadow_nx_mask;
174 static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
175 static u64 __read_mostly shadow_user_mask;
176 static u64 __read_mostly shadow_accessed_mask;
177 static u64 __read_mostly shadow_dirty_mask;
178 static u64 __read_mostly shadow_mmio_mask;
179 static u64 __read_mostly shadow_present_mask;
180 
181 static void mmu_spte_set(u64 *sptep, u64 spte);
182 static void mmu_free_roots(struct kvm_vcpu *vcpu);
183 
184 void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask)
185 {
186         shadow_mmio_mask = mmio_mask;
187 }
188 EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_mask);
189 
190 /*
191  * the low bit of the generation number is always presumed to be zero.
192  * This disables mmio caching during memslot updates.  The concept is
193  * similar to a seqcount but instead of retrying the access we just punt
194  * and ignore the cache.
195  *
196  * spte bits 3-11 are used as bits 1-9 of the generation number,
197  * the bits 52-61 are used as bits 10-19 of the generation number.
198  */
199 #define MMIO_SPTE_GEN_LOW_SHIFT         2
200 #define MMIO_SPTE_GEN_HIGH_SHIFT        52
201 
202 #define MMIO_GEN_SHIFT                  20
203 #define MMIO_GEN_LOW_SHIFT              10
204 #define MMIO_GEN_LOW_MASK               ((1 << MMIO_GEN_LOW_SHIFT) - 2)
205 #define MMIO_GEN_MASK                   ((1 << MMIO_GEN_SHIFT) - 1)
206 
207 static u64 generation_mmio_spte_mask(unsigned int gen)
208 {
209         u64 mask;
210 
211         WARN_ON(gen & ~MMIO_GEN_MASK);
212 
213         mask = (gen & MMIO_GEN_LOW_MASK) << MMIO_SPTE_GEN_LOW_SHIFT;
214         mask |= ((u64)gen >> MMIO_GEN_LOW_SHIFT) << MMIO_SPTE_GEN_HIGH_SHIFT;
215         return mask;
216 }
217 
218 static unsigned int get_mmio_spte_generation(u64 spte)
219 {
220         unsigned int gen;
221 
222         spte &= ~shadow_mmio_mask;
223 
224         gen = (spte >> MMIO_SPTE_GEN_LOW_SHIFT) & MMIO_GEN_LOW_MASK;
225         gen |= (spte >> MMIO_SPTE_GEN_HIGH_SHIFT) << MMIO_GEN_LOW_SHIFT;
226         return gen;
227 }
228 
229 static unsigned int kvm_current_mmio_generation(struct kvm_vcpu *vcpu)
230 {
231         return kvm_vcpu_memslots(vcpu)->generation & MMIO_GEN_MASK;
232 }
233 
234 static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn,
235                            unsigned access)
236 {
237         unsigned int gen = kvm_current_mmio_generation(vcpu);
238         u64 mask = generation_mmio_spte_mask(gen);
239 
240         access &= ACC_WRITE_MASK | ACC_USER_MASK;
241         mask |= shadow_mmio_mask | access | gfn << PAGE_SHIFT;
242 
243         trace_mark_mmio_spte(sptep, gfn, access, gen);
244         mmu_spte_set(sptep, mask);
245 }
246 
247 static bool is_mmio_spte(u64 spte)
248 {
249         return (spte & shadow_mmio_mask) == shadow_mmio_mask;
250 }
251 
252 static gfn_t get_mmio_spte_gfn(u64 spte)
253 {
254         u64 mask = generation_mmio_spte_mask(MMIO_GEN_MASK) | shadow_mmio_mask;
255         return (spte & ~mask) >> PAGE_SHIFT;
256 }
257 
258 static unsigned get_mmio_spte_access(u64 spte)
259 {
260         u64 mask = generation_mmio_spte_mask(MMIO_GEN_MASK) | shadow_mmio_mask;
261         return (spte & ~mask) & ~PAGE_MASK;
262 }
263 
264 static bool set_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
265                           kvm_pfn_t pfn, unsigned access)
266 {
267         if (unlikely(is_noslot_pfn(pfn))) {
268                 mark_mmio_spte(vcpu, sptep, gfn, access);
269                 return true;
270         }
271 
272         return false;
273 }
274 
275 static bool check_mmio_spte(struct kvm_vcpu *vcpu, u64 spte)
276 {
277         unsigned int kvm_gen, spte_gen;
278 
279         kvm_gen = kvm_current_mmio_generation(vcpu);
280         spte_gen = get_mmio_spte_generation(spte);
281 
282         trace_check_mmio_spte(spte, kvm_gen, spte_gen);
283         return likely(kvm_gen == spte_gen);
284 }
285 
286 void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
287                 u64 dirty_mask, u64 nx_mask, u64 x_mask, u64 p_mask)
288 {
289         shadow_user_mask = user_mask;
290         shadow_accessed_mask = accessed_mask;
291         shadow_dirty_mask = dirty_mask;
292         shadow_nx_mask = nx_mask;
293         shadow_x_mask = x_mask;
294         shadow_present_mask = p_mask;
295 }
296 EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);
297 
298 static int is_cpuid_PSE36(void)
299 {
300         return 1;
301 }
302 
303 static int is_nx(struct kvm_vcpu *vcpu)
304 {
305         return vcpu->arch.efer & EFER_NX;
306 }
307 
308 static int is_shadow_present_pte(u64 pte)
309 {
310         return (pte & 0xFFFFFFFFull) && !is_mmio_spte(pte);
311 }
312 
313 static int is_large_pte(u64 pte)
314 {
315         return pte & PT_PAGE_SIZE_MASK;
316 }
317 
318 static int is_last_spte(u64 pte, int level)
319 {
320         if (level == PT_PAGE_TABLE_LEVEL)
321                 return 1;
322         if (is_large_pte(pte))
323                 return 1;
324         return 0;
325 }
326 
327 static kvm_pfn_t spte_to_pfn(u64 pte)
328 {
329         return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
330 }
331 
332 static gfn_t pse36_gfn_delta(u32 gpte)
333 {
334         int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;
335 
336         return (gpte & PT32_DIR_PSE36_MASK) << shift;
337 }
338 
339 #ifdef CONFIG_X86_64
340 static void __set_spte(u64 *sptep, u64 spte)
341 {
342         WRITE_ONCE(*sptep, spte);
343 }
344 
345 static void __update_clear_spte_fast(u64 *sptep, u64 spte)
346 {
347         WRITE_ONCE(*sptep, spte);
348 }
349 
350 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
351 {
352         return xchg(sptep, spte);
353 }
354 
355 static u64 __get_spte_lockless(u64 *sptep)
356 {
357         return ACCESS_ONCE(*sptep);
358 }
359 #else
360 union split_spte {
361         struct {
362                 u32 spte_low;
363                 u32 spte_high;
364         };
365         u64 spte;
366 };
367 
368 static void count_spte_clear(u64 *sptep, u64 spte)
369 {
370         struct kvm_mmu_page *sp =  page_header(__pa(sptep));
371 
372         if (is_shadow_present_pte(spte))
373                 return;
374 
375         /* Ensure the spte is completely set before we increase the count */
376         smp_wmb();
377         sp->clear_spte_count++;
378 }
379 
380 static void __set_spte(u64 *sptep, u64 spte)
381 {
382         union split_spte *ssptep, sspte;
383 
384         ssptep = (union split_spte *)sptep;
385         sspte = (union split_spte)spte;
386 
387         ssptep->spte_high = sspte.spte_high;
388 
389         /*
390          * If we map the spte from nonpresent to present, We should store
391          * the high bits firstly, then set present bit, so cpu can not
392          * fetch this spte while we are setting the spte.
393          */
394         smp_wmb();
395 
396         WRITE_ONCE(ssptep->spte_low, sspte.spte_low);
397 }
398 
399 static void __update_clear_spte_fast(u64 *sptep, u64 spte)
400 {
401         union split_spte *ssptep, sspte;
402 
403         ssptep = (union split_spte *)sptep;
404         sspte = (union split_spte)spte;
405 
406         WRITE_ONCE(ssptep->spte_low, sspte.spte_low);
407 
408         /*
409          * If we map the spte from present to nonpresent, we should clear
410          * present bit firstly to avoid vcpu fetch the old high bits.
411          */
412         smp_wmb();
413 
414         ssptep->spte_high = sspte.spte_high;
415         count_spte_clear(sptep, spte);
416 }
417 
418 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
419 {
420         union split_spte *ssptep, sspte, orig;
421 
422         ssptep = (union split_spte *)sptep;
423         sspte = (union split_spte)spte;
424 
425         /* xchg acts as a barrier before the setting of the high bits */
426         orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low);
427         orig.spte_high = ssptep->spte_high;
428         ssptep->spte_high = sspte.spte_high;
429         count_spte_clear(sptep, spte);
430 
431         return orig.spte;
432 }
433 
434 /*
435  * The idea using the light way get the spte on x86_32 guest is from
436  * gup_get_pte(arch/x86/mm/gup.c).
437  *
438  * An spte tlb flush may be pending, because kvm_set_pte_rmapp
439  * coalesces them and we are running out of the MMU lock.  Therefore
440  * we need to protect against in-progress updates of the spte.
441  *
442  * Reading the spte while an update is in progress may get the old value
443  * for the high part of the spte.  The race is fine for a present->non-present
444  * change (because the high part of the spte is ignored for non-present spte),
445  * but for a present->present change we must reread the spte.
446  *
447  * All such changes are done in two steps (present->non-present and
448  * non-present->present), hence it is enough to count the number of
449  * present->non-present updates: if it changed while reading the spte,
450  * we might have hit the race.  This is done using clear_spte_count.
451  */
452 static u64 __get_spte_lockless(u64 *sptep)
453 {
454         struct kvm_mmu_page *sp =  page_header(__pa(sptep));
455         union split_spte spte, *orig = (union split_spte *)sptep;
456         int count;
457 
458 retry:
459         count = sp->clear_spte_count;
460         smp_rmb();
461 
462         spte.spte_low = orig->spte_low;
463         smp_rmb();
464 
465         spte.spte_high = orig->spte_high;
466         smp_rmb();
467 
468         if (unlikely(spte.spte_low != orig->spte_low ||
469               count != sp->clear_spte_count))
470                 goto retry;
471 
472         return spte.spte;
473 }
474 #endif
475 
476 static bool spte_is_locklessly_modifiable(u64 spte)
477 {
478         return (spte & (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE)) ==
479                 (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE);
480 }
481 
482 static bool spte_has_volatile_bits(u64 spte)
483 {
484         /*
485          * Always atomically update spte if it can be updated
486          * out of mmu-lock, it can ensure dirty bit is not lost,
487          * also, it can help us to get a stable is_writable_pte()
488          * to ensure tlb flush is not missed.
489          */
490         if (spte_is_locklessly_modifiable(spte))
491                 return true;
492 
493         if (!shadow_accessed_mask)
494                 return false;
495 
496         if (!is_shadow_present_pte(spte))
497                 return false;
498 
499         if ((spte & shadow_accessed_mask) &&
500               (!is_writable_pte(spte) || (spte & shadow_dirty_mask)))
501                 return false;
502 
503         return true;
504 }
505 
506 static bool spte_is_bit_cleared(u64 old_spte, u64 new_spte, u64 bit_mask)
507 {
508         return (old_spte & bit_mask) && !(new_spte & bit_mask);
509 }
510 
511 static bool spte_is_bit_changed(u64 old_spte, u64 new_spte, u64 bit_mask)
512 {
513         return (old_spte & bit_mask) != (new_spte & bit_mask);
514 }
515 
516 /* Rules for using mmu_spte_set:
517  * Set the sptep from nonpresent to present.
518  * Note: the sptep being assigned *must* be either not present
519  * or in a state where the hardware will not attempt to update
520  * the spte.
521  */
522 static void mmu_spte_set(u64 *sptep, u64 new_spte)
523 {
524         WARN_ON(is_shadow_present_pte(*sptep));
525         __set_spte(sptep, new_spte);
526 }
527 
528 /* Rules for using mmu_spte_update:
529  * Update the state bits, it means the mapped pfn is not changed.
530  *
531  * Whenever we overwrite a writable spte with a read-only one we
532  * should flush remote TLBs. Otherwise rmap_write_protect
533  * will find a read-only spte, even though the writable spte
534  * might be cached on a CPU's TLB, the return value indicates this
535  * case.
536  */
537 static bool mmu_spte_update(u64 *sptep, u64 new_spte)
538 {
539         u64 old_spte = *sptep;
540         bool ret = false;
541 
542         WARN_ON(!is_shadow_present_pte(new_spte));
543 
544         if (!is_shadow_present_pte(old_spte)) {
545                 mmu_spte_set(sptep, new_spte);
546                 return ret;
547         }
548 
549         if (!spte_has_volatile_bits(old_spte))
550                 __update_clear_spte_fast(sptep, new_spte);
551         else
552                 old_spte = __update_clear_spte_slow(sptep, new_spte);
553 
554         /*
555          * For the spte updated out of mmu-lock is safe, since
556          * we always atomically update it, see the comments in
557          * spte_has_volatile_bits().
558          */
559         if (spte_is_locklessly_modifiable(old_spte) &&
560               !is_writable_pte(new_spte))
561                 ret = true;
562 
563         if (!shadow_accessed_mask) {
564                 /*
565                  * We don't set page dirty when dropping non-writable spte.
566                  * So do it now if the new spte is becoming non-writable.
567                  */
568                 if (ret)
569                         kvm_set_pfn_dirty(spte_to_pfn(old_spte));
570                 return ret;
571         }
572 
573         /*
574          * Flush TLB when accessed/dirty bits are changed in the page tables,
575          * to guarantee consistency between TLB and page tables.
576          */
577         if (spte_is_bit_changed(old_spte, new_spte,
578                                 shadow_accessed_mask | shadow_dirty_mask))
579                 ret = true;
580 
581         if (spte_is_bit_cleared(old_spte, new_spte, shadow_accessed_mask))
582                 kvm_set_pfn_accessed(spte_to_pfn(old_spte));
583         if (spte_is_bit_cleared(old_spte, new_spte, shadow_dirty_mask))
584                 kvm_set_pfn_dirty(spte_to_pfn(old_spte));
585 
586         return ret;
587 }
588 
589 /*
590  * Rules for using mmu_spte_clear_track_bits:
591  * It sets the sptep from present to nonpresent, and track the
592  * state bits, it is used to clear the last level sptep.
593  */
594 static int mmu_spte_clear_track_bits(u64 *sptep)
595 {
596         kvm_pfn_t pfn;
597         u64 old_spte = *sptep;
598 
599         if (!spte_has_volatile_bits(old_spte))
600                 __update_clear_spte_fast(sptep, 0ull);
601         else
602                 old_spte = __update_clear_spte_slow(sptep, 0ull);
603 
604         if (!is_shadow_present_pte(old_spte))
605                 return 0;
606 
607         pfn = spte_to_pfn(old_spte);
608 
609         /*
610          * KVM does not hold the refcount of the page used by
611          * kvm mmu, before reclaiming the page, we should
612          * unmap it from mmu first.
613          */
614         WARN_ON(!kvm_is_reserved_pfn(pfn) && !page_count(pfn_to_page(pfn)));
615 
616         if (!shadow_accessed_mask || old_spte & shadow_accessed_mask)
617                 kvm_set_pfn_accessed(pfn);
618         if (old_spte & (shadow_dirty_mask ? shadow_dirty_mask :
619                                             PT_WRITABLE_MASK))
620                 kvm_set_pfn_dirty(pfn);
621         return 1;
622 }
623 
624 /*
625  * Rules for using mmu_spte_clear_no_track:
626  * Directly clear spte without caring the state bits of sptep,
627  * it is used to set the upper level spte.
628  */
629 static void mmu_spte_clear_no_track(u64 *sptep)
630 {
631         __update_clear_spte_fast(sptep, 0ull);
632 }
633 
634 static u64 mmu_spte_get_lockless(u64 *sptep)
635 {
636         return __get_spte_lockless(sptep);
637 }
638 
639 static void walk_shadow_page_lockless_begin(struct kvm_vcpu *vcpu)
640 {
641         /*
642          * Prevent page table teardown by making any free-er wait during
643          * kvm_flush_remote_tlbs() IPI to all active vcpus.
644          */
645         local_irq_disable();
646 
647         /*
648          * Make sure a following spte read is not reordered ahead of the write
649          * to vcpu->mode.
650          */
651         smp_store_mb(vcpu->mode, READING_SHADOW_PAGE_TABLES);
652 }
653 
654 static void walk_shadow_page_lockless_end(struct kvm_vcpu *vcpu)
655 {
656         /*
657          * Make sure the write to vcpu->mode is not reordered in front of
658          * reads to sptes.  If it does, kvm_commit_zap_page() can see us
659          * OUTSIDE_GUEST_MODE and proceed to free the shadow page table.
660          */
661         smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE);
662         local_irq_enable();
663 }
664 
665 static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
666                                   struct kmem_cache *base_cache, int min)
667 {
668         void *obj;
669 
670         if (cache->nobjs >= min)
671                 return 0;
672         while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
673                 obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
674                 if (!obj)
675                         return -ENOMEM;
676                 cache->objects[cache->nobjs++] = obj;
677         }
678         return 0;
679 }
680 
681 static int mmu_memory_cache_free_objects(struct kvm_mmu_memory_cache *cache)
682 {
683         return cache->nobjs;
684 }
685 
686 static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc,
687                                   struct kmem_cache *cache)
688 {
689         while (mc->nobjs)
690                 kmem_cache_free(cache, mc->objects[--mc->nobjs]);
691 }
692 
693 static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
694                                        int min)
695 {
696         void *page;
697 
698         if (cache->nobjs >= min)
699                 return 0;
700         while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
701                 page = (void *)__get_free_page(GFP_KERNEL);
702                 if (!page)
703                         return -ENOMEM;
704                 cache->objects[cache->nobjs++] = page;
705         }
706         return 0;
707 }
708 
709 static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
710 {
711         while (mc->nobjs)
712                 free_page((unsigned long)mc->objects[--mc->nobjs]);
713 }
714 
715 static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
716 {
717         int r;
718 
719         r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
720                                    pte_list_desc_cache, 8 + PTE_PREFETCH_NUM);
721         if (r)
722                 goto out;
723         r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
724         if (r)
725                 goto out;
726         r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
727                                    mmu_page_header_cache, 4);
728 out:
729         return r;
730 }
731 
732 static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
733 {
734         mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
735                                 pte_list_desc_cache);
736         mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
737         mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache,
738                                 mmu_page_header_cache);
739 }
740 
741 static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
742 {
743         void *p;
744 
745         BUG_ON(!mc->nobjs);
746         p = mc->objects[--mc->nobjs];
747         return p;
748 }
749 
750 static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu)
751 {
752         return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache);
753 }
754 
755 static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc)
756 {
757         kmem_cache_free(pte_list_desc_cache, pte_list_desc);
758 }
759 
760 static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
761 {
762         if (!sp->role.direct)
763                 return sp->gfns[index];
764 
765         return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
766 }
767 
768 static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
769 {
770         if (sp->role.direct)
771                 BUG_ON(gfn != kvm_mmu_page_get_gfn(sp, index));
772         else
773                 sp->gfns[index] = gfn;
774 }
775 
776 /*
777  * Return the pointer to the large page information for a given gfn,
778  * handling slots that are not large page aligned.
779  */
780 static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
781                                               struct kvm_memory_slot *slot,
782                                               int level)
783 {
784         unsigned long idx;
785 
786         idx = gfn_to_index(gfn, slot->base_gfn, level);
787         return &slot->arch.lpage_info[level - 2][idx];
788 }
789 
790 static void update_gfn_disallow_lpage_count(struct kvm_memory_slot *slot,
791                                             gfn_t gfn, int count)
792 {
793         struct kvm_lpage_info *linfo;
794         int i;
795 
796         for (i = PT_DIRECTORY_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) {
797                 linfo = lpage_info_slot(gfn, slot, i);
798                 linfo->disallow_lpage += count;
799                 WARN_ON(linfo->disallow_lpage < 0);
800         }
801 }
802 
803 void kvm_mmu_gfn_disallow_lpage(struct kvm_memory_slot *slot, gfn_t gfn)
804 {
805         update_gfn_disallow_lpage_count(slot, gfn, 1);
806 }
807 
808 void kvm_mmu_gfn_allow_lpage(struct kvm_memory_slot *slot, gfn_t gfn)
809 {
810         update_gfn_disallow_lpage_count(slot, gfn, -1);
811 }
812 
813 static void account_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
814 {
815         struct kvm_memslots *slots;
816         struct kvm_memory_slot *slot;
817         gfn_t gfn;
818 
819         kvm->arch.indirect_shadow_pages++;
820         gfn = sp->gfn;
821         slots = kvm_memslots_for_spte_role(kvm, sp->role);
822         slot = __gfn_to_memslot(slots, gfn);
823 
824         /* the non-leaf shadow pages are keeping readonly. */
825         if (sp->role.level > PT_PAGE_TABLE_LEVEL)
826                 return kvm_slot_page_track_add_page(kvm, slot, gfn,
827                                                     KVM_PAGE_TRACK_WRITE);
828 
829         kvm_mmu_gfn_disallow_lpage(slot, gfn);
830 }
831 
832 static void unaccount_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
833 {
834         struct kvm_memslots *slots;
835         struct kvm_memory_slot *slot;
836         gfn_t gfn;
837 
838         kvm->arch.indirect_shadow_pages--;
839         gfn = sp->gfn;
840         slots = kvm_memslots_for_spte_role(kvm, sp->role);
841         slot = __gfn_to_memslot(slots, gfn);
842         if (sp->role.level > PT_PAGE_TABLE_LEVEL)
843                 return kvm_slot_page_track_remove_page(kvm, slot, gfn,
844                                                        KVM_PAGE_TRACK_WRITE);
845 
846         kvm_mmu_gfn_allow_lpage(slot, gfn);
847 }
848 
849 static bool __mmu_gfn_lpage_is_disallowed(gfn_t gfn, int level,
850                                           struct kvm_memory_slot *slot)
851 {
852         struct kvm_lpage_info *linfo;
853 
854         if (slot) {
855                 linfo = lpage_info_slot(gfn, slot, level);
856                 return !!linfo->disallow_lpage;
857         }
858 
859         return true;
860 }
861 
862 static bool mmu_gfn_lpage_is_disallowed(struct kvm_vcpu *vcpu, gfn_t gfn,
863                                         int level)
864 {
865         struct kvm_memory_slot *slot;
866 
867         slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
868         return __mmu_gfn_lpage_is_disallowed(gfn, level, slot);
869 }
870 
871 static int host_mapping_level(struct kvm *kvm, gfn_t gfn)
872 {
873         unsigned long page_size;
874         int i, ret = 0;
875 
876         page_size = kvm_host_page_size(kvm, gfn);
877 
878         for (i = PT_PAGE_TABLE_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) {
879                 if (page_size >= KVM_HPAGE_SIZE(i))
880                         ret = i;
881                 else
882                         break;
883         }
884 
885         return ret;
886 }
887 
888 static inline bool memslot_valid_for_gpte(struct kvm_memory_slot *slot,
889                                           bool no_dirty_log)
890 {
891         if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
892                 return false;
893         if (no_dirty_log && slot->dirty_bitmap)
894                 return false;
895 
896         return true;
897 }
898 
899 static struct kvm_memory_slot *
900 gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t gfn,
901                             bool no_dirty_log)
902 {
903         struct kvm_memory_slot *slot;
904 
905         slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
906         if (!memslot_valid_for_gpte(slot, no_dirty_log))
907                 slot = NULL;
908 
909         return slot;
910 }
911 
912 static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn,
913                          bool *force_pt_level)
914 {
915         int host_level, level, max_level;
916         struct kvm_memory_slot *slot;
917 
918         if (unlikely(*force_pt_level))
919                 return PT_PAGE_TABLE_LEVEL;
920 
921         slot = kvm_vcpu_gfn_to_memslot(vcpu, large_gfn);
922         *force_pt_level = !memslot_valid_for_gpte(slot, true);
923         if (unlikely(*force_pt_level))
924                 return PT_PAGE_TABLE_LEVEL;
925 
926         host_level = host_mapping_level(vcpu->kvm, large_gfn);
927 
928         if (host_level == PT_PAGE_TABLE_LEVEL)
929                 return host_level;
930 
931         max_level = min(kvm_x86_ops->get_lpage_level(), host_level);
932 
933         for (level = PT_DIRECTORY_LEVEL; level <= max_level; ++level)
934                 if (__mmu_gfn_lpage_is_disallowed(large_gfn, level, slot))
935                         break;
936 
937         return level - 1;
938 }
939 
940 /*
941  * About rmap_head encoding:
942  *
943  * If the bit zero of rmap_head->val is clear, then it points to the only spte
944  * in this rmap chain. Otherwise, (rmap_head->val & ~1) points to a struct
945  * pte_list_desc containing more mappings.
946  */
947 
948 /*
949  * Returns the number of pointers in the rmap chain, not counting the new one.
950  */
951 static int pte_list_add(struct kvm_vcpu *vcpu, u64 *spte,
952                         struct kvm_rmap_head *rmap_head)
953 {
954         struct pte_list_desc *desc;
955         int i, count = 0;
956 
957         if (!rmap_head->val) {
958                 rmap_printk("pte_list_add: %p %llx 0->1\n", spte, *spte);
959                 rmap_head->val = (unsigned long)spte;
960         } else if (!(rmap_head->val & 1)) {
961                 rmap_printk("pte_list_add: %p %llx 1->many\n", spte, *spte);
962                 desc = mmu_alloc_pte_list_desc(vcpu);
963                 desc->sptes[0] = (u64 *)rmap_head->val;
964                 desc->sptes[1] = spte;
965                 rmap_head->val = (unsigned long)desc | 1;
966                 ++count;
967         } else {
968                 rmap_printk("pte_list_add: %p %llx many->many\n", spte, *spte);
969                 desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
970                 while (desc->sptes[PTE_LIST_EXT-1] && desc->more) {
971                         desc = desc->more;
972                         count += PTE_LIST_EXT;
973                 }
974                 if (desc->sptes[PTE_LIST_EXT-1]) {
975                         desc->more = mmu_alloc_pte_list_desc(vcpu);
976                         desc = desc->more;
977                 }
978                 for (i = 0; desc->sptes[i]; ++i)
979                         ++count;
980                 desc->sptes[i] = spte;
981         }
982         return count;
983 }
984 
985 static void
986 pte_list_desc_remove_entry(struct kvm_rmap_head *rmap_head,
987                            struct pte_list_desc *desc, int i,
988                            struct pte_list_desc *prev_desc)
989 {
990         int j;
991 
992         for (j = PTE_LIST_EXT - 1; !desc->sptes[j] && j > i; --j)
993                 ;
994         desc->sptes[i] = desc->sptes[j];
995         desc->sptes[j] = NULL;
996         if (j != 0)
997                 return;
998         if (!prev_desc && !desc->more)
999                 rmap_head->val = (unsigned long)desc->sptes[0];
1000         else
1001                 if (prev_desc)
1002                         prev_desc->more = desc->more;
1003                 else
1004                         rmap_head->val = (unsigned long)desc->more | 1;
1005         mmu_free_pte_list_desc(desc);
1006 }
1007 
1008 static void pte_list_remove(u64 *spte, struct kvm_rmap_head *rmap_head)
1009 {
1010         struct pte_list_desc *desc;
1011         struct pte_list_desc *prev_desc;
1012         int i;
1013 
1014         if (!rmap_head->val) {
1015                 printk(KERN_ERR "pte_list_remove: %p 0->BUG\n", spte);
1016                 BUG();
1017         } else if (!(rmap_head->val & 1)) {
1018                 rmap_printk("pte_list_remove:  %p 1->0\n", spte);
1019                 if ((u64 *)rmap_head->val != spte) {
1020                         printk(KERN_ERR "pte_list_remove:  %p 1->BUG\n", spte);
1021                         BUG();
1022                 }
1023                 rmap_head->val = 0;
1024         } else {
1025                 rmap_printk("pte_list_remove:  %p many->many\n", spte);
1026                 desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1027                 prev_desc = NULL;
1028                 while (desc) {
1029                         for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i) {
1030                                 if (desc->sptes[i] == spte) {
1031                                         pte_list_desc_remove_entry(rmap_head,
1032                                                         desc, i, prev_desc);
1033                                         return;
1034                                 }
1035                         }
1036                         prev_desc = desc;
1037                         desc = desc->more;
1038                 }
1039                 pr_err("pte_list_remove: %p many->many\n", spte);
1040                 BUG();
1041         }
1042 }
1043 
1044 static struct kvm_rmap_head *__gfn_to_rmap(gfn_t gfn, int level,
1045                                            struct kvm_memory_slot *slot)
1046 {
1047         unsigned long idx;
1048 
1049         idx = gfn_to_index(gfn, slot->base_gfn, level);
1050         return &slot->arch.rmap[level - PT_PAGE_TABLE_LEVEL][idx];
1051 }
1052 
1053 static struct kvm_rmap_head *gfn_to_rmap(struct kvm *kvm, gfn_t gfn,
1054                                          struct kvm_mmu_page *sp)
1055 {
1056         struct kvm_memslots *slots;
1057         struct kvm_memory_slot *slot;
1058 
1059         slots = kvm_memslots_for_spte_role(kvm, sp->role);
1060         slot = __gfn_to_memslot(slots, gfn);
1061         return __gfn_to_rmap(gfn, sp->role.level, slot);
1062 }
1063 
1064 static bool rmap_can_add(struct kvm_vcpu *vcpu)
1065 {
1066         struct kvm_mmu_memory_cache *cache;
1067 
1068         cache = &vcpu->arch.mmu_pte_list_desc_cache;
1069         return mmu_memory_cache_free_objects(cache);
1070 }
1071 
1072 static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
1073 {
1074         struct kvm_mmu_page *sp;
1075         struct kvm_rmap_head *rmap_head;
1076 
1077         sp = page_header(__pa(spte));
1078         kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
1079         rmap_head = gfn_to_rmap(vcpu->kvm, gfn, sp);
1080         return pte_list_add(vcpu, spte, rmap_head);
1081 }
1082 
1083 static void rmap_remove(struct kvm *kvm, u64 *spte)
1084 {
1085         struct kvm_mmu_page *sp;
1086         gfn_t gfn;
1087         struct kvm_rmap_head *rmap_head;
1088 
1089         sp = page_header(__pa(spte));
1090         gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
1091         rmap_head = gfn_to_rmap(kvm, gfn, sp);
1092         pte_list_remove(spte, rmap_head);
1093 }
1094 
1095 /*
1096  * Used by the following functions to iterate through the sptes linked by a
1097  * rmap.  All fields are private and not assumed to be used outside.
1098  */
1099 struct rmap_iterator {
1100         /* private fields */
1101         struct pte_list_desc *desc;     /* holds the sptep if not NULL */
1102         int pos;                        /* index of the sptep */
1103 };
1104 
1105 /*
1106  * Iteration must be started by this function.  This should also be used after
1107  * removing/dropping sptes from the rmap link because in such cases the
1108  * information in the itererator may not be valid.
1109  *
1110  * Returns sptep if found, NULL otherwise.
1111  */
1112 static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head,
1113                            struct rmap_iterator *iter)
1114 {
1115         u64 *sptep;
1116 
1117         if (!rmap_head->val)
1118                 return NULL;
1119 
1120         if (!(rmap_head->val & 1)) {
1121                 iter->desc = NULL;
1122                 sptep = (u64 *)rmap_head->val;
1123                 goto out;
1124         }
1125 
1126         iter->desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1127         iter->pos = 0;
1128         sptep = iter->desc->sptes[iter->pos];
1129 out:
1130         BUG_ON(!is_shadow_present_pte(*sptep));
1131         return sptep;
1132 }
1133 
1134 /*
1135  * Must be used with a valid iterator: e.g. after rmap_get_first().
1136  *
1137  * Returns sptep if found, NULL otherwise.
1138  */
1139 static u64 *rmap_get_next(struct rmap_iterator *iter)
1140 {
1141         u64 *sptep;
1142 
1143         if (iter->desc) {
1144                 if (iter->pos < PTE_LIST_EXT - 1) {
1145                         ++iter->pos;
1146                         sptep = iter->desc->sptes[iter->pos];
1147                         if (sptep)
1148                                 goto out;
1149                 }
1150 
1151                 iter->desc = iter->desc->more;
1152 
1153                 if (iter->desc) {
1154                         iter->pos = 0;
1155                         /* desc->sptes[0] cannot be NULL */
1156                         sptep = iter->desc->sptes[iter->pos];
1157                         goto out;
1158                 }
1159         }
1160 
1161         return NULL;
1162 out:
1163         BUG_ON(!is_shadow_present_pte(*sptep));
1164         return sptep;
1165 }
1166 
1167 #define for_each_rmap_spte(_rmap_head_, _iter_, _spte_)                 \
1168         for (_spte_ = rmap_get_first(_rmap_head_, _iter_);              \
1169              _spte_; _spte_ = rmap_get_next(_iter_))
1170 
1171 static void drop_spte(struct kvm *kvm, u64 *sptep)
1172 {
1173         if (mmu_spte_clear_track_bits(sptep))
1174                 rmap_remove(kvm, sptep);
1175 }
1176 
1177 
1178 static bool __drop_large_spte(struct kvm *kvm, u64 *sptep)
1179 {
1180         if (is_large_pte(*sptep)) {
1181                 WARN_ON(page_header(__pa(sptep))->role.level ==
1182                         PT_PAGE_TABLE_LEVEL);
1183                 drop_spte(kvm, sptep);
1184                 --kvm->stat.lpages;
1185                 return true;
1186         }
1187 
1188         return false;
1189 }
1190 
1191 static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
1192 {
1193         if (__drop_large_spte(vcpu->kvm, sptep))
1194                 kvm_flush_remote_tlbs(vcpu->kvm);
1195 }
1196 
1197 /*
1198  * Write-protect on the specified @sptep, @pt_protect indicates whether
1199  * spte write-protection is caused by protecting shadow page table.
1200  *
1201  * Note: write protection is difference between dirty logging and spte
1202  * protection:
1203  * - for dirty logging, the spte can be set to writable at anytime if
1204  *   its dirty bitmap is properly set.
1205  * - for spte protection, the spte can be writable only after unsync-ing
1206  *   shadow page.
1207  *
1208  * Return true if tlb need be flushed.
1209  */
1210 static bool spte_write_protect(u64 *sptep, bool pt_protect)
1211 {
1212         u64 spte = *sptep;
1213 
1214         if (!is_writable_pte(spte) &&
1215               !(pt_protect && spte_is_locklessly_modifiable(spte)))
1216                 return false;
1217 
1218         rmap_printk("rmap_write_protect: spte %p %llx\n", sptep, *sptep);
1219 
1220         if (pt_protect)
1221                 spte &= ~SPTE_MMU_WRITEABLE;
1222         spte = spte & ~PT_WRITABLE_MASK;
1223 
1224         return mmu_spte_update(sptep, spte);
1225 }
1226 
1227 static bool __rmap_write_protect(struct kvm *kvm,
1228                                  struct kvm_rmap_head *rmap_head,
1229                                  bool pt_protect)
1230 {
1231         u64 *sptep;
1232         struct rmap_iterator iter;
1233         bool flush = false;
1234 
1235         for_each_rmap_spte(rmap_head, &iter, sptep)
1236                 flush |= spte_write_protect(sptep, pt_protect);
1237 
1238         return flush;
1239 }
1240 
1241 static bool spte_clear_dirty(u64 *sptep)
1242 {
1243         u64 spte = *sptep;
1244 
1245         rmap_printk("rmap_clear_dirty: spte %p %llx\n", sptep, *sptep);
1246 
1247         spte &= ~shadow_dirty_mask;
1248 
1249         return mmu_spte_update(sptep, spte);
1250 }
1251 
1252 static bool __rmap_clear_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head)
1253 {
1254         u64 *sptep;
1255         struct rmap_iterator iter;
1256         bool flush = false;
1257 
1258         for_each_rmap_spte(rmap_head, &iter, sptep)
1259                 flush |= spte_clear_dirty(sptep);
1260 
1261         return flush;
1262 }
1263 
1264 static bool spte_set_dirty(u64 *sptep)
1265 {
1266         u64 spte = *sptep;
1267 
1268         rmap_printk("rmap_set_dirty: spte %p %llx\n", sptep, *sptep);
1269 
1270         spte |= shadow_dirty_mask;
1271 
1272         return mmu_spte_update(sptep, spte);
1273 }
1274 
1275 static bool __rmap_set_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head)
1276 {
1277         u64 *sptep;
1278         struct rmap_iterator iter;
1279         bool flush = false;
1280 
1281         for_each_rmap_spte(rmap_head, &iter, sptep)
1282                 flush |= spte_set_dirty(sptep);
1283 
1284         return flush;
1285 }
1286 
1287 /**
1288  * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages
1289  * @kvm: kvm instance
1290  * @slot: slot to protect
1291  * @gfn_offset: start of the BITS_PER_LONG pages we care about
1292  * @mask: indicates which pages we should protect
1293  *
1294  * Used when we do not need to care about huge page mappings: e.g. during dirty
1295  * logging we do not have any such mappings.
1296  */
1297 static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
1298                                      struct kvm_memory_slot *slot,
1299                                      gfn_t gfn_offset, unsigned long mask)
1300 {
1301         struct kvm_rmap_head *rmap_head;
1302 
1303         while (mask) {
1304                 rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
1305                                           PT_PAGE_TABLE_LEVEL, slot);
1306                 __rmap_write_protect(kvm, rmap_head, false);
1307 
1308                 /* clear the first set bit */
1309                 mask &= mask - 1;
1310         }
1311 }
1312 
1313 /**
1314  * kvm_mmu_clear_dirty_pt_masked - clear MMU D-bit for PT level pages
1315  * @kvm: kvm instance
1316  * @slot: slot to clear D-bit
1317  * @gfn_offset: start of the BITS_PER_LONG pages we care about
1318  * @mask: indicates which pages we should clear D-bit
1319  *
1320  * Used for PML to re-log the dirty GPAs after userspace querying dirty_bitmap.
1321  */
1322 void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm,
1323                                      struct kvm_memory_slot *slot,
1324                                      gfn_t gfn_offset, unsigned long mask)
1325 {
1326         struct kvm_rmap_head *rmap_head;
1327 
1328         while (mask) {
1329                 rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
1330                                           PT_PAGE_TABLE_LEVEL, slot);
1331                 __rmap_clear_dirty(kvm, rmap_head);
1332 
1333                 /* clear the first set bit */
1334                 mask &= mask - 1;
1335         }
1336 }
1337 EXPORT_SYMBOL_GPL(kvm_mmu_clear_dirty_pt_masked);
1338 
1339 /**
1340  * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected
1341  * PT level pages.
1342  *
1343  * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to
1344  * enable dirty logging for them.
1345  *
1346  * Used when we do not need to care about huge page mappings: e.g. during dirty
1347  * logging we do not have any such mappings.
1348  */
1349 void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
1350                                 struct kvm_memory_slot *slot,
1351                                 gfn_t gfn_offset, unsigned long mask)
1352 {
1353         if (kvm_x86_ops->enable_log_dirty_pt_masked)
1354                 kvm_x86_ops->enable_log_dirty_pt_masked(kvm, slot, gfn_offset,
1355                                 mask);
1356         else
1357                 kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
1358 }
1359 
1360 bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm,
1361                                     struct kvm_memory_slot *slot, u64 gfn)
1362 {
1363         struct kvm_rmap_head *rmap_head;
1364         int i;
1365         bool write_protected = false;
1366 
1367         for (i = PT_PAGE_TABLE_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) {
1368                 rmap_head = __gfn_to_rmap(gfn, i, slot);
1369                 write_protected |= __rmap_write_protect(kvm, rmap_head, true);
1370         }
1371 
1372         return write_protected;
1373 }
1374 
1375 static bool rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn)
1376 {
1377         struct kvm_memory_slot *slot;
1378 
1379         slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1380         return kvm_mmu_slot_gfn_write_protect(vcpu->kvm, slot, gfn);
1381 }
1382 
1383 static bool kvm_zap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head)
1384 {
1385         u64 *sptep;
1386         struct rmap_iterator iter;
1387         bool flush = false;
1388 
1389         while ((sptep = rmap_get_first(rmap_head, &iter))) {
1390                 rmap_printk("%s: spte %p %llx.\n", __func__, sptep, *sptep);
1391 
1392                 drop_spte(kvm, sptep);
1393                 flush = true;
1394         }
1395 
1396         return flush;
1397 }
1398 
1399 static int kvm_unmap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1400                            struct kvm_memory_slot *slot, gfn_t gfn, int level,
1401                            unsigned long data)
1402 {
1403         return kvm_zap_rmapp(kvm, rmap_head);
1404 }
1405 
1406 static int kvm_set_pte_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1407                              struct kvm_memory_slot *slot, gfn_t gfn, int level,
1408                              unsigned long data)
1409 {
1410         u64 *sptep;
1411         struct rmap_iterator iter;
1412         int need_flush = 0;
1413         u64 new_spte;
1414         pte_t *ptep = (pte_t *)data;
1415         kvm_pfn_t new_pfn;
1416 
1417         WARN_ON(pte_huge(*ptep));
1418         new_pfn = pte_pfn(*ptep);
1419 
1420 restart:
1421         for_each_rmap_spte(rmap_head, &iter, sptep) {
1422                 rmap_printk("kvm_set_pte_rmapp: spte %p %llx gfn %llx (%d)\n",
1423                              sptep, *sptep, gfn, level);
1424 
1425                 need_flush = 1;
1426 
1427                 if (pte_write(*ptep)) {
1428                         drop_spte(kvm, sptep);
1429                         goto restart;
1430                 } else {
1431                         new_spte = *sptep & ~PT64_BASE_ADDR_MASK;
1432                         new_spte |= (u64)new_pfn << PAGE_SHIFT;
1433 
1434                         new_spte &= ~PT_WRITABLE_MASK;
1435                         new_spte &= ~SPTE_HOST_WRITEABLE;
1436                         new_spte &= ~shadow_accessed_mask;
1437 
1438                         mmu_spte_clear_track_bits(sptep);
1439                         mmu_spte_set(sptep, new_spte);
1440                 }
1441         }
1442 
1443         if (need_flush)
1444                 kvm_flush_remote_tlbs(kvm);
1445 
1446         return 0;
1447 }
1448 
1449 struct slot_rmap_walk_iterator {
1450         /* input fields. */
1451         struct kvm_memory_slot *slot;
1452         gfn_t start_gfn;
1453         gfn_t end_gfn;
1454         int start_level;
1455         int end_level;
1456 
1457         /* output fields. */
1458         gfn_t gfn;
1459         struct kvm_rmap_head *rmap;
1460         int level;
1461 
1462         /* private field. */
1463         struct kvm_rmap_head *end_rmap;
1464 };
1465 
1466 static void
1467 rmap_walk_init_level(struct slot_rmap_walk_iterator *iterator, int level)
1468 {
1469         iterator->level = level;
1470         iterator->gfn = iterator->start_gfn;
1471         iterator->rmap = __gfn_to_rmap(iterator->gfn, level, iterator->slot);
1472         iterator->end_rmap = __gfn_to_rmap(iterator->end_gfn, level,
1473                                            iterator->slot);
1474 }
1475 
1476 static void
1477 slot_rmap_walk_init(struct slot_rmap_walk_iterator *iterator,
1478                     struct kvm_memory_slot *slot, int start_level,
1479                     int end_level, gfn_t start_gfn, gfn_t end_gfn)
1480 {
1481         iterator->slot = slot;
1482         iterator->start_level = start_level;
1483         iterator->end_level = end_level;
1484         iterator->start_gfn = start_gfn;
1485         iterator->end_gfn = end_gfn;
1486 
1487         rmap_walk_init_level(iterator, iterator->start_level);
1488 }
1489 
1490 static bool slot_rmap_walk_okay(struct slot_rmap_walk_iterator *iterator)
1491 {
1492         return !!iterator->rmap;
1493 }
1494 
1495 static void slot_rmap_walk_next(struct slot_rmap_walk_iterator *iterator)
1496 {
1497         if (++iterator->rmap <= iterator->end_rmap) {
1498                 iterator->gfn += (1UL << KVM_HPAGE_GFN_SHIFT(iterator->level));
1499                 return;
1500         }
1501 
1502         if (++iterator->level > iterator->end_level) {
1503                 iterator->rmap = NULL;
1504                 return;
1505         }
1506 
1507         rmap_walk_init_level(iterator, iterator->level);
1508 }
1509 
1510 #define for_each_slot_rmap_range(_slot_, _start_level_, _end_level_,    \
1511            _start_gfn, _end_gfn, _iter_)                                \
1512         for (slot_rmap_walk_init(_iter_, _slot_, _start_level_,         \
1513                                  _end_level_, _start_gfn, _end_gfn);    \
1514              slot_rmap_walk_okay(_iter_);                               \
1515              slot_rmap_walk_next(_iter_))
1516 
1517 static int kvm_handle_hva_range(struct kvm *kvm,
1518                                 unsigned long start,
1519                                 unsigned long end,
1520                                 unsigned long data,
1521                                 int (*handler)(struct kvm *kvm,
1522                                                struct kvm_rmap_head *rmap_head,
1523                                                struct kvm_memory_slot *slot,
1524                                                gfn_t gfn,
1525                                                int level,
1526                                                unsigned long data))
1527 {
1528         struct kvm_memslots *slots;
1529         struct kvm_memory_slot *memslot;
1530         struct slot_rmap_walk_iterator iterator;
1531         int ret = 0;
1532         int i;
1533 
1534         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
1535                 slots = __kvm_memslots(kvm, i);
1536                 kvm_for_each_memslot(memslot, slots) {
1537                         unsigned long hva_start, hva_end;
1538                         gfn_t gfn_start, gfn_end;
1539 
1540                         hva_start = max(start, memslot->userspace_addr);
1541                         hva_end = min(end, memslot->userspace_addr +
1542                                       (memslot->npages << PAGE_SHIFT));
1543                         if (hva_start >= hva_end)
1544                                 continue;
1545                         /*
1546                          * {gfn(page) | page intersects with [hva_start, hva_end)} =
1547                          * {gfn_start, gfn_start+1, ..., gfn_end-1}.
1548                          */
1549                         gfn_start = hva_to_gfn_memslot(hva_start, memslot);
1550                         gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
1551 
1552                         for_each_slot_rmap_range(memslot, PT_PAGE_TABLE_LEVEL,
1553                                                  PT_MAX_HUGEPAGE_LEVEL,
1554                                                  gfn_start, gfn_end - 1,
1555                                                  &iterator)
1556                                 ret |= handler(kvm, iterator.rmap, memslot,
1557                                                iterator.gfn, iterator.level, data);
1558                 }
1559         }
1560 
1561         return ret;
1562 }
1563 
1564 static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
1565                           unsigned long data,
1566                           int (*handler)(struct kvm *kvm,
1567                                          struct kvm_rmap_head *rmap_head,
1568                                          struct kvm_memory_slot *slot,
1569                                          gfn_t gfn, int level,
1570                                          unsigned long data))
1571 {
1572         return kvm_handle_hva_range(kvm, hva, hva + 1, data, handler);
1573 }
1574 
1575 int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
1576 {
1577         return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp);
1578 }
1579 
1580 int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
1581 {
1582         return kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp);
1583 }
1584 
1585 void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
1586 {
1587         kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
1588 }
1589 
1590 static int kvm_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1591                          struct kvm_memory_slot *slot, gfn_t gfn, int level,
1592                          unsigned long data)
1593 {
1594         u64 *sptep;
1595         struct rmap_iterator uninitialized_var(iter);
1596         int young = 0;
1597 
1598         BUG_ON(!shadow_accessed_mask);
1599 
1600         for_each_rmap_spte(rmap_head, &iter, sptep) {
1601                 if (*sptep & shadow_accessed_mask) {
1602                         young = 1;
1603                         clear_bit((ffs(shadow_accessed_mask) - 1),
1604                                  (unsigned long *)sptep);
1605                 }
1606         }
1607 
1608         trace_kvm_age_page(gfn, level, slot, young);
1609         return young;
1610 }
1611 
1612 static int kvm_test_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1613                               struct kvm_memory_slot *slot, gfn_t gfn,
1614                               int level, unsigned long data)
1615 {
1616         u64 *sptep;
1617         struct rmap_iterator iter;
1618         int young = 0;
1619 
1620         /*
1621          * If there's no access bit in the secondary pte set by the
1622          * hardware it's up to gup-fast/gup to set the access bit in
1623          * the primary pte or in the page structure.
1624          */
1625         if (!shadow_accessed_mask)
1626                 goto out;
1627 
1628         for_each_rmap_spte(rmap_head, &iter, sptep) {
1629                 if (*sptep & shadow_accessed_mask) {
1630                         young = 1;
1631                         break;
1632                 }
1633         }
1634 out:
1635         return young;
1636 }
1637 
1638 #define RMAP_RECYCLE_THRESHOLD 1000
1639 
1640 static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
1641 {
1642         struct kvm_rmap_head *rmap_head;
1643         struct kvm_mmu_page *sp;
1644 
1645         sp = page_header(__pa(spte));
1646 
1647         rmap_head = gfn_to_rmap(vcpu->kvm, gfn, sp);
1648 
1649         kvm_unmap_rmapp(vcpu->kvm, rmap_head, NULL, gfn, sp->role.level, 0);
1650         kvm_flush_remote_tlbs(vcpu->kvm);
1651 }
1652 
1653 int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
1654 {
1655         /*
1656          * In case of absence of EPT Access and Dirty Bits supports,
1657          * emulate the accessed bit for EPT, by checking if this page has
1658          * an EPT mapping, and clearing it if it does. On the next access,
1659          * a new EPT mapping will be established.
1660          * This has some overhead, but not as much as the cost of swapping
1661          * out actively used pages or breaking up actively used hugepages.
1662          */
1663         if (!shadow_accessed_mask)
1664                 return kvm_handle_hva_range(kvm, start, end, 0,
1665                                             kvm_unmap_rmapp);
1666 
1667         return kvm_handle_hva_range(kvm, start, end, 0, kvm_age_rmapp);
1668 }
1669 
1670 int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
1671 {
1672         return kvm_handle_hva(kvm, hva, 0, kvm_test_age_rmapp);
1673 }
1674 
1675 #ifdef MMU_DEBUG
1676 static int is_empty_shadow_page(u64 *spt)
1677 {
1678         u64 *pos;
1679         u64 *end;
1680 
1681         for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
1682                 if (is_shadow_present_pte(*pos)) {
1683                         printk(KERN_ERR "%s: %p %llx\n", __func__,
1684                                pos, *pos);
1685                         return 0;
1686                 }
1687         return 1;
1688 }
1689 #endif
1690 
1691 /*
1692  * This value is the sum of all of the kvm instances's
1693  * kvm->arch.n_used_mmu_pages values.  We need a global,
1694  * aggregate version in order to make the slab shrinker
1695  * faster
1696  */
1697 static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, int nr)
1698 {
1699         kvm->arch.n_used_mmu_pages += nr;
1700         percpu_counter_add(&kvm_total_used_mmu_pages, nr);
1701 }
1702 
1703 static void kvm_mmu_free_page(struct kvm_mmu_page *sp)
1704 {
1705         MMU_WARN_ON(!is_empty_shadow_page(sp->spt));
1706         hlist_del(&sp->hash_link);
1707         list_del(&sp->link);
1708         free_page((unsigned long)sp->spt);
1709         if (!sp->role.direct)
1710                 free_page((unsigned long)sp->gfns);
1711         kmem_cache_free(mmu_page_header_cache, sp);
1712 }
1713 
1714 static unsigned kvm_page_table_hashfn(gfn_t gfn)
1715 {
1716         return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
1717 }
1718 
1719 static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
1720                                     struct kvm_mmu_page *sp, u64 *parent_pte)
1721 {
1722         if (!parent_pte)
1723                 return;
1724 
1725         pte_list_add(vcpu, parent_pte, &sp->parent_ptes);
1726 }
1727 
1728 static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
1729                                        u64 *parent_pte)
1730 {
1731         pte_list_remove(parent_pte, &sp->parent_ptes);
1732 }
1733 
1734 static void drop_parent_pte(struct kvm_mmu_page *sp,
1735                             u64 *parent_pte)
1736 {
1737         mmu_page_remove_parent_pte(sp, parent_pte);
1738         mmu_spte_clear_no_track(parent_pte);
1739 }
1740 
1741 static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, int direct)
1742 {
1743         struct kvm_mmu_page *sp;
1744 
1745         sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache);
1746         sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
1747         if (!direct)
1748                 sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
1749         set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
1750 
1751         /*
1752          * The active_mmu_pages list is the FIFO list, do not move the
1753          * page until it is zapped. kvm_zap_obsolete_pages depends on
1754          * this feature. See the comments in kvm_zap_obsolete_pages().
1755          */
1756         list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
1757         kvm_mod_used_mmu_pages(vcpu->kvm, +1);
1758         return sp;
1759 }
1760 
1761 static void mark_unsync(u64 *spte);
1762 static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
1763 {
1764         u64 *sptep;
1765         struct rmap_iterator iter;
1766 
1767         for_each_rmap_spte(&sp->parent_ptes, &iter, sptep) {
1768                 mark_unsync(sptep);
1769         }
1770 }
1771 
1772 static void mark_unsync(u64 *spte)
1773 {
1774         struct kvm_mmu_page *sp;
1775         unsigned int index;
1776 
1777         sp = page_header(__pa(spte));
1778         index = spte - sp->spt;
1779         if (__test_and_set_bit(index, sp->unsync_child_bitmap))
1780                 return;
1781         if (sp->unsync_children++)
1782                 return;
1783         kvm_mmu_mark_parents_unsync(sp);
1784 }
1785 
1786 static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
1787                                struct kvm_mmu_page *sp)
1788 {
1789         return 0;
1790 }
1791 
1792 static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
1793 {
1794 }
1795 
1796 static void nonpaging_update_pte(struct kvm_vcpu *vcpu,
1797                                  struct kvm_mmu_page *sp, u64 *spte,
1798                                  const void *pte)
1799 {
1800         WARN_ON(1);
1801 }
1802 
1803 #define KVM_PAGE_ARRAY_NR 16
1804 
1805 struct kvm_mmu_pages {
1806         struct mmu_page_and_offset {
1807                 struct kvm_mmu_page *sp;
1808                 unsigned int idx;
1809         } page[KVM_PAGE_ARRAY_NR];
1810         unsigned int nr;
1811 };
1812 
1813 static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
1814                          int idx)
1815 {
1816         int i;
1817 
1818         if (sp->unsync)
1819                 for (i=0; i < pvec->nr; i++)
1820                         if (pvec->page[i].sp == sp)
1821                                 return 0;
1822 
1823         pvec->page[pvec->nr].sp = sp;
1824         pvec->page[pvec->nr].idx = idx;
1825         pvec->nr++;
1826         return (pvec->nr == KVM_PAGE_ARRAY_NR);
1827 }
1828 
1829 static inline void clear_unsync_child_bit(struct kvm_mmu_page *sp, int idx)
1830 {
1831         --sp->unsync_children;
1832         WARN_ON((int)sp->unsync_children < 0);
1833         __clear_bit(idx, sp->unsync_child_bitmap);
1834 }
1835 
1836 static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
1837                            struct kvm_mmu_pages *pvec)
1838 {
1839         int i, ret, nr_unsync_leaf = 0;
1840 
1841         for_each_set_bit(i, sp->unsync_child_bitmap, 512) {
1842                 struct kvm_mmu_page *child;
1843                 u64 ent = sp->spt[i];
1844 
1845                 if (!is_shadow_present_pte(ent) || is_large_pte(ent)) {
1846                         clear_unsync_child_bit(sp, i);
1847                         continue;
1848                 }
1849 
1850                 child = page_header(ent & PT64_BASE_ADDR_MASK);
1851 
1852                 if (child->unsync_children) {
1853                         if (mmu_pages_add(pvec, child, i))
1854                                 return -ENOSPC;
1855 
1856                         ret = __mmu_unsync_walk(child, pvec);
1857                         if (!ret) {
1858                                 clear_unsync_child_bit(sp, i);
1859                                 continue;
1860                         } else if (ret > 0) {
1861                                 nr_unsync_leaf += ret;
1862                         } else
1863                                 return ret;
1864                 } else if (child->unsync) {
1865                         nr_unsync_leaf++;
1866                         if (mmu_pages_add(pvec, child, i))
1867                                 return -ENOSPC;
1868                 } else
1869                         clear_unsync_child_bit(sp, i);
1870         }
1871 
1872         return nr_unsync_leaf;
1873 }
1874 
1875 #define INVALID_INDEX (-1)
1876 
1877 static int mmu_unsync_walk(struct kvm_mmu_page *sp,
1878                            struct kvm_mmu_pages *pvec)
1879 {
1880         pvec->nr = 0;
1881         if (!sp->unsync_children)
1882                 return 0;
1883 
1884         mmu_pages_add(pvec, sp, INVALID_INDEX);
1885         return __mmu_unsync_walk(sp, pvec);
1886 }
1887 
1888 static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1889 {
1890         WARN_ON(!sp->unsync);
1891         trace_kvm_mmu_sync_page(sp);
1892         sp->unsync = 0;
1893         --kvm->stat.mmu_unsync;
1894 }
1895 
1896 static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
1897                                     struct list_head *invalid_list);
1898 static void kvm_mmu_commit_zap_page(struct kvm *kvm,
1899                                     struct list_head *invalid_list);
1900 
1901 /*
1902  * NOTE: we should pay more attention on the zapped-obsolete page
1903  * (is_obsolete_sp(sp) && sp->role.invalid) when you do hash list walk
1904  * since it has been deleted from active_mmu_pages but still can be found
1905  * at hast list.
1906  *
1907  * for_each_gfn_valid_sp() has skipped that kind of pages.
1908  */
1909 #define for_each_gfn_valid_sp(_kvm, _sp, _gfn)                          \
1910         hlist_for_each_entry(_sp,                                       \
1911           &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)], hash_link) \
1912                 if ((_sp)->gfn != (_gfn) || is_obsolete_sp((_kvm), (_sp)) \
1913                         || (_sp)->role.invalid) {} else
1914 
1915 #define for_each_gfn_indirect_valid_sp(_kvm, _sp, _gfn)                 \
1916         for_each_gfn_valid_sp(_kvm, _sp, _gfn)                          \
1917                 if ((_sp)->role.direct) {} else
1918 
1919 /* @sp->gfn should be write-protected at the call site */
1920 static bool __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
1921                             struct list_head *invalid_list)
1922 {
1923         if (sp->role.cr4_pae != !!is_pae(vcpu)) {
1924                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1925                 return false;
1926         }
1927 
1928         if (vcpu->arch.mmu.sync_page(vcpu, sp) == 0) {
1929                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1930                 return false;
1931         }
1932 
1933         return true;
1934 }
1935 
1936 static void kvm_mmu_flush_or_zap(struct kvm_vcpu *vcpu,
1937                                  struct list_head *invalid_list,
1938                                  bool remote_flush, bool local_flush)
1939 {
1940         if (!list_empty(invalid_list)) {
1941                 kvm_mmu_commit_zap_page(vcpu->kvm, invalid_list);
1942                 return;
1943         }
1944 
1945         if (remote_flush)
1946                 kvm_flush_remote_tlbs(vcpu->kvm);
1947         else if (local_flush)
1948                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1949 }
1950 
1951 #ifdef CONFIG_KVM_MMU_AUDIT
1952 #include "mmu_audit.c"
1953 #else
1954 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) { }
1955 static void mmu_audit_disable(void) { }
1956 #endif
1957 
1958 static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp)
1959 {
1960         return unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen);
1961 }
1962 
1963 static bool kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
1964                          struct list_head *invalid_list)
1965 {
1966         kvm_unlink_unsync_page(vcpu->kvm, sp);
1967         return __kvm_sync_page(vcpu, sp, invalid_list);
1968 }
1969 
1970 /* @gfn should be write-protected at the call site */
1971 static bool kvm_sync_pages(struct kvm_vcpu *vcpu, gfn_t gfn,
1972                            struct list_head *invalid_list)
1973 {
1974         struct kvm_mmu_page *s;
1975         bool ret = false;
1976 
1977         for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
1978                 if (!s->unsync)
1979                         continue;
1980 
1981                 WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
1982                 ret |= kvm_sync_page(vcpu, s, invalid_list);
1983         }
1984 
1985         return ret;
1986 }
1987 
1988 struct mmu_page_path {
1989         struct kvm_mmu_page *parent[PT64_ROOT_LEVEL];
1990         unsigned int idx[PT64_ROOT_LEVEL];
1991 };
1992 
1993 #define for_each_sp(pvec, sp, parents, i)                       \
1994                 for (i = mmu_pages_first(&pvec, &parents);      \
1995                         i < pvec.nr && ({ sp = pvec.page[i].sp; 1;});   \
1996                         i = mmu_pages_next(&pvec, &parents, i))
1997 
1998 static int mmu_pages_next(struct kvm_mmu_pages *pvec,
1999                           struct mmu_page_path *parents,
2000                           int i)
2001 {
2002         int n;
2003 
2004         for (n = i+1; n < pvec->nr; n++) {
2005                 struct kvm_mmu_page *sp = pvec->page[n].sp;
2006                 unsigned idx = pvec->page[n].idx;
2007                 int level = sp->role.level;
2008 
2009                 parents->idx[level-1] = idx;
2010                 if (level == PT_PAGE_TABLE_LEVEL)
2011                         break;
2012 
2013                 parents->parent[level-2] = sp;
2014         }
2015 
2016         return n;
2017 }
2018 
2019 static int mmu_pages_first(struct kvm_mmu_pages *pvec,
2020                            struct mmu_page_path *parents)
2021 {
2022         struct kvm_mmu_page *sp;
2023         int level;
2024 
2025         if (pvec->nr == 0)
2026                 return 0;
2027 
2028         WARN_ON(pvec->page[0].idx != INVALID_INDEX);
2029 
2030         sp = pvec->page[0].sp;
2031         level = sp->role.level;
2032         WARN_ON(level == PT_PAGE_TABLE_LEVEL);
2033 
2034         parents->parent[level-2] = sp;
2035 
2036         /* Also set up a sentinel.  Further entries in pvec are all
2037          * children of sp, so this element is never overwritten.
2038          */
2039         parents->parent[level-1] = NULL;
2040         return mmu_pages_next(pvec, parents, 0);
2041 }
2042 
2043 static void mmu_pages_clear_parents(struct mmu_page_path *parents)
2044 {
2045         struct kvm_mmu_page *sp;
2046         unsigned int level = 0;
2047 
2048         do {
2049                 unsigned int idx = parents->idx[level];
2050                 sp = parents->parent[level];
2051                 if (!sp)
2052                         return;
2053 
2054                 WARN_ON(idx == INVALID_INDEX);
2055                 clear_unsync_child_bit(sp, idx);
2056                 level++;
2057         } while (!sp->unsync_children);
2058 }
2059 
2060 static void mmu_sync_children(struct kvm_vcpu *vcpu,
2061                               struct kvm_mmu_page *parent)
2062 {
2063         int i;
2064         struct kvm_mmu_page *sp;
2065         struct mmu_page_path parents;
2066         struct kvm_mmu_pages pages;
2067         LIST_HEAD(invalid_list);
2068         bool flush = false;
2069 
2070         while (mmu_unsync_walk(parent, &pages)) {
2071                 bool protected = false;
2072 
2073                 for_each_sp(pages, sp, parents, i)
2074                         protected |= rmap_write_protect(vcpu, sp->gfn);
2075 
2076                 if (protected) {
2077                         kvm_flush_remote_tlbs(vcpu->kvm);
2078                         flush = false;
2079                 }
2080 
2081                 for_each_sp(pages, sp, parents, i) {
2082                         flush |= kvm_sync_page(vcpu, sp, &invalid_list);
2083                         mmu_pages_clear_parents(&parents);
2084                 }
2085                 if (need_resched() || spin_needbreak(&vcpu->kvm->mmu_lock)) {
2086                         kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush);
2087                         cond_resched_lock(&vcpu->kvm->mmu_lock);
2088                         flush = false;
2089                 }
2090         }
2091 
2092         kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush);
2093 }
2094 
2095 static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp)
2096 {
2097         atomic_set(&sp->write_flooding_count,  0);
2098 }
2099 
2100 static void clear_sp_write_flooding_count(u64 *spte)
2101 {
2102         struct kvm_mmu_page *sp =  page_header(__pa(spte));
2103 
2104         __clear_sp_write_flooding_count(sp);
2105 }
2106 
2107 static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
2108                                              gfn_t gfn,
2109                                              gva_t gaddr,
2110                                              unsigned level,
2111                                              int direct,
2112                                              unsigned access)
2113 {
2114         union kvm_mmu_page_role role;
2115         unsigned quadrant;
2116         struct kvm_mmu_page *sp;
2117         bool need_sync = false;
2118         bool flush = false;
2119         LIST_HEAD(invalid_list);
2120 
2121         role = vcpu->arch.mmu.base_role;
2122         role.level = level;
2123         role.direct = direct;
2124         if (role.direct)
2125                 role.cr4_pae = 0;
2126         role.access = access;
2127         if (!vcpu->arch.mmu.direct_map
2128             && vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
2129                 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
2130                 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
2131                 role.quadrant = quadrant;
2132         }
2133         for_each_gfn_valid_sp(vcpu->kvm, sp, gfn) {
2134                 if (!need_sync && sp->unsync)
2135                         need_sync = true;
2136 
2137                 if (sp->role.word != role.word)
2138                         continue;
2139 
2140                 if (sp->unsync) {
2141                         /* The page is good, but __kvm_sync_page might still end
2142                          * up zapping it.  If so, break in order to rebuild it.
2143                          */
2144                         if (!__kvm_sync_page(vcpu, sp, &invalid_list))
2145                                 break;
2146 
2147                         WARN_ON(!list_empty(&invalid_list));
2148                         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2149                 }
2150 
2151                 if (sp->unsync_children)
2152                         kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
2153 
2154                 __clear_sp_write_flooding_count(sp);
2155                 trace_kvm_mmu_get_page(sp, false);
2156                 return sp;
2157         }
2158 
2159         ++vcpu->kvm->stat.mmu_cache_miss;
2160 
2161         sp = kvm_mmu_alloc_page(vcpu, direct);
2162 
2163         sp->gfn = gfn;
2164         sp->role = role;
2165         hlist_add_head(&sp->hash_link,
2166                 &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]);
2167         if (!direct) {
2168                 /*
2169                  * we should do write protection before syncing pages
2170                  * otherwise the content of the synced shadow page may
2171                  * be inconsistent with guest page table.
2172                  */
2173                 account_shadowed(vcpu->kvm, sp);
2174                 if (level == PT_PAGE_TABLE_LEVEL &&
2175                       rmap_write_protect(vcpu, gfn))
2176                         kvm_flush_remote_tlbs(vcpu->kvm);
2177 
2178                 if (level > PT_PAGE_TABLE_LEVEL && need_sync)
2179                         flush |= kvm_sync_pages(vcpu, gfn, &invalid_list);
2180         }
2181         sp->mmu_valid_gen = vcpu->kvm->arch.mmu_valid_gen;
2182         clear_page(sp->spt);
2183         trace_kvm_mmu_get_page(sp, true);
2184 
2185         kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush);
2186         return sp;
2187 }
2188 
2189 static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
2190                              struct kvm_vcpu *vcpu, u64 addr)
2191 {
2192         iterator->addr = addr;
2193         iterator->shadow_addr = vcpu->arch.mmu.root_hpa;
2194         iterator->level = vcpu->arch.mmu.shadow_root_level;
2195 
2196         if (iterator->level == PT64_ROOT_LEVEL &&
2197             vcpu->arch.mmu.root_level < PT64_ROOT_LEVEL &&
2198             !vcpu->arch.mmu.direct_map)
2199                 --iterator->level;
2200 
2201         if (iterator->level == PT32E_ROOT_LEVEL) {
2202                 iterator->shadow_addr
2203                         = vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
2204                 iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
2205                 --iterator->level;
2206                 if (!iterator->shadow_addr)
2207                         iterator->level = 0;
2208         }
2209 }
2210 
2211 static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
2212 {
2213         if (iterator->level < PT_PAGE_TABLE_LEVEL)
2214                 return false;
2215 
2216         iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
2217         iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
2218         return true;
2219 }
2220 
2221 static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator,
2222                                u64 spte)
2223 {
2224         if (is_last_spte(spte, iterator->level)) {
2225                 iterator->level = 0;
2226                 return;
2227         }
2228 
2229         iterator->shadow_addr = spte & PT64_BASE_ADDR_MASK;
2230         --iterator->level;
2231 }
2232 
2233 static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
2234 {
2235         return __shadow_walk_next(iterator, *iterator->sptep);
2236 }
2237 
2238 static void link_shadow_page(struct kvm_vcpu *vcpu, u64 *sptep,
2239                              struct kvm_mmu_page *sp)
2240 {
2241         u64 spte;
2242 
2243         BUILD_BUG_ON(VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK);
2244 
2245         spte = __pa(sp->spt) | shadow_present_mask | PT_WRITABLE_MASK |
2246                shadow_user_mask | shadow_x_mask | shadow_accessed_mask;
2247 
2248         mmu_spte_set(sptep, spte);
2249 
2250         mmu_page_add_parent_pte(vcpu, sp, sptep);
2251 
2252         if (sp->unsync_children || sp->unsync)
2253                 mark_unsync(sptep);
2254 }
2255 
2256 static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2257                                    unsigned direct_access)
2258 {
2259         if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
2260                 struct kvm_mmu_page *child;
2261 
2262                 /*
2263                  * For the direct sp, if the guest pte's dirty bit
2264                  * changed form clean to dirty, it will corrupt the
2265                  * sp's access: allow writable in the read-only sp,
2266                  * so we should update the spte at this point to get
2267                  * a new sp with the correct access.
2268                  */
2269                 child = page_header(*sptep & PT64_BASE_ADDR_MASK);
2270                 if (child->role.access == direct_access)
2271                         return;
2272 
2273                 drop_parent_pte(child, sptep);
2274                 kvm_flush_remote_tlbs(vcpu->kvm);
2275         }
2276 }
2277 
2278 static bool mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
2279                              u64 *spte)
2280 {
2281         u64 pte;
2282         struct kvm_mmu_page *child;
2283 
2284         pte = *spte;
2285         if (is_shadow_present_pte(pte)) {
2286                 if (is_last_spte(pte, sp->role.level)) {
2287                         drop_spte(kvm, spte);
2288                         if (is_large_pte(pte))
2289                                 --kvm->stat.lpages;
2290                 } else {
2291                         child = page_header(pte & PT64_BASE_ADDR_MASK);
2292                         drop_parent_pte(child, spte);
2293                 }
2294                 return true;
2295         }
2296 
2297         if (is_mmio_spte(pte))
2298                 mmu_spte_clear_no_track(spte);
2299 
2300         return false;
2301 }
2302 
2303 static void kvm_mmu_page_unlink_children(struct kvm *kvm,
2304                                          struct kvm_mmu_page *sp)
2305 {
2306         unsigned i;
2307 
2308         for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
2309                 mmu_page_zap_pte(kvm, sp, sp->spt + i);
2310 }
2311 
2312 static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
2313 {
2314         u64 *sptep;
2315         struct rmap_iterator iter;
2316 
2317         while ((sptep = rmap_get_first(&sp->parent_ptes, &iter)))
2318                 drop_parent_pte(sp, sptep);
2319 }
2320 
2321 static int mmu_zap_unsync_children(struct kvm *kvm,
2322                                    struct kvm_mmu_page *parent,
2323                                    struct list_head *invalid_list)
2324 {
2325         int i, zapped = 0;
2326         struct mmu_page_path parents;
2327         struct kvm_mmu_pages pages;
2328 
2329         if (parent->role.level == PT_PAGE_TABLE_LEVEL)
2330                 return 0;
2331 
2332         while (mmu_unsync_walk(parent, &pages)) {
2333                 struct kvm_mmu_page *sp;
2334 
2335                 for_each_sp(pages, sp, parents, i) {
2336                         kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
2337                         mmu_pages_clear_parents(&parents);
2338                         zapped++;
2339                 }
2340         }
2341 
2342         return zapped;
2343 }
2344 
2345 static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
2346                                     struct list_head *invalid_list)
2347 {
2348         int ret;
2349 
2350         trace_kvm_mmu_prepare_zap_page(sp);
2351         ++kvm->stat.mmu_shadow_zapped;
2352         ret = mmu_zap_unsync_children(kvm, sp, invalid_list);
2353         kvm_mmu_page_unlink_children(kvm, sp);
2354         kvm_mmu_unlink_parents(kvm, sp);
2355 
2356         if (!sp->role.invalid && !sp->role.direct)
2357                 unaccount_shadowed(kvm, sp);
2358 
2359         if (sp->unsync)
2360                 kvm_unlink_unsync_page(kvm, sp);
2361         if (!sp->root_count) {
2362                 /* Count self */
2363                 ret++;
2364                 list_move(&sp->link, invalid_list);
2365                 kvm_mod_used_mmu_pages(kvm, -1);
2366         } else {
2367                 list_move(&sp->link, &kvm->arch.active_mmu_pages);
2368 
2369                 /*
2370                  * The obsolete pages can not be used on any vcpus.
2371                  * See the comments in kvm_mmu_invalidate_zap_all_pages().
2372                  */
2373                 if (!sp->role.invalid && !is_obsolete_sp(kvm, sp))
2374                         kvm_reload_remote_mmus(kvm);
2375         }
2376 
2377         sp->role.invalid = 1;
2378         return ret;
2379 }
2380 
2381 static void kvm_mmu_commit_zap_page(struct kvm *kvm,
2382                                     struct list_head *invalid_list)
2383 {
2384         struct kvm_mmu_page *sp, *nsp;
2385 
2386         if (list_empty(invalid_list))
2387                 return;
2388 
2389         /*
2390          * We need to make sure everyone sees our modifications to
2391          * the page tables and see changes to vcpu->mode here. The barrier
2392          * in the kvm_flush_remote_tlbs() achieves this. This pairs
2393          * with vcpu_enter_guest and walk_shadow_page_lockless_begin/end.
2394          *
2395          * In addition, kvm_flush_remote_tlbs waits for all vcpus to exit
2396          * guest mode and/or lockless shadow page table walks.
2397          */
2398         kvm_flush_remote_tlbs(kvm);
2399 
2400         list_for_each_entry_safe(sp, nsp, invalid_list, link) {
2401                 WARN_ON(!sp->role.invalid || sp->root_count);
2402                 kvm_mmu_free_page(sp);
2403         }
2404 }
2405 
2406 static bool prepare_zap_oldest_mmu_page(struct kvm *kvm,
2407                                         struct list_head *invalid_list)
2408 {
2409         struct kvm_mmu_page *sp;
2410 
2411         if (list_empty(&kvm->arch.active_mmu_pages))
2412                 return false;
2413 
2414         sp = list_last_entry(&kvm->arch.active_mmu_pages,
2415                              struct kvm_mmu_page, link);
2416         kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
2417 
2418         return true;
2419 }
2420 
2421 /*
2422  * Changing the number of mmu pages allocated to the vm
2423  * Note: if goal_nr_mmu_pages is too small, you will get dead lock
2424  */
2425 void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int goal_nr_mmu_pages)
2426 {
2427         LIST_HEAD(invalid_list);
2428 
2429         spin_lock(&kvm->mmu_lock);
2430 
2431         if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
2432                 /* Need to free some mmu pages to achieve the goal. */
2433                 while (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages)
2434                         if (!prepare_zap_oldest_mmu_page(kvm, &invalid_list))
2435                                 break;
2436 
2437                 kvm_mmu_commit_zap_page(kvm, &invalid_list);
2438                 goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
2439         }
2440 
2441         kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
2442 
2443         spin_unlock(&kvm->mmu_lock);
2444 }
2445 
2446 int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
2447 {
2448         struct kvm_mmu_page *sp;
2449         LIST_HEAD(invalid_list);
2450         int r;
2451 
2452         pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
2453         r = 0;
2454         spin_lock(&kvm->mmu_lock);
2455         for_each_gfn_indirect_valid_sp(kvm, sp, gfn) {
2456                 pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
2457                          sp->role.word);
2458                 r = 1;
2459                 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
2460         }
2461         kvm_mmu_commit_zap_page(kvm, &invalid_list);
2462         spin_unlock(&kvm->mmu_lock);
2463 
2464         return r;
2465 }
2466 EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page);
2467 
2468 static void kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
2469 {
2470         trace_kvm_mmu_unsync_page(sp);
2471         ++vcpu->kvm->stat.mmu_unsync;
2472         sp->unsync = 1;
2473 
2474         kvm_mmu_mark_parents_unsync(sp);
2475 }
2476 
2477 static bool mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
2478                                    bool can_unsync)
2479 {
2480         struct kvm_mmu_page *sp;
2481 
2482         if (kvm_page_track_is_active(vcpu, gfn, KVM_PAGE_TRACK_WRITE))
2483                 return true;
2484 
2485         for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) {
2486                 if (!can_unsync)
2487                         return true;
2488 
2489                 if (sp->unsync)
2490                         continue;
2491 
2492                 WARN_ON(sp->role.level != PT_PAGE_TABLE_LEVEL);
2493                 kvm_unsync_page(vcpu, sp);
2494         }
2495 
2496         return false;
2497 }
2498 
2499 static bool kvm_is_mmio_pfn(kvm_pfn_t pfn)
2500 {
2501         if (pfn_valid(pfn))
2502                 return !is_zero_pfn(pfn) && PageReserved(pfn_to_page(pfn));
2503 
2504         return true;
2505 }
2506 
2507 static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2508                     unsigned pte_access, int level,
2509                     gfn_t gfn, kvm_pfn_t pfn, bool speculative,
2510                     bool can_unsync, bool host_writable)
2511 {
2512         u64 spte = 0;
2513         int ret = 0;
2514 
2515         if (set_mmio_spte(vcpu, sptep, gfn, pfn, pte_access))
2516                 return 0;
2517 
2518         /*
2519          * For the EPT case, shadow_present_mask is 0 if hardware
2520          * supports exec-only page table entries.  In that case,
2521          * ACC_USER_MASK and shadow_user_mask are used to represent
2522          * read access.  See FNAME(gpte_access) in paging_tmpl.h.
2523          */
2524         spte |= shadow_present_mask;
2525         if (!speculative)
2526                 spte |= shadow_accessed_mask;
2527 
2528         if (pte_access & ACC_EXEC_MASK)
2529                 spte |= shadow_x_mask;
2530         else
2531                 spte |= shadow_nx_mask;
2532 
2533         if (pte_access & ACC_USER_MASK)
2534                 spte |= shadow_user_mask;
2535 
2536         if (level > PT_PAGE_TABLE_LEVEL)
2537                 spte |= PT_PAGE_SIZE_MASK;
2538         if (tdp_enabled)
2539                 spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
2540                         kvm_is_mmio_pfn(pfn));
2541 
2542         if (host_writable)
2543                 spte |= SPTE_HOST_WRITEABLE;
2544         else
2545                 pte_access &= ~ACC_WRITE_MASK;
2546 
2547         spte |= (u64)pfn << PAGE_SHIFT;
2548 
2549         if (pte_access & ACC_WRITE_MASK) {
2550 
2551                 /*
2552                  * Other vcpu creates new sp in the window between
2553                  * mapping_level() and acquiring mmu-lock. We can
2554                  * allow guest to retry the access, the mapping can
2555                  * be fixed if guest refault.
2556                  */
2557                 if (level > PT_PAGE_TABLE_LEVEL &&
2558                     mmu_gfn_lpage_is_disallowed(vcpu, gfn, level))
2559                         goto done;
2560 
2561                 spte |= PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE;
2562 
2563                 /*
2564                  * Optimization: for pte sync, if spte was writable the hash
2565                  * lookup is unnecessary (and expensive). Write protection
2566                  * is responsibility of mmu_get_page / kvm_sync_page.
2567                  * Same reasoning can be applied to dirty page accounting.
2568                  */
2569                 if (!can_unsync && is_writable_pte(*sptep))
2570                         goto set_pte;
2571 
2572                 if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
2573                         pgprintk("%s: found shadow page for %llx, marking ro\n",
2574                                  __func__, gfn);
2575                         ret = 1;
2576                         pte_access &= ~ACC_WRITE_MASK;
2577                         spte &= ~(PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE);
2578                 }
2579         }
2580 
2581         if (pte_access & ACC_WRITE_MASK) {
2582                 kvm_vcpu_mark_page_dirty(vcpu, gfn);
2583                 spte |= shadow_dirty_mask;
2584         }
2585 
2586 set_pte:
2587         if (mmu_spte_update(sptep, spte))
2588                 kvm_flush_remote_tlbs(vcpu->kvm);
2589 done:
2590         return ret;
2591 }
2592 
2593 static bool mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, unsigned pte_access,
2594                          int write_fault, int level, gfn_t gfn, kvm_pfn_t pfn,
2595                          bool speculative, bool host_writable)
2596 {
2597         int was_rmapped = 0;
2598         int rmap_count;
2599         bool emulate = false;
2600 
2601         pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__,
2602                  *sptep, write_fault, gfn);
2603 
2604         if (is_shadow_present_pte(*sptep)) {
2605                 /*
2606                  * If we overwrite a PTE page pointer with a 2MB PMD, unlink
2607                  * the parent of the now unreachable PTE.
2608                  */
2609                 if (level > PT_PAGE_TABLE_LEVEL &&
2610                     !is_large_pte(*sptep)) {
2611                         struct kvm_mmu_page *child;
2612                         u64 pte = *sptep;
2613 
2614                         child = page_header(pte & PT64_BASE_ADDR_MASK);
2615                         drop_parent_pte(child, sptep);
2616                         kvm_flush_remote_tlbs(vcpu->kvm);
2617                 } else if (pfn != spte_to_pfn(*sptep)) {
2618                         pgprintk("hfn old %llx new %llx\n",
2619                                  spte_to_pfn(*sptep), pfn);
2620                         drop_spte(vcpu->kvm, sptep);
2621                         kvm_flush_remote_tlbs(vcpu->kvm);
2622                 } else
2623                         was_rmapped = 1;
2624         }
2625 
2626         if (set_spte(vcpu, sptep, pte_access, level, gfn, pfn, speculative,
2627               true, host_writable)) {
2628                 if (write_fault)
2629                         emulate = true;
2630                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2631         }
2632 
2633         if (unlikely(is_mmio_spte(*sptep)))
2634                 emulate = true;
2635 
2636         pgprintk("%s: setting spte %llx\n", __func__, *sptep);
2637         pgprintk("instantiating %s PTE (%s) at %llx (%llx) addr %p\n",
2638                  is_large_pte(*sptep)? "2MB" : "4kB",
2639                  *sptep & PT_PRESENT_MASK ?"RW":"R", gfn,
2640                  *sptep, sptep);
2641         if (!was_rmapped && is_large_pte(*sptep))
2642                 ++vcpu->kvm->stat.lpages;
2643 
2644         if (is_shadow_present_pte(*sptep)) {
2645                 if (!was_rmapped) {
2646                         rmap_count = rmap_add(vcpu, sptep, gfn);
2647                         if (rmap_count > RMAP_RECYCLE_THRESHOLD)
2648                                 rmap_recycle(vcpu, sptep, gfn);
2649                 }
2650         }
2651 
2652         kvm_release_pfn_clean(pfn);
2653 
2654         return emulate;
2655 }
2656 
2657 static kvm_pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
2658                                      bool no_dirty_log)
2659 {
2660         struct kvm_memory_slot *slot;
2661 
2662         slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log);
2663         if (!slot)
2664                 return KVM_PFN_ERR_FAULT;
2665 
2666         return gfn_to_pfn_memslot_atomic(slot, gfn);
2667 }
2668 
2669 static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
2670                                     struct kvm_mmu_page *sp,
2671                                     u64 *start, u64 *end)
2672 {
2673         struct page *pages[PTE_PREFETCH_NUM];
2674         struct kvm_memory_slot *slot;
2675         unsigned access = sp->role.access;
2676         int i, ret;
2677         gfn_t gfn;
2678 
2679         gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
2680         slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK);
2681         if (!slot)
2682                 return -1;
2683 
2684         ret = gfn_to_page_many_atomic(slot, gfn, pages, end - start);
2685         if (ret <= 0)
2686                 return -1;
2687 
2688         for (i = 0; i < ret; i++, gfn++, start++)
2689                 mmu_set_spte(vcpu, start, access, 0, sp->role.level, gfn,
2690                              page_to_pfn(pages[i]), true, true);
2691 
2692         return 0;
2693 }
2694 
2695 static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
2696                                   struct kvm_mmu_page *sp, u64 *sptep)
2697 {
2698         u64 *spte, *start = NULL;
2699         int i;
2700 
2701         WARN_ON(!sp->role.direct);
2702 
2703         i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
2704         spte = sp->spt + i;
2705 
2706         for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
2707                 if (is_shadow_present_pte(*spte) || spte == sptep) {
2708                         if (!start)
2709                                 continue;
2710                         if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
2711                                 break;
2712                         start = NULL;
2713                 } else if (!start)
2714                         start = spte;
2715         }
2716 }
2717 
2718 static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
2719 {
2720         struct kvm_mmu_page *sp;
2721 
2722         /*
2723          * Since it's no accessed bit on EPT, it's no way to
2724          * distinguish between actually accessed translations
2725          * and prefetched, so disable pte prefetch if EPT is
2726          * enabled.
2727          */
2728         if (!shadow_accessed_mask)
2729                 return;
2730 
2731         sp = page_header(__pa(sptep));
2732         if (sp->role.level > PT_PAGE_TABLE_LEVEL)
2733                 return;
2734 
2735         __direct_pte_prefetch(vcpu, sp, sptep);
2736 }
2737 
2738 static int __direct_map(struct kvm_vcpu *vcpu, int write, int map_writable,
2739                         int level, gfn_t gfn, kvm_pfn_t pfn, bool prefault)
2740 {
2741         struct kvm_shadow_walk_iterator iterator;
2742         struct kvm_mmu_page *sp;
2743         int emulate = 0;
2744         gfn_t pseudo_gfn;
2745 
2746         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
2747                 return 0;
2748 
2749         for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
2750                 if (iterator.level == level) {
2751                         emulate = mmu_set_spte(vcpu, iterator.sptep, ACC_ALL,
2752                                                write, level, gfn, pfn, prefault,
2753                                                map_writable);
2754                         direct_pte_prefetch(vcpu, iterator.sptep);
2755                         ++vcpu->stat.pf_fixed;
2756                         break;
2757                 }
2758 
2759                 drop_large_spte(vcpu, iterator.sptep);
2760                 if (!is_shadow_present_pte(*iterator.sptep)) {
2761                         u64 base_addr = iterator.addr;
2762 
2763                         base_addr &= PT64_LVL_ADDR_MASK(iterator.level);
2764                         pseudo_gfn = base_addr >> PAGE_SHIFT;
2765                         sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
2766                                               iterator.level - 1, 1, ACC_ALL);
2767 
2768                         link_shadow_page(vcpu, iterator.sptep, sp);
2769                 }
2770         }
2771         return emulate;
2772 }
2773 
2774 static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk)
2775 {
2776         siginfo_t info;
2777 
2778         info.si_signo   = SIGBUS;
2779         info.si_errno   = 0;
2780         info.si_code    = BUS_MCEERR_AR;
2781         info.si_addr    = (void __user *)address;
2782         info.si_addr_lsb = PAGE_SHIFT;
2783 
2784         send_sig_info(SIGBUS, &info, tsk);
2785 }
2786 
2787 static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn)
2788 {
2789         /*
2790          * Do not cache the mmio info caused by writing the readonly gfn
2791          * into the spte otherwise read access on readonly gfn also can
2792          * caused mmio page fault and treat it as mmio access.
2793          * Return 1 to tell kvm to emulate it.
2794          */
2795         if (pfn == KVM_PFN_ERR_RO_FAULT)
2796                 return 1;
2797 
2798         if (pfn == KVM_PFN_ERR_HWPOISON) {
2799                 kvm_send_hwpoison_signal(kvm_vcpu_gfn_to_hva(vcpu, gfn), current);
2800                 return 0;
2801         }
2802 
2803         return -EFAULT;
2804 }
2805 
2806 static void transparent_hugepage_adjust(struct kvm_vcpu *vcpu,
2807                                         gfn_t *gfnp, kvm_pfn_t *pfnp,
2808                                         int *levelp)
2809 {
2810         kvm_pfn_t pfn = *pfnp;
2811         gfn_t gfn = *gfnp;
2812         int level = *levelp;
2813 
2814         /*
2815          * Check if it's a transparent hugepage. If this would be an
2816          * hugetlbfs page, level wouldn't be set to
2817          * PT_PAGE_TABLE_LEVEL and there would be no adjustment done
2818          * here.
2819          */
2820         if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn) &&
2821             level == PT_PAGE_TABLE_LEVEL &&
2822             PageTransCompoundMap(pfn_to_page(pfn)) &&
2823             !mmu_gfn_lpage_is_disallowed(vcpu, gfn, PT_DIRECTORY_LEVEL)) {
2824                 unsigned long mask;
2825                 /*
2826                  * mmu_notifier_retry was successful and we hold the
2827                  * mmu_lock here, so the pmd can't become splitting
2828                  * from under us, and in turn
2829                  * __split_huge_page_refcount() can't run from under
2830                  * us and we can safely transfer the refcount from
2831                  * PG_tail to PG_head as we switch the pfn to tail to
2832                  * head.
2833                  */
2834                 *levelp = level = PT_DIRECTORY_LEVEL;
2835                 mask = KVM_PAGES_PER_HPAGE(level) - 1;
2836                 VM_BUG_ON((gfn & mask) != (pfn & mask));
2837                 if (pfn & mask) {
2838                         gfn &= ~mask;
2839                         *gfnp = gfn;
2840                         kvm_release_pfn_clean(pfn);
2841                         pfn &= ~mask;
2842                         kvm_get_pfn(pfn);
2843                         *pfnp = pfn;
2844                 }
2845         }
2846 }
2847 
2848 static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn,
2849                                 kvm_pfn_t pfn, unsigned access, int *ret_val)
2850 {
2851         /* The pfn is invalid, report the error! */
2852         if (unlikely(is_error_pfn(pfn))) {
2853                 *ret_val = kvm_handle_bad_page(vcpu, gfn, pfn);
2854                 return true;
2855         }
2856 
2857         if (unlikely(is_noslot_pfn(pfn)))
2858                 vcpu_cache_mmio_info(vcpu, gva, gfn, access);
2859 
2860         return false;
2861 }
2862 
2863 static bool page_fault_can_be_fast(u32 error_code)
2864 {
2865         /*
2866          * Do not fix the mmio spte with invalid generation number which
2867          * need to be updated by slow page fault path.
2868          */
2869         if (unlikely(error_code & PFERR_RSVD_MASK))
2870                 return false;
2871 
2872         /*
2873          * #PF can be fast only if the shadow page table is present and it
2874          * is caused by write-protect, that means we just need change the
2875          * W bit of the spte which can be done out of mmu-lock.
2876          */
2877         if (!(error_code & PFERR_PRESENT_MASK) ||
2878               !(error_code & PFERR_WRITE_MASK))
2879                 return false;
2880 
2881         return true;
2882 }
2883 
2884 static bool
2885 fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
2886                         u64 *sptep, u64 spte)
2887 {
2888         gfn_t gfn;
2889 
2890         WARN_ON(!sp->role.direct);
2891 
2892         /*
2893          * The gfn of direct spte is stable since it is calculated
2894          * by sp->gfn.
2895          */
2896         gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt);
2897 
2898         /*
2899          * Theoretically we could also set dirty bit (and flush TLB) here in
2900          * order to eliminate unnecessary PML logging. See comments in
2901          * set_spte. But fast_page_fault is very unlikely to happen with PML
2902          * enabled, so we do not do this. This might result in the same GPA
2903          * to be logged in PML buffer again when the write really happens, and
2904          * eventually to be called by mark_page_dirty twice. But it's also no
2905          * harm. This also avoids the TLB flush needed after setting dirty bit
2906          * so non-PML cases won't be impacted.
2907          *
2908          * Compare with set_spte where instead shadow_dirty_mask is set.
2909          */
2910         if (cmpxchg64(sptep, spte, spte | PT_WRITABLE_MASK) == spte)
2911                 kvm_vcpu_mark_page_dirty(vcpu, gfn);
2912 
2913         return true;
2914 }
2915 
2916 /*
2917  * Return value:
2918  * - true: let the vcpu to access on the same address again.
2919  * - false: let the real page fault path to fix it.
2920  */
2921 static bool fast_page_fault(struct kvm_vcpu *vcpu, gva_t gva, int level,
2922                             u32 error_code)
2923 {
2924         struct kvm_shadow_walk_iterator iterator;
2925         struct kvm_mmu_page *sp;
2926         bool ret = false;
2927         u64 spte = 0ull;
2928 
2929         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
2930                 return false;
2931 
2932         if (!page_fault_can_be_fast(error_code))
2933                 return false;
2934 
2935         walk_shadow_page_lockless_begin(vcpu);
2936         for_each_shadow_entry_lockless(vcpu, gva, iterator, spte)
2937                 if (!is_shadow_present_pte(spte) || iterator.level < level)
2938                         break;
2939 
2940         /*
2941          * If the mapping has been changed, let the vcpu fault on the
2942          * same address again.
2943          */
2944         if (!is_shadow_present_pte(spte)) {
2945                 ret = true;
2946                 goto exit;
2947         }
2948 
2949         sp = page_header(__pa(iterator.sptep));
2950         if (!is_last_spte(spte, sp->role.level))
2951                 goto exit;
2952 
2953         /*
2954          * Check if it is a spurious fault caused by TLB lazily flushed.
2955          *
2956          * Need not check the access of upper level table entries since
2957          * they are always ACC_ALL.
2958          */
2959          if (is_writable_pte(spte)) {
2960                 ret = true;
2961                 goto exit;
2962         }
2963 
2964         /*
2965          * Currently, to simplify the code, only the spte write-protected
2966          * by dirty-log can be fast fixed.
2967          */
2968         if (!spte_is_locklessly_modifiable(spte))
2969                 goto exit;
2970 
2971         /*
2972          * Do not fix write-permission on the large spte since we only dirty
2973          * the first page into the dirty-bitmap in fast_pf_fix_direct_spte()
2974          * that means other pages are missed if its slot is dirty-logged.
2975          *
2976          * Instead, we let the slow page fault path create a normal spte to
2977          * fix the access.
2978          *
2979          * See the comments in kvm_arch_commit_memory_region().
2980          */
2981         if (sp->role.level > PT_PAGE_TABLE_LEVEL)
2982                 goto exit;
2983 
2984         /*
2985          * Currently, fast page fault only works for direct mapping since
2986          * the gfn is not stable for indirect shadow page.
2987          * See Documentation/virtual/kvm/locking.txt to get more detail.
2988          */
2989         ret = fast_pf_fix_direct_spte(vcpu, sp, iterator.sptep, spte);
2990 exit:
2991         trace_fast_page_fault(vcpu, gva, error_code, iterator.sptep,
2992                               spte, ret);
2993         walk_shadow_page_lockless_end(vcpu);
2994 
2995         return ret;
2996 }
2997 
2998 static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
2999                          gva_t gva, kvm_pfn_t *pfn, bool write, bool *writable);
3000 static void make_mmu_pages_available(struct kvm_vcpu *vcpu);
3001 
3002 static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, u32 error_code,
3003                          gfn_t gfn, bool prefault)
3004 {
3005         int r;
3006         int level;
3007         bool force_pt_level = false;
3008         kvm_pfn_t pfn;
3009         unsigned long mmu_seq;
3010         bool map_writable, write = error_code & PFERR_WRITE_MASK;
3011 
3012         level = mapping_level(vcpu, gfn, &force_pt_level);
3013         if (likely(!force_pt_level)) {
3014                 /*
3015                  * This path builds a PAE pagetable - so we can map
3016                  * 2mb pages at maximum. Therefore check if the level
3017                  * is larger than that.
3018                  */
3019                 if (level > PT_DIRECTORY_LEVEL)
3020                         level = PT_DIRECTORY_LEVEL;
3021 
3022                 gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
3023         }
3024 
3025         if (fast_page_fault(vcpu, v, level, error_code))
3026                 return 0;
3027 
3028         mmu_seq = vcpu->kvm->mmu_notifier_seq;
3029         smp_rmb();
3030 
3031         if (try_async_pf(vcpu, prefault, gfn, v, &pfn, write, &map_writable))
3032                 return 0;
3033 
3034         if (handle_abnormal_pfn(vcpu, v, gfn, pfn, ACC_ALL, &r))
3035                 return r;
3036 
3037         spin_lock(&vcpu->kvm->mmu_lock);
3038         if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
3039                 goto out_unlock;
3040         make_mmu_pages_available(vcpu);
3041         if (likely(!force_pt_level))
3042                 transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
3043         r = __direct_map(vcpu, write, map_writable, level, gfn, pfn, prefault);
3044         spin_unlock(&vcpu->kvm->mmu_lock);
3045 
3046         return r;
3047 
3048 out_unlock:
3049         spin_unlock(&vcpu->kvm->mmu_lock);
3050         kvm_release_pfn_clean(pfn);
3051         return 0;
3052 }
3053 
3054 
3055 static void mmu_free_roots(struct kvm_vcpu *vcpu)
3056 {
3057         int i;
3058         struct kvm_mmu_page *sp;
3059         LIST_HEAD(invalid_list);
3060 
3061         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
3062                 return;
3063 
3064         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL &&
3065             (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL ||
3066              vcpu->arch.mmu.direct_map)) {
3067                 hpa_t root = vcpu->arch.mmu.root_hpa;
3068 
3069                 spin_lock(&vcpu->kvm->mmu_lock);
3070                 sp = page_header(root);
3071                 --sp->root_count;
3072                 if (!sp->root_count && sp->role.invalid) {
3073                         kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
3074                         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
3075                 }
3076                 spin_unlock(&vcpu->kvm->mmu_lock);
3077                 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
3078                 return;
3079         }
3080 
3081         spin_lock(&vcpu->kvm->mmu_lock);
3082         for (i = 0; i < 4; ++i) {
3083                 hpa_t root = vcpu->arch.mmu.pae_root[i];
3084 
3085                 if (root) {
3086                         root &= PT64_BASE_ADDR_MASK;
3087                         sp = page_header(root);
3088                         --sp->root_count;
3089                         if (!sp->root_count && sp->role.invalid)
3090                                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
3091                                                          &invalid_list);
3092                 }
3093                 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
3094         }
3095         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
3096         spin_unlock(&vcpu->kvm->mmu_lock);
3097         vcpu->arch.mmu.root_hpa = INVALID_PAGE;
3098 }
3099 
3100 static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
3101 {
3102         int ret = 0;
3103 
3104         if (!kvm_is_visible_gfn(vcpu->kvm, root_gfn)) {
3105                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
3106                 ret = 1;
3107         }
3108 
3109         return ret;
3110 }
3111 
3112 static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
3113 {
3114         struct kvm_mmu_page *sp;
3115         unsigned i;
3116 
3117         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
3118                 spin_lock(&vcpu->kvm->mmu_lock);
3119                 make_mmu_pages_available(vcpu);
3120                 sp = kvm_mmu_get_page(vcpu, 0, 0, PT64_ROOT_LEVEL, 1, ACC_ALL);
3121                 ++sp->root_count;
3122                 spin_unlock(&vcpu->kvm->mmu_lock);
3123                 vcpu->arch.mmu.root_hpa = __pa(sp->spt);
3124         } else if (vcpu->arch.mmu.shadow_root_level == PT32E_ROOT_LEVEL) {
3125                 for (i = 0; i < 4; ++i) {
3126                         hpa_t root = vcpu->arch.mmu.pae_root[i];
3127 
3128                         MMU_WARN_ON(VALID_PAGE(root));
3129                         spin_lock(&vcpu->kvm->mmu_lock);
3130                         make_mmu_pages_available(vcpu);
3131                         sp = kvm_mmu_get_page(vcpu, i << (30 - PAGE_SHIFT),
3132                                         i << 30, PT32_ROOT_LEVEL, 1, ACC_ALL);
3133                         root = __pa(sp->spt);
3134                         ++sp->root_count;
3135                         spin_unlock(&vcpu->kvm->mmu_lock);
3136                         vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
3137                 }
3138                 vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
3139         } else
3140                 BUG();
3141 
3142         return 0;
3143 }
3144 
3145 static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
3146 {
3147         struct kvm_mmu_page *sp;
3148         u64 pdptr, pm_mask;
3149         gfn_t root_gfn;
3150         int i;
3151 
3152         root_gfn = vcpu->arch.mmu.get_cr3(vcpu) >> PAGE_SHIFT;
3153 
3154         if (mmu_check_root(vcpu, root_gfn))
3155                 return 1;
3156 
3157         /*
3158          * Do we shadow a long mode page table? If so we need to
3159          * write-protect the guests page table root.
3160          */
3161         if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
3162                 hpa_t root = vcpu->arch.mmu.root_hpa;
3163 
3164                 MMU_WARN_ON(VALID_PAGE(root));
3165 
3166                 spin_lock(&vcpu->kvm->mmu_lock);
3167                 make_mmu_pages_available(vcpu);
3168                 sp = kvm_mmu_get_page(vcpu, root_gfn, 0, PT64_ROOT_LEVEL,
3169                                       0, ACC_ALL);
3170                 root = __pa(sp->spt);
3171                 ++sp->root_count;
3172                 spin_unlock(&vcpu->kvm->mmu_lock);
3173                 vcpu->arch.mmu.root_hpa = root;
3174                 return 0;
3175         }
3176 
3177         /*
3178          * We shadow a 32 bit page table. This may be a legacy 2-level
3179          * or a PAE 3-level page table. In either case we need to be aware that
3180          * the shadow page table may be a PAE or a long mode page table.
3181          */
3182         pm_mask = PT_PRESENT_MASK;
3183         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL)
3184                 pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;
3185 
3186         for (i = 0; i < 4; ++i) {
3187                 hpa_t root = vcpu->arch.mmu.pae_root[i];
3188 
3189                 MMU_WARN_ON(VALID_PAGE(root));
3190                 if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
3191                         pdptr = vcpu->arch.mmu.get_pdptr(vcpu, i);
3192                         if (!(pdptr & PT_PRESENT_MASK)) {
3193                                 vcpu->arch.mmu.pae_root[i] = 0;
3194                                 continue;
3195                         }
3196                         root_gfn = pdptr >> PAGE_SHIFT;
3197                         if (mmu_check_root(vcpu, root_gfn))
3198                                 return 1;
3199                 }
3200                 spin_lock(&vcpu->kvm->mmu_lock);
3201                 make_mmu_pages_available(vcpu);
3202                 sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30, PT32_ROOT_LEVEL,
3203                                       0, ACC_ALL);
3204                 root = __pa(sp->spt);
3205                 ++sp->root_count;
3206                 spin_unlock(&vcpu->kvm->mmu_lock);
3207 
3208                 vcpu->arch.mmu.pae_root[i] = root | pm_mask;
3209         }
3210         vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
3211 
3212         /*
3213          * If we shadow a 32 bit page table with a long mode page
3214          * table we enter this path.
3215          */
3216         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
3217                 if (vcpu->arch.mmu.lm_root == NULL) {
3218                         /*
3219                          * The additional page necessary for this is only
3220                          * allocated on demand.
3221                          */
3222 
3223                         u64 *lm_root;
3224 
3225                         lm_root = (void*)get_zeroed_page(GFP_KERNEL);
3226                         if (lm_root == NULL)
3227                                 return 1;
3228 
3229                         lm_root[0] = __pa(vcpu->arch.mmu.pae_root) | pm_mask;
3230 
3231                         vcpu->arch.mmu.lm_root = lm_root;
3232                 }
3233 
3234                 vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.lm_root);
3235         }
3236 
3237         return 0;
3238 }
3239 
3240 static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
3241 {
3242         if (vcpu->arch.mmu.direct_map)
3243                 return mmu_alloc_direct_roots(vcpu);
3244         else
3245                 return mmu_alloc_shadow_roots(vcpu);
3246 }
3247 
3248 static void mmu_sync_roots(struct kvm_vcpu *vcpu)
3249 {
3250         int i;
3251         struct kvm_mmu_page *sp;
3252 
3253         if (vcpu->arch.mmu.direct_map)
3254                 return;
3255 
3256         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
3257                 return;
3258 
3259         vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
3260         kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
3261         if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
3262                 hpa_t root = vcpu->arch.mmu.root_hpa;
3263                 sp = page_header(root);
3264                 mmu_sync_children(vcpu, sp);
3265                 kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3266                 return;
3267         }
3268         for (i = 0; i < 4; ++i) {
3269                 hpa_t root = vcpu->arch.mmu.pae_root[i];
3270 
3271                 if (root && VALID_PAGE(root)) {
3272                         root &= PT64_BASE_ADDR_MASK;
3273                         sp = page_header(root);
3274                         mmu_sync_children(vcpu, sp);
3275                 }
3276         }
3277         kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3278 }
3279 
3280 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
3281 {
3282         spin_lock(&vcpu->kvm->mmu_lock);
3283         mmu_sync_roots(vcpu);
3284         spin_unlock(&vcpu->kvm->mmu_lock);
3285 }
3286 EXPORT_SYMBOL_GPL(kvm_mmu_sync_roots);
3287 
3288 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr,
3289                                   u32 access, struct x86_exception *exception)
3290 {
3291         if (exception)
3292                 exception->error_code = 0;
3293         return vaddr;
3294 }
3295 
3296 static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gva_t vaddr,
3297                                          u32 access,
3298                                          struct x86_exception *exception)
3299 {
3300         if (exception)
3301                 exception->error_code = 0;
3302         return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access, exception);
3303 }
3304 
3305 static bool
3306 __is_rsvd_bits_set(struct rsvd_bits_validate *rsvd_check, u64 pte, int level)
3307 {
3308         int bit7 = (pte >> 7) & 1, low6 = pte & 0x3f;
3309 
3310         return (pte & rsvd_check->rsvd_bits_mask[bit7][level-1]) |
3311                 ((rsvd_check->bad_mt_xwr & (1ull << low6)) != 0);
3312 }
3313 
3314 static bool is_rsvd_bits_set(struct kvm_mmu *mmu, u64 gpte, int level)
3315 {
3316         return __is_rsvd_bits_set(&mmu->guest_rsvd_check, gpte, level);
3317 }
3318 
3319 static bool is_shadow_zero_bits_set(struct kvm_mmu *mmu, u64 spte, int level)
3320 {
3321         return __is_rsvd_bits_set(&mmu->shadow_zero_check, spte, level);
3322 }
3323 
3324 static bool mmio_info_in_cache(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3325 {
3326         if (direct)
3327                 return vcpu_match_mmio_gpa(vcpu, addr);
3328 
3329         return vcpu_match_mmio_gva(vcpu, addr);
3330 }
3331 
3332 /* return true if reserved bit is detected on spte. */
3333 static bool
3334 walk_shadow_page_get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep)
3335 {
3336         struct kvm_shadow_walk_iterator iterator;
3337         u64 sptes[PT64_ROOT_LEVEL], spte = 0ull;
3338         int root, leaf;
3339         bool reserved = false;
3340 
3341         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
3342                 goto exit;
3343 
3344         walk_shadow_page_lockless_begin(vcpu);
3345 
3346         for (shadow_walk_init(&iterator, vcpu, addr),
3347                  leaf = root = iterator.level;
3348              shadow_walk_okay(&iterator);
3349              __shadow_walk_next(&iterator, spte)) {
3350                 spte = mmu_spte_get_lockless(iterator.sptep);
3351 
3352                 sptes[leaf - 1] = spte;
3353                 leaf--;
3354 
3355                 if (!is_shadow_present_pte(spte))
3356                         break;
3357 
3358                 reserved |= is_shadow_zero_bits_set(&vcpu->arch.mmu, spte,
3359                                                     iterator.level);
3360         }
3361 
3362         walk_shadow_page_lockless_end(vcpu);
3363 
3364         if (reserved) {
3365                 pr_err("%s: detect reserved bits on spte, addr 0x%llx, dump hierarchy:\n",
3366                        __func__, addr);
3367                 while (root > leaf) {
3368                         pr_err("------ spte 0x%llx level %d.\n",
3369                                sptes[root - 1], root);
3370                         root--;
3371                 }
3372         }
3373 exit:
3374         *sptep = spte;
3375         return reserved;
3376 }
3377 
3378 int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3379 {
3380         u64 spte;
3381         bool reserved;
3382 
3383         if (mmio_info_in_cache(vcpu, addr, direct))
3384                 return RET_MMIO_PF_EMULATE;
3385 
3386         reserved = walk_shadow_page_get_mmio_spte(vcpu, addr, &spte);
3387         if (WARN_ON(reserved))
3388                 return RET_MMIO_PF_BUG;
3389 
3390         if (is_mmio_spte(spte)) {
3391                 gfn_t gfn = get_mmio_spte_gfn(spte);
3392                 unsigned access = get_mmio_spte_access(spte);
3393 
3394                 if (!check_mmio_spte(vcpu, spte))
3395                         return RET_MMIO_PF_INVALID;
3396 
3397                 if (direct)
3398                         addr = 0;
3399 
3400                 trace_handle_mmio_page_fault(addr, gfn, access);
3401                 vcpu_cache_mmio_info(vcpu, addr, gfn, access);
3402                 return RET_MMIO_PF_EMULATE;
3403         }
3404 
3405         /*
3406          * If the page table is zapped by other cpus, let CPU fault again on
3407          * the address.
3408          */
3409         return RET_MMIO_PF_RETRY;
3410 }
3411 EXPORT_SYMBOL_GPL(handle_mmio_page_fault);
3412 
3413 static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu,
3414                                          u32 error_code, gfn_t gfn)
3415 {
3416         if (unlikely(error_code & PFERR_RSVD_MASK))
3417                 return false;
3418 
3419         if (!(error_code & PFERR_PRESENT_MASK) ||
3420               !(error_code & PFERR_WRITE_MASK))
3421                 return false;
3422 
3423         /*
3424          * guest is writing the page which is write tracked which can
3425          * not be fixed by page fault handler.
3426          */
3427         if (kvm_page_track_is_active(vcpu, gfn, KVM_PAGE_TRACK_WRITE))
3428                 return true;
3429 
3430         return false;
3431 }
3432 
3433 static void shadow_page_table_clear_flood(struct kvm_vcpu *vcpu, gva_t addr)
3434 {
3435         struct kvm_shadow_walk_iterator iterator;
3436         u64 spte;
3437 
3438         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
3439                 return;
3440 
3441         walk_shadow_page_lockless_begin(vcpu);
3442         for_each_shadow_entry_lockless(vcpu, addr, iterator, spte) {
3443                 clear_sp_write_flooding_count(iterator.sptep);
3444                 if (!is_shadow_present_pte(spte))
3445                         break;
3446         }
3447         walk_shadow_page_lockless_end(vcpu);
3448 }
3449 
3450 static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
3451                                 u32 error_code, bool prefault)
3452 {
3453         gfn_t gfn = gva >> PAGE_SHIFT;
3454         int r;
3455 
3456         pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
3457 
3458         if (page_fault_handle_page_track(vcpu, error_code, gfn))
3459                 return 1;
3460 
3461         r = mmu_topup_memory_caches(vcpu);
3462         if (r)
3463                 return r;
3464 
3465         MMU_WARN_ON(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
3466 
3467 
3468         return nonpaging_map(vcpu, gva & PAGE_MASK,
3469                              error_code, gfn, prefault);
3470 }
3471 
3472 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn)
3473 {
3474         struct kvm_arch_async_pf arch;
3475 
3476         arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
3477         arch.gfn = gfn;
3478         arch.direct_map = vcpu->arch.mmu.direct_map;
3479         arch.cr3 = vcpu->arch.mmu.get_cr3(vcpu);
3480 
3481         return kvm_setup_async_pf(vcpu, gva, kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch);
3482 }
3483 
3484 static bool can_do_async_pf(struct kvm_vcpu *vcpu)
3485 {
3486         if (unlikely(!lapic_in_kernel(vcpu) ||
3487                      kvm_event_needs_reinjection(vcpu)))
3488                 return false;
3489 
3490         return kvm_x86_ops->interrupt_allowed(vcpu);
3491 }
3492 
3493 static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
3494                          gva_t gva, kvm_pfn_t *pfn, bool write, bool *writable)
3495 {
3496         struct kvm_memory_slot *slot;
3497         bool async;
3498 
3499         slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
3500         async = false;
3501         *pfn = __gfn_to_pfn_memslot(slot, gfn, false, &async, write, writable);
3502         if (!async)
3503                 return false; /* *pfn has correct page already */
3504 
3505         if (!prefault && can_do_async_pf(vcpu)) {
3506                 trace_kvm_try_async_get_page(gva, gfn);
3507                 if (kvm_find_async_pf_gfn(vcpu, gfn)) {
3508                         trace_kvm_async_pf_doublefault(gva, gfn);
3509                         kvm_make_request(KVM_REQ_APF_HALT, vcpu);
3510                         return true;
3511                 } else if (kvm_arch_setup_async_pf(vcpu, gva, gfn))
3512                         return true;
3513         }
3514 
3515         *pfn = __gfn_to_pfn_memslot(slot, gfn, false, NULL, write, writable);
3516         return false;
3517 }
3518 
3519 static bool
3520 check_hugepage_cache_consistency(struct kvm_vcpu *vcpu, gfn_t gfn, int level)
3521 {
3522         int page_num = KVM_PAGES_PER_HPAGE(level);
3523 
3524         gfn &= ~(page_num - 1);
3525 
3526         return kvm_mtrr_check_gfn_range_consistency(vcpu, gfn, page_num);
3527 }
3528 
3529 static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code,
3530                           bool prefault)
3531 {
3532         kvm_pfn_t pfn;
3533         int r;
3534         int level;
3535         bool force_pt_level;
3536         gfn_t gfn = gpa >> PAGE_SHIFT;
3537         unsigned long mmu_seq;
3538         int write = error_code & PFERR_WRITE_MASK;
3539         bool map_writable;
3540 
3541         MMU_WARN_ON(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
3542 
3543         if (page_fault_handle_page_track(vcpu, error_code, gfn))
3544                 return 1;
3545 
3546         r = mmu_topup_memory_caches(vcpu);
3547         if (r)
3548                 return r;
3549 
3550         force_pt_level = !check_hugepage_cache_consistency(vcpu, gfn,
3551                                                            PT_DIRECTORY_LEVEL);
3552         level = mapping_level(vcpu, gfn, &force_pt_level);
3553         if (likely(!force_pt_level)) {
3554                 if (level > PT_DIRECTORY_LEVEL &&
3555                     !check_hugepage_cache_consistency(vcpu, gfn, level))
3556                         level = PT_DIRECTORY_LEVEL;
3557                 gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
3558         }
3559 
3560         if (fast_page_fault(vcpu, gpa, level, error_code))
3561                 return 0;
3562 
3563         mmu_seq = vcpu->kvm->mmu_notifier_seq;
3564         smp_rmb();
3565 
3566         if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable))
3567                 return 0;
3568 
3569         if (handle_abnormal_pfn(vcpu, 0, gfn, pfn, ACC_ALL, &r))
3570                 return r;
3571 
3572         spin_lock(&vcpu->kvm->mmu_lock);
3573         if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
3574                 goto out_unlock;
3575         make_mmu_pages_available(vcpu);
3576         if (likely(!force_pt_level))
3577                 transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
3578         r = __direct_map(vcpu, write, map_writable, level, gfn, pfn, prefault);
3579         spin_unlock(&vcpu->kvm->mmu_lock);
3580 
3581         return r;
3582 
3583 out_unlock:
3584         spin_unlock(&vcpu->kvm->mmu_lock);
3585         kvm_release_pfn_clean(pfn);
3586         return 0;
3587 }
3588 
3589 static void nonpaging_init_context(struct kvm_vcpu *vcpu,
3590                                    struct kvm_mmu *context)
3591 {
3592         context->page_fault = nonpaging_page_fault;
3593         context->gva_to_gpa = nonpaging_gva_to_gpa;
3594         context->sync_page = nonpaging_sync_page;
3595         context->invlpg = nonpaging_invlpg;
3596         context->update_pte = nonpaging_update_pte;
3597         context->root_level = 0;
3598         context->shadow_root_level = PT32E_ROOT_LEVEL;
3599         context->root_hpa = INVALID_PAGE;
3600         context->direct_map = true;
3601         context->nx = false;
3602 }
3603 
3604 void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu)
3605 {
3606         mmu_free_roots(vcpu);
3607 }
3608 
3609 static unsigned long get_cr3(struct kvm_vcpu *vcpu)
3610 {
3611         return kvm_read_cr3(vcpu);
3612 }
3613 
3614 static void inject_page_fault(struct kvm_vcpu *vcpu,
3615                               struct x86_exception *fault)
3616 {
3617         vcpu->arch.mmu.inject_page_fault(vcpu, fault);
3618 }
3619 
3620 static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
3621                            unsigned access, int *nr_present)
3622 {
3623         if (unlikely(is_mmio_spte(*sptep))) {
3624                 if (gfn != get_mmio_spte_gfn(*sptep)) {
3625                         mmu_spte_clear_no_track(sptep);
3626                         return true;
3627                 }
3628 
3629                 (*nr_present)++;
3630                 mark_mmio_spte(vcpu, sptep, gfn, access);
3631                 return true;
3632         }
3633 
3634         return false;
3635 }
3636 
3637 static inline bool is_last_gpte(struct kvm_mmu *mmu,
3638                                 unsigned level, unsigned gpte)
3639 {
3640         /*
3641          * PT_PAGE_TABLE_LEVEL always terminates.  The RHS has bit 7 set
3642          * iff level <= PT_PAGE_TABLE_LEVEL, which for our purpose means
3643          * level == PT_PAGE_TABLE_LEVEL; set PT_PAGE_SIZE_MASK in gpte then.
3644          */
3645         gpte |= level - PT_PAGE_TABLE_LEVEL - 1;
3646 
3647         /*
3648          * The RHS has bit 7 set iff level < mmu->last_nonleaf_level.
3649          * If it is clear, there are no large pages at this level, so clear
3650          * PT_PAGE_SIZE_MASK in gpte if that is the case.
3651          */
3652         gpte &= level - mmu->last_nonleaf_level;
3653 
3654         return gpte & PT_PAGE_SIZE_MASK;
3655 }
3656 
3657 #define PTTYPE_EPT 18 /* arbitrary */
3658 #define PTTYPE PTTYPE_EPT
3659 #include "paging_tmpl.h"
3660 #undef PTTYPE
3661 
3662 #define PTTYPE 64
3663 #include "paging_tmpl.h"
3664 #undef PTTYPE
3665 
3666 #define PTTYPE 32
3667 #include "paging_tmpl.h"
3668 #undef PTTYPE
3669 
3670 static void
3671 __reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
3672                         struct rsvd_bits_validate *rsvd_check,
3673                         int maxphyaddr, int level, bool nx, bool gbpages,
3674                         bool pse, bool amd)
3675 {
3676         u64 exb_bit_rsvd = 0;
3677         u64 gbpages_bit_rsvd = 0;
3678         u64 nonleaf_bit8_rsvd = 0;
3679 
3680         rsvd_check->bad_mt_xwr = 0;
3681 
3682         if (!nx)
3683                 exb_bit_rsvd = rsvd_bits(63, 63);
3684         if (!gbpages)
3685                 gbpages_bit_rsvd = rsvd_bits(7, 7);
3686 
3687         /*
3688          * Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for
3689          * leaf entries) on AMD CPUs only.
3690          */
3691         if (amd)
3692                 nonleaf_bit8_rsvd = rsvd_bits(8, 8);
3693 
3694         switch (level) {
3695         case PT32_ROOT_LEVEL:
3696                 /* no rsvd bits for 2 level 4K page table entries */
3697                 rsvd_check->rsvd_bits_mask[0][1] = 0;
3698                 rsvd_check->rsvd_bits_mask[0][0] = 0;
3699                 rsvd_check->rsvd_bits_mask[1][0] =
3700                         rsvd_check->rsvd_bits_mask[0][0];
3701 
3702                 if (!pse) {
3703                         rsvd_check->rsvd_bits_mask[1][1] = 0;
3704                         break;
3705                 }
3706 
3707                 if (is_cpuid_PSE36())
3708                         /* 36bits PSE 4MB page */
3709                         rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
3710                 else
3711                         /* 32 bits PSE 4MB page */
3712                         rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
3713                 break;
3714         case PT32E_ROOT_LEVEL:
3715                 rsvd_check->rsvd_bits_mask[0][2] =
3716                         rsvd_bits(maxphyaddr, 63) |
3717                         rsvd_bits(5, 8) | rsvd_bits(1, 2);      /* PDPTE */
3718                 rsvd_check->rsvd_bits_mask[0][1] = exb_bit_rsvd |
3719                         rsvd_bits(maxphyaddr, 62);      /* PDE */
3720                 rsvd_check->rsvd_bits_mask[0][0] = exb_bit_rsvd |
3721                         rsvd_bits(maxphyaddr, 62);      /* PTE */
3722                 rsvd_check->rsvd_bits_mask[1][1] = exb_bit_rsvd |
3723                         rsvd_bits(maxphyaddr, 62) |
3724                         rsvd_bits(13, 20);              /* large page */
3725                 rsvd_check->rsvd_bits_mask[1][0] =
3726                         rsvd_check->rsvd_bits_mask[0][0];
3727                 break;
3728         case PT64_ROOT_LEVEL:
3729                 rsvd_check->rsvd_bits_mask[0][3] = exb_bit_rsvd |
3730                         nonleaf_bit8_rsvd | rsvd_bits(7, 7) |
3731                         rsvd_bits(maxphyaddr, 51);
3732                 rsvd_check->rsvd_bits_mask[0][2] = exb_bit_rsvd |
3733                         nonleaf_bit8_rsvd | gbpages_bit_rsvd |
3734                         rsvd_bits(maxphyaddr, 51);
3735                 rsvd_check->rsvd_bits_mask[0][1] = exb_bit_rsvd |
3736                         rsvd_bits(maxphyaddr, 51);
3737                 rsvd_check->rsvd_bits_mask[0][0] = exb_bit_rsvd |
3738                         rsvd_bits(maxphyaddr, 51);
3739                 rsvd_check->rsvd_bits_mask[1][3] =
3740                         rsvd_check->rsvd_bits_mask[0][3];
3741                 rsvd_check->rsvd_bits_mask[1][2] = exb_bit_rsvd |
3742                         gbpages_bit_rsvd | rsvd_bits(maxphyaddr, 51) |
3743                         rsvd_bits(13, 29);
3744                 rsvd_check->rsvd_bits_mask[1][1] = exb_bit_rsvd |
3745                         rsvd_bits(maxphyaddr, 51) |
3746                         rsvd_bits(13, 20);              /* large page */
3747                 rsvd_check->rsvd_bits_mask[1][0] =
3748                         rsvd_check->rsvd_bits_mask[0][0];
3749                 break;
3750         }
3751 }
3752 
3753 static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
3754                                   struct kvm_mmu *context)
3755 {
3756         __reset_rsvds_bits_mask(vcpu, &context->guest_rsvd_check,
3757                                 cpuid_maxphyaddr(vcpu), context->root_level,
3758                                 context->nx, guest_cpuid_has_gbpages(vcpu),
3759                                 is_pse(vcpu), guest_cpuid_is_amd(vcpu));
3760 }
3761 
3762 static void
3763 __reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
3764                             int maxphyaddr, bool execonly)
3765 {
3766         u64 bad_mt_xwr;
3767 
3768         rsvd_check->rsvd_bits_mask[0][3] =
3769                 rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 7);
3770         rsvd_check->rsvd_bits_mask[0][2] =
3771                 rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 6);
3772         rsvd_check->rsvd_bits_mask[0][1] =
3773                 rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 6);
3774         rsvd_check->rsvd_bits_mask[0][0] = rsvd_bits(maxphyaddr, 51);
3775 
3776         /* large page */
3777         rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3];
3778         rsvd_check->rsvd_bits_mask[1][2] =
3779                 rsvd_bits(maxphyaddr, 51) | rsvd_bits(12, 29);
3780         rsvd_check->rsvd_bits_mask[1][1] =
3781                 rsvd_bits(maxphyaddr, 51) | rsvd_bits(12, 20);
3782         rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0];
3783 
3784         bad_mt_xwr = 0xFFull << (2 * 8);        /* bits 3..5 must not be 2 */
3785         bad_mt_xwr |= 0xFFull << (3 * 8);       /* bits 3..5 must not be 3 */
3786         bad_mt_xwr |= 0xFFull << (7 * 8);       /* bits 3..5 must not be 7 */
3787         bad_mt_xwr |= REPEAT_BYTE(1ull << 2);   /* bits 0..2 must not be 010 */
3788         bad_mt_xwr |= REPEAT_BYTE(1ull << 6);   /* bits 0..2 must not be 110 */
3789         if (!execonly) {
3790                 /* bits 0..2 must not be 100 unless VMX capabilities allow it */
3791                 bad_mt_xwr |= REPEAT_BYTE(1ull << 4);
3792         }
3793         rsvd_check->bad_mt_xwr = bad_mt_xwr;
3794 }
3795 
3796 static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu,
3797                 struct kvm_mmu *context, bool execonly)
3798 {
3799         __reset_rsvds_bits_mask_ept(&context->guest_rsvd_check,
3800                                     cpuid_maxphyaddr(vcpu), execonly);
3801 }
3802 
3803 /*
3804  * the page table on host is the shadow page table for the page
3805  * table in guest or amd nested guest, its mmu features completely
3806  * follow the features in guest.
3807  */
3808 void
3809 reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, struct kvm_mmu *context)
3810 {
3811         bool uses_nx = context->nx || context->base_role.smep_andnot_wp;
3812 
3813         /*
3814          * Passing "true" to the last argument is okay; it adds a check
3815          * on bit 8 of the SPTEs which KVM doesn't use anyway.
3816          */
3817         __reset_rsvds_bits_mask(vcpu, &context->shadow_zero_check,
3818                                 boot_cpu_data.x86_phys_bits,
3819                                 context->shadow_root_level, uses_nx,
3820                                 guest_cpuid_has_gbpages(vcpu), is_pse(vcpu),
3821                                 true);
3822 }
3823 EXPORT_SYMBOL_GPL(reset_shadow_zero_bits_mask);
3824 
3825 static inline bool boot_cpu_is_amd(void)
3826 {
3827         WARN_ON_ONCE(!tdp_enabled);
3828         return shadow_x_mask == 0;
3829 }
3830 
3831 /*
3832  * the direct page table on host, use as much mmu features as
3833  * possible, however, kvm currently does not do execution-protection.
3834  */
3835 static void
3836 reset_tdp_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
3837                                 struct kvm_mmu *context)
3838 {
3839         if (boot_cpu_is_amd())
3840                 __reset_rsvds_bits_mask(vcpu, &context->shadow_zero_check,
3841                                         boot_cpu_data.x86_phys_bits,
3842                                         context->shadow_root_level, false,
3843                                         boot_cpu_has(X86_FEATURE_GBPAGES),
3844                                         true, true);
3845         else
3846                 __reset_rsvds_bits_mask_ept(&context->shadow_zero_check,
3847                                             boot_cpu_data.x86_phys_bits,
3848                                             false);
3849 
3850 }
3851 
3852 /*
3853  * as the comments in reset_shadow_zero_bits_mask() except it
3854  * is the shadow page table for intel nested guest.
3855  */
3856 static void
3857 reset_ept_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
3858                                 struct kvm_mmu *context, bool execonly)
3859 {
3860         __reset_rsvds_bits_mask_ept(&context->shadow_zero_check,
3861                                     boot_cpu_data.x86_phys_bits, execonly);
3862 }
3863 
3864 static void update_permission_bitmask(struct kvm_vcpu *vcpu,
3865                                       struct kvm_mmu *mmu, bool ept)
3866 {
3867         unsigned bit, byte, pfec;
3868         u8 map;
3869         bool fault, x, w, u, wf, uf, ff, smapf, cr4_smap, cr4_smep, smap = 0;
3870 
3871         cr4_smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
3872         cr4_smap = kvm_read_cr4_bits(vcpu, X86_CR4_SMAP);
3873         for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {
3874                 pfec = byte << 1;
3875                 map = 0;
3876                 wf = pfec & PFERR_WRITE_MASK;
3877                 uf = pfec & PFERR_USER_MASK;
3878                 ff = pfec & PFERR_FETCH_MASK;
3879                 /*
3880                  * PFERR_RSVD_MASK bit is set in PFEC if the access is not
3881                  * subject to SMAP restrictions, and cleared otherwise. The
3882                  * bit is only meaningful if the SMAP bit is set in CR4.
3883                  */
3884                 smapf = !(pfec & PFERR_RSVD_MASK);
3885                 for (bit = 0; bit < 8; ++bit) {
3886                         x = bit & ACC_EXEC_MASK;
3887                         w = bit & ACC_WRITE_MASK;
3888                         u = bit & ACC_USER_MASK;
3889 
3890                         if (!ept) {
3891                                 /* Not really needed: !nx will cause pte.nx to fault */
3892                                 x |= !mmu->nx;
3893                                 /* Allow supervisor writes if !cr0.wp */
3894                                 w |= !is_write_protection(vcpu) && !uf;
3895                                 /* Disallow supervisor fetches of user code if cr4.smep */
3896                                 x &= !(cr4_smep && u && !uf);
3897 
3898                                 /*
3899                                  * SMAP:kernel-mode data accesses from user-mode
3900                                  * mappings should fault. A fault is considered
3901                                  * as a SMAP violation if all of the following
3902                                  * conditions are ture:
3903                                  *   - X86_CR4_SMAP is set in CR4
3904                                  *   - An user page is accessed
3905                                  *   - Page fault in kernel mode
3906                                  *   - if CPL = 3 or X86_EFLAGS_AC is clear
3907                                  *
3908                                  *   Here, we cover the first three conditions.
3909                                  *   The fourth is computed dynamically in
3910                                  *   permission_fault() and is in smapf.
3911                                  *
3912                                  *   Also, SMAP does not affect instruction
3913                                  *   fetches, add the !ff check here to make it
3914                                  *   clearer.
3915                                  */
3916                                 smap = cr4_smap && u && !uf && !ff;
3917                         }
3918 
3919                         fault = (ff && !x) || (uf && !u) || (wf && !w) ||
3920                                 (smapf && smap);
3921                         map |= fault << bit;
3922                 }
3923                 mmu->permissions[byte] = map;
3924         }
3925 }
3926 
3927 /*
3928 * PKU is an additional mechanism by which the paging controls access to
3929 * user-mode addresses based on the value in the PKRU register.  Protection
3930 * key violations are reported through a bit in the page fault error code.
3931 * Unlike other bits of the error code, the PK bit is not known at the
3932 * call site of e.g. gva_to_gpa; it must be computed directly in
3933 * permission_fault based on two bits of PKRU, on some machine state (CR4,
3934 * CR0, EFER, CPL), and on other bits of the error code and the page tables.
3935 *
3936 * In particular the following conditions come from the error code, the
3937 * page tables and the machine state:
3938 * - PK is always zero unless CR4.PKE=1 and EFER.LMA=1
3939 * - PK is always zero if RSVD=1 (reserved bit set) or F=1 (instruction fetch)
3940 * - PK is always zero if U=0 in the page tables
3941 * - PKRU.WD is ignored if CR0.WP=0 and the access is a supervisor access.
3942 *
3943 * The PKRU bitmask caches the result of these four conditions.  The error
3944 * code (minus the P bit) and the page table's U bit form an index into the
3945 * PKRU bitmask.  Two bits of the PKRU bitmask are then extracted and ANDed
3946 * with the two bits of the PKRU register corresponding to the protection key.
3947 * For the first three conditions above the bits will be 00, thus masking
3948 * away both AD and WD.  For all reads or if the last condition holds, WD
3949 * only will be masked away.
3950 */
3951 static void update_pkru_bitmask(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
3952                                 bool ept)
3953 {
3954         unsigned bit;
3955         bool wp;
3956 
3957         if (ept) {
3958                 mmu->pkru_mask = 0;
3959                 return;
3960         }
3961 
3962         /* PKEY is enabled only if CR4.PKE and EFER.LMA are both set. */
3963         if (!kvm_read_cr4_bits(vcpu, X86_CR4_PKE) || !is_long_mode(vcpu)) {
3964                 mmu->pkru_mask = 0;
3965                 return;
3966         }
3967 
3968         wp = is_write_protection(vcpu);
3969 
3970         for (bit = 0; bit < ARRAY_SIZE(mmu->permissions); ++bit) {
3971                 unsigned pfec, pkey_bits;
3972                 bool check_pkey, check_write, ff, uf, wf, pte_user;
3973 
3974                 pfec = bit << 1;
3975                 ff = pfec & PFERR_FETCH_MASK;
3976                 uf = pfec & PFERR_USER_MASK;
3977                 wf = pfec & PFERR_WRITE_MASK;
3978 
3979                 /* PFEC.RSVD is replaced by ACC_USER_MASK. */
3980                 pte_user = pfec & PFERR_RSVD_MASK;
3981 
3982                 /*
3983                  * Only need to check the access which is not an
3984                  * instruction fetch and is to a user page.
3985                  */
3986                 check_pkey = (!ff && pte_user);
3987                 /*
3988                  * write access is controlled by PKRU if it is a
3989                  * user access or CR0.WP = 1.
3990                  */
3991                 check_write = check_pkey && wf && (uf || wp);
3992 
3993                 /* PKRU.AD stops both read and write access. */
3994                 pkey_bits = !!check_pkey;
3995                 /* PKRU.WD stops write access. */
3996                 pkey_bits |= (!!check_write) << 1;
3997 
3998                 mmu->pkru_mask |= (pkey_bits & 3) << pfec;
3999         }
4000 }
4001 
4002 static void update_last_nonleaf_level(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
4003 {
4004         unsigned root_level = mmu->root_level;
4005 
4006         mmu->last_nonleaf_level = root_level;
4007         if (root_level == PT32_ROOT_LEVEL && is_pse(vcpu))
4008                 mmu->last_nonleaf_level++;
4009 }
4010 
4011 static void paging64_init_context_common(struct kvm_vcpu *vcpu,
4012                                          struct kvm_mmu *context,
4013                                          int level)
4014 {
4015         context->nx = is_nx(vcpu);
4016         context->root_level = level;
4017 
4018         reset_rsvds_bits_mask(vcpu, context);
4019         update_permission_bitmask(vcpu, context, false);
4020         update_pkru_bitmask(vcpu, context, false);
4021         update_last_nonleaf_level(vcpu, context);
4022 
4023         MMU_WARN_ON(!is_pae(vcpu));
4024         context->page_fault = paging64_page_fault;
4025         context->gva_to_gpa = paging64_gva_to_gpa;
4026         context->sync_page = paging64_sync_page;
4027         context->invlpg = paging64_invlpg;
4028         context->update_pte = paging64_update_pte;
4029         context->shadow_root_level = level;
4030         context->root_hpa = INVALID_PAGE;
4031         context->direct_map = false;
4032 }
4033 
4034 static void paging64_init_context(struct kvm_vcpu *vcpu,
4035                                   struct kvm_mmu *context)
4036 {
4037         paging64_init_context_common(vcpu, context, PT64_ROOT_LEVEL);
4038 }
4039 
4040 static void paging32_init_context(struct kvm_vcpu *vcpu,
4041                                   struct kvm_mmu *context)
4042 {
4043         context->nx = false;
4044         context->root_level = PT32_ROOT_LEVEL;
4045 
4046         reset_rsvds_bits_mask(vcpu, context);
4047         update_permission_bitmask(vcpu, context, false);
4048         update_pkru_bitmask(vcpu, context, false);
4049         update_last_nonleaf_level(vcpu, context);
4050 
4051         context->page_fault = paging32_page_fault;
4052         context->gva_to_gpa = paging32_gva_to_gpa;
4053         context->sync_page = paging32_sync_page;
4054         context->invlpg = paging32_invlpg;
4055         context->update_pte = paging32_update_pte;
4056         context->shadow_root_level = PT32E_ROOT_LEVEL;
4057         context->root_hpa = INVALID_PAGE;
4058         context->direct_map = false;
4059 }
4060 
4061 static void paging32E_init_context(struct kvm_vcpu *vcpu,
4062                                    struct kvm_mmu *context)
4063 {
4064         paging64_init_context_common(vcpu, context, PT32E_ROOT_LEVEL);
4065 }
4066 
4067 static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
4068 {
4069         struct kvm_mmu *context = &vcpu->arch.mmu;
4070 
4071         context->base_role.word = 0;
4072         context->base_role.smm = is_smm(vcpu);
4073         context->page_fault = tdp_page_fault;
4074         context->sync_page = nonpaging_sync_page;
4075         context->invlpg = nonpaging_invlpg;
4076         context->update_pte = nonpaging_update_pte;
4077         context->shadow_root_level = kvm_x86_ops->get_tdp_level();
4078         context->root_hpa = INVALID_PAGE;
4079         context->direct_map = true;
4080         context->set_cr3 = kvm_x86_ops->set_tdp_cr3;
4081         context->get_cr3 = get_cr3;
4082         context->get_pdptr = kvm_pdptr_read;
4083         context->inject_page_fault = kvm_inject_page_fault;
4084 
4085         if (!is_paging(vcpu)) {
4086                 context->nx = false;
4087                 context->gva_to_gpa = nonpaging_gva_to_gpa;
4088                 context->root_level = 0;
4089         } else if (is_long_mode(vcpu)) {
4090                 context->nx = is_nx(vcpu);
4091                 context->root_level = PT64_ROOT_LEVEL;
4092                 reset_rsvds_bits_mask(vcpu, context);
4093                 context->gva_to_gpa = paging64_gva_to_gpa;
4094         } else if (is_pae(vcpu)) {
4095                 context->nx = is_nx(vcpu);
4096                 context->root_level = PT32E_ROOT_LEVEL;
4097                 reset_rsvds_bits_mask(vcpu, context);
4098                 context->gva_to_gpa = paging64_gva_to_gpa;
4099         } else {
4100                 context->nx = false;
4101                 context->root_level = PT32_ROOT_LEVEL;
4102                 reset_rsvds_bits_mask(vcpu, context);
4103                 context->gva_to_gpa = paging32_gva_to_gpa;
4104         }
4105 
4106         update_permission_bitmask(vcpu, context, false);
4107         update_pkru_bitmask(vcpu, context, false);
4108         update_last_nonleaf_level(vcpu, context);
4109         reset_tdp_shadow_zero_bits_mask(vcpu, context);
4110 }
4111 
4112 void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu)
4113 {
4114         bool smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
4115         bool smap = kvm_read_cr4_bits(vcpu, X86_CR4_SMAP);
4116         struct kvm_mmu *context = &vcpu->arch.mmu;
4117 
4118         MMU_WARN_ON(VALID_PAGE(context->root_hpa));
4119 
4120         if (!is_paging(vcpu))
4121                 nonpaging_init_context(vcpu, context);
4122         else if (is_long_mode(vcpu))
4123                 paging64_init_context(vcpu, context);
4124         else if (is_pae(vcpu))
4125                 paging32E_init_context(vcpu, context);
4126         else
4127                 paging32_init_context(vcpu, context);
4128 
4129         context->base_role.nxe = is_nx(vcpu);
4130         context->base_role.cr4_pae = !!is_pae(vcpu);
4131         context->base_role.cr0_wp  = is_write_protection(vcpu);
4132         context->base_role.smep_andnot_wp
4133                 = smep && !is_write_protection(vcpu);
4134         context->base_role.smap_andnot_wp
4135                 = smap && !is_write_protection(vcpu);
4136         context->base_role.smm = is_smm(vcpu);
4137         reset_shadow_zero_bits_mask(vcpu, context);
4138 }
4139 EXPORT_SYMBOL_GPL(kvm_init_shadow_mmu);
4140 
4141 void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly)
4142 {
4143         struct kvm_mmu *context = &vcpu->arch.mmu;
4144 
4145         MMU_WARN_ON(VALID_PAGE(context->root_hpa));
4146 
4147         context->shadow_root_level = kvm_x86_ops->get_tdp_level();
4148 
4149         context->nx = true;
4150         context->page_fault = ept_page_fault;
4151         context->gva_to_gpa = ept_gva_to_gpa;
4152         context->sync_page = ept_sync_page;
4153         context->invlpg = ept_invlpg;
4154         context->update_pte = ept_update_pte;
4155         context->root_level = context->shadow_root_level;
4156         context->root_hpa = INVALID_PAGE;
4157         context->direct_map = false;
4158 
4159         update_permission_bitmask(vcpu, context, true);
4160         update_pkru_bitmask(vcpu, context, true);
4161         reset_rsvds_bits_mask_ept(vcpu, context, execonly);
4162         reset_ept_shadow_zero_bits_mask(vcpu, context, execonly);
4163 }
4164 EXPORT_SYMBOL_GPL(kvm_init_shadow_ept_mmu);
4165 
4166 static void init_kvm_softmmu(struct kvm_vcpu *vcpu)
4167 {
4168         struct kvm_mmu *context = &vcpu->arch.mmu;
4169 
4170         kvm_init_shadow_mmu(vcpu);
4171         context->set_cr3           = kvm_x86_ops->set_cr3;
4172         context->get_cr3           = get_cr3;
4173         context->get_pdptr         = kvm_pdptr_read;
4174         context->inject_page_fault = kvm_inject_page_fault;
4175 }
4176 
4177 static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
4178 {
4179         struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;
4180 
4181         g_context->get_cr3           = get_cr3;
4182         g_context->get_pdptr         = kvm_pdptr_read;
4183         g_context->inject_page_fault = kvm_inject_page_fault;
4184 
4185         /*
4186          * Note that arch.mmu.gva_to_gpa translates l2_gpa to l1_gpa using
4187          * L1's nested page tables (e.g. EPT12). The nested translation
4188          * of l2_gva to l1_gpa is done by arch.nested_mmu.gva_to_gpa using
4189          * L2's page tables as the first level of translation and L1's
4190          * nested page tables as the second level of translation. Basically
4191          * the gva_to_gpa functions between mmu and nested_mmu are swapped.
4192          */
4193         if (!is_paging(vcpu)) {
4194                 g_context->nx = false;
4195                 g_context->root_level = 0;
4196                 g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested;
4197         } else if (is_long_mode(vcpu)) {
4198                 g_context->nx = is_nx(vcpu);
4199                 g_context->root_level = PT64_ROOT_LEVEL;
4200                 reset_rsvds_bits_mask(vcpu, g_context);
4201                 g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
4202         } else if (is_pae(vcpu)) {
4203                 g_context->nx = is_nx(vcpu);
4204                 g_context->root_level = PT32E_ROOT_LEVEL;
4205                 reset_rsvds_bits_mask(vcpu, g_context);
4206                 g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
4207         } else {
4208                 g_context->nx = false;
4209                 g_context->root_level = PT32_ROOT_LEVEL;
4210                 reset_rsvds_bits_mask(vcpu, g_context);
4211                 g_context->gva_to_gpa = paging32_gva_to_gpa_nested;
4212         }
4213 
4214         update_permission_bitmask(vcpu, g_context, false);
4215         update_pkru_bitmask(vcpu, g_context, false);
4216         update_last_nonleaf_level(vcpu, g_context);
4217 }
4218 
4219 static void init_kvm_mmu(struct kvm_vcpu *vcpu)
4220 {
4221         if (mmu_is_nested(vcpu))
4222                 init_kvm_nested_mmu(vcpu);
4223         else if (tdp_enabled)
4224                 init_kvm_tdp_mmu(vcpu);
4225         else
4226                 init_kvm_softmmu(vcpu);
4227 }
4228 
4229 void kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
4230 {
4231         kvm_mmu_unload(vcpu);
4232         init_kvm_mmu(vcpu);
4233 }
4234 EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
4235 
4236 int kvm_mmu_load(struct kvm_vcpu *vcpu)
4237 {
4238         int r;
4239 
4240         r = mmu_topup_memory_caches(vcpu);
4241         if (r)
4242                 goto out;
4243         r = mmu_alloc_roots(vcpu);
4244         kvm_mmu_sync_roots(vcpu);
4245         if (r)
4246                 goto out;
4247         /* set_cr3() should ensure TLB has been flushed */
4248         vcpu->arch.mmu.set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
4249 out:
4250         return r;
4251 }
4252 EXPORT_SYMBOL_GPL(kvm_mmu_load);
4253 
4254 void kvm_mmu_unload(struct kvm_vcpu *vcpu)
4255 {
4256         mmu_free_roots(vcpu);
4257         WARN_ON(VALID_PAGE(vcpu->arch.mmu.root_hpa));
4258 }
4259 EXPORT_SYMBOL_GPL(kvm_mmu_unload);
4260 
4261 static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
4262                                   struct kvm_mmu_page *sp, u64 *spte,
4263                                   const void *new)
4264 {
4265         if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
4266                 ++vcpu->kvm->stat.mmu_pde_zapped;
4267                 return;
4268         }
4269 
4270         ++vcpu->kvm->stat.mmu_pte_updated;
4271         vcpu->arch.mmu.update_pte(vcpu, sp, spte, new);
4272 }
4273 
4274 static bool need_remote_flush(u64 old, u64 new)
4275 {
4276         if (!is_shadow_present_pte(old))
4277                 return false;
4278         if (!is_shadow_present_pte(new))
4279                 return true;
4280         if ((old ^ new) & PT64_BASE_ADDR_MASK)
4281                 return true;
4282         old ^= shadow_nx_mask;
4283         new ^= shadow_nx_mask;
4284         return (old & ~new & PT64_PERM_MASK) != 0;
4285 }
4286 
4287 static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa,
4288                                     const u8 *new, int *bytes)
4289 {
4290         u64 gentry;
4291         int r;
4292 
4293         /*
4294          * Assume that the pte write on a page table of the same type
4295          * as the current vcpu paging mode since we update the sptes only
4296          * when they have the same mode.
4297          */
4298         if (is_pae(vcpu) && *bytes == 4) {
4299                 /* Handle a 32-bit guest writing two halves of a 64-bit gpte */
4300                 *gpa &= ~(gpa_t)7;
4301                 *bytes = 8;
4302                 r = kvm_vcpu_read_guest(vcpu, *gpa, &gentry, 8);
4303                 if (r)
4304                         gentry = 0;
4305                 new = (const u8 *)&gentry;
4306         }
4307 
4308         switch (*bytes) {
4309         case 4:
4310                 gentry = *(const u32 *)new;
4311                 break;
4312         case 8:
4313                 gentry = *(const u64 *)new;
4314                 break;
4315         default:
4316                 gentry = 0;
4317                 break;
4318         }
4319 
4320         return gentry;
4321 }
4322 
4323 /*
4324  * If we're seeing too many writes to a page, it may no longer be a page table,
4325  * or we may be forking, in which case it is better to unmap the page.
4326  */
4327 static bool detect_write_flooding(struct kvm_mmu_page *sp)
4328 {
4329         /*
4330          * Skip write-flooding detected for the sp whose level is 1, because
4331          * it can become unsync, then the guest page is not write-protected.
4332          */
4333         if (sp->role.level == PT_PAGE_TABLE_LEVEL)
4334                 return false;
4335 
4336         atomic_inc(&sp->write_flooding_count);
4337         return atomic_read(&sp->write_flooding_count) >= 3;
4338 }
4339 
4340 /*
4341  * Misaligned accesses are too much trouble to fix up; also, they usually
4342  * indicate a page is not used as a page table.
4343  */
4344 static bool detect_write_misaligned(struct kvm_mmu_page *sp, gpa_t gpa,
4345                                     int bytes)
4346 {
4347         unsigned offset, pte_size, misaligned;
4348 
4349         pgprintk("misaligned: gpa %llx bytes %d role %x\n",
4350                  gpa, bytes, sp->role.word);
4351 
4352         offset = offset_in_page(gpa);
4353         pte_size = sp->role.cr4_pae ? 8 : 4;
4354 
4355         /*
4356          * Sometimes, the OS only writes the last one bytes to update status
4357          * bits, for example, in linux, andb instruction is used in clear_bit().
4358          */
4359         if (!(offset & (pte_size - 1)) && bytes == 1)
4360                 return false;
4361 
4362         misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
4363         misaligned |= bytes < 4;
4364 
4365         return misaligned;
4366 }
4367 
4368 static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte)
4369 {
4370         unsigned page_offset, quadrant;
4371         u64 *spte;
4372         int level;
4373 
4374         page_offset = offset_in_page(gpa);
4375         level = sp->role.level;
4376         *nspte = 1;
4377         if (!sp->role.cr4_pae) {
4378                 page_offset <<= 1;      /* 32->64 */
4379                 /*
4380                  * A 32-bit pde maps 4MB while the shadow pdes map
4381                  * only 2MB.  So we need to double the offset again
4382                  * and zap two pdes instead of one.
4383                  */
4384                 if (level == PT32_ROOT_LEVEL) {
4385                         page_offset &= ~7; /* kill rounding error */
4386                         page_offset <<= 1;
4387                         *nspte = 2;
4388                 }
4389                 quadrant = page_offset >> PAGE_SHIFT;
4390                 page_offset &= ~PAGE_MASK;
4391                 if (quadrant != sp->role.quadrant)
4392                         return NULL;
4393         }
4394 
4395         spte = &sp->spt[page_offset / sizeof(*spte)];
4396         return spte;
4397 }
4398 
4399 static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
4400                               const u8 *new, int bytes,
4401                               struct kvm_page_track_notifier_node *node)
4402 {
4403         gfn_t gfn = gpa >> PAGE_SHIFT;
4404         struct kvm_mmu_page *sp;
4405         LIST_HEAD(invalid_list);
4406         u64 entry, gentry, *spte;
4407         int npte;
4408         bool remote_flush, local_flush;
4409         union kvm_mmu_page_role mask = { };
4410 
4411         mask.cr0_wp = 1;
4412         mask.cr4_pae = 1;
4413         mask.nxe = 1;
4414         mask.smep_andnot_wp = 1;
4415         mask.smap_andnot_wp = 1;
4416         mask.smm = 1;
4417 
4418         /*
4419          * If we don't have indirect shadow pages, it means no page is
4420          * write-protected, so we can exit simply.
4421          */
4422         if (!ACCESS_ONCE(vcpu->kvm->arch.indirect_shadow_pages))
4423                 return;
4424 
4425         remote_flush = local_flush = false;
4426 
4427         pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
4428 
4429         gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, new, &bytes);
4430 
4431         /*
4432          * No need to care whether allocation memory is successful
4433          * or not since pte prefetch is skiped if it does not have
4434          * enough objects in the cache.
4435          */
4436         mmu_topup_memory_caches(vcpu);
4437 
4438         spin_lock(&vcpu->kvm->mmu_lock);
4439         ++vcpu->kvm->stat.mmu_pte_write;
4440         kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE);
4441 
4442         for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) {
4443                 if (detect_write_misaligned(sp, gpa, bytes) ||
4444                       detect_write_flooding(sp)) {
4445                         kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
4446                         ++vcpu->kvm->stat.mmu_flooded;
4447                         continue;
4448                 }
4449 
4450                 spte = get_written_sptes(sp, gpa, &npte);
4451                 if (!spte)
4452                         continue;
4453 
4454                 local_flush = true;
4455                 while (npte--) {
4456                         entry = *spte;
4457                         mmu_page_zap_pte(vcpu->kvm, sp, spte);
4458                         if (gentry &&
4459                               !((sp->role.word ^ vcpu->arch.mmu.base_role.word)
4460                               & mask.word) && rmap_can_add(vcpu))
4461                                 mmu_pte_write_new_pte(vcpu, sp, spte, &gentry);
4462                         if (need_remote_flush(entry, *spte))
4463                                 remote_flush = true;
4464                         ++spte;
4465                 }
4466         }
4467         kvm_mmu_flush_or_zap(vcpu, &invalid_list, remote_flush, local_flush);
4468         kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE);
4469         spin_unlock(&vcpu->kvm->mmu_lock);
4470 }
4471 
4472 int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
4473 {
4474         gpa_t gpa;
4475         int r;
4476 
4477         if (vcpu->arch.mmu.direct_map)
4478                 return 0;
4479 
4480         gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);
4481 
4482         r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
4483 
4484         return r;
4485 }
4486 EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
4487 
4488 static void make_mmu_pages_available(struct kvm_vcpu *vcpu)
4489 {
4490         LIST_HEAD(invalid_list);
4491 
4492         if (likely(kvm_mmu_available_pages(vcpu->kvm) >= KVM_MIN_FREE_MMU_PAGES))
4493                 return;
4494 
4495         while (kvm_mmu_available_pages(vcpu->kvm) < KVM_REFILL_PAGES) {
4496                 if (!prepare_zap_oldest_mmu_page(vcpu->kvm, &invalid_list))
4497                         break;
4498 
4499                 ++vcpu->kvm->stat.mmu_recycled;
4500         }
4501         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
4502 }
4503 
4504 int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u64 error_code,
4505                        void *insn, int insn_len)
4506 {
4507         int r, emulation_type = EMULTYPE_RETRY;
4508         enum emulation_result er;
4509         bool direct = vcpu->arch.mmu.direct_map || mmu_is_nested(vcpu);
4510 
4511         if (unlikely(error_code & PFERR_RSVD_MASK)) {
4512                 r = handle_mmio_page_fault(vcpu, cr2, direct);
4513                 if (r == RET_MMIO_PF_EMULATE) {
4514                         emulation_type = 0;
4515                         goto emulate;
4516                 }
4517                 if (r == RET_MMIO_PF_RETRY)
4518                         return 1;
4519                 if (r < 0)
4520                         return r;
4521         }
4522 
4523         r = vcpu->arch.mmu.page_fault(vcpu, cr2, lower_32_bits(error_code),
4524                                       false);
4525         if (r < 0)
4526                 return r;
4527         if (!r)
4528                 return 1;
4529 
4530         /*
4531          * Before emulating the instruction, check if the error code
4532          * was due to a RO violation while translating the guest page.
4533          * This can occur when using nested virtualization with nested
4534          * paging in both guests. If true, we simply unprotect the page
4535          * and resume the guest.
4536          *
4537          * Note: AMD only (since it supports the PFERR_GUEST_PAGE_MASK used
4538          *       in PFERR_NEXT_GUEST_PAGE)
4539          */
4540         if (error_code == PFERR_NESTED_GUEST_PAGE) {
4541                 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(cr2));
4542                 return 1;
4543         }
4544 
4545         if (mmio_info_in_cache(vcpu, cr2, direct))
4546                 emulation_type = 0;
4547 emulate:
4548         er = x86_emulate_instruction(vcpu, cr2, emulation_type, insn, insn_len);
4549 
4550         switch (er) {
4551         case EMULATE_DONE:
4552                 return 1;
4553         case EMULATE_USER_EXIT:
4554                 ++vcpu->stat.mmio_exits;
4555                 /* fall through */
4556         case EMULATE_FAIL:
4557                 return 0;
4558         default:
4559                 BUG();
4560         }
4561 }
4562 EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
4563 
4564 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
4565 {
4566         vcpu->arch.mmu.invlpg(vcpu, gva);
4567         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
4568         ++vcpu->stat.invlpg;
4569 }
4570 EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
4571 
4572 void kvm_enable_tdp(void)
4573 {
4574         tdp_enabled = true;
4575 }
4576 EXPORT_SYMBOL_GPL(kvm_enable_tdp);
4577 
4578 void kvm_disable_tdp(void)
4579 {
4580         tdp_enabled = false;
4581 }
4582 EXPORT_SYMBOL_GPL(kvm_disable_tdp);
4583 
4584 static void free_mmu_pages(struct kvm_vcpu *vcpu)
4585 {
4586         free_page((unsigned long)vcpu->arch.mmu.pae_root);
4587         if (vcpu->arch.mmu.lm_root != NULL)
4588                 free_page((unsigned long)vcpu->arch.mmu.lm_root);
4589 }
4590 
4591 static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
4592 {
4593         struct page *page;
4594         int i;
4595 
4596         /*
4597          * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
4598          * Therefore we need to allocate shadow page tables in the first
4599          * 4GB of memory, which happens to fit the DMA32 zone.
4600          */
4601         page = alloc_page(GFP_KERNEL | __GFP_DMA32);
4602         if (!page)
4603                 return -ENOMEM;
4604 
4605         vcpu->arch.mmu.pae_root = page_address(page);
4606         for (i = 0; i < 4; ++i)
4607                 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
4608 
4609         return 0;
4610 }
4611 
4612 int kvm_mmu_create(struct kvm_vcpu *vcpu)
4613 {
4614         vcpu->arch.walk_mmu = &vcpu->arch.mmu;
4615         vcpu->arch.mmu.root_hpa = INVALID_PAGE;
4616         vcpu->arch.mmu.translate_gpa = translate_gpa;
4617         vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;
4618 
4619         return alloc_mmu_pages(vcpu);
4620 }
4621 
4622 void kvm_mmu_setup(struct kvm_vcpu *vcpu)
4623 {
4624         MMU_WARN_ON(VALID_PAGE(vcpu->arch.mmu.root_hpa));
4625 
4626         init_kvm_mmu(vcpu);
4627 }
4628 
4629 static void kvm_mmu_invalidate_zap_pages_in_memslot(struct kvm *kvm,
4630                         struct kvm_memory_slot *slot,
4631                         struct kvm_page_track_notifier_node *node)
4632 {
4633         kvm_mmu_invalidate_zap_all_pages(kvm);
4634 }
4635 
4636 void kvm_mmu_init_vm(struct kvm *kvm)
4637 {
4638         struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker;
4639 
4640         node->track_write = kvm_mmu_pte_write;
4641         node->track_flush_slot = kvm_mmu_invalidate_zap_pages_in_memslot;
4642         kvm_page_track_register_notifier(kvm, node);
4643 }
4644 
4645 void kvm_mmu_uninit_vm(struct kvm *kvm)
4646 {
4647         struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker;
4648 
4649         kvm_page_track_unregister_notifier(kvm, node);
4650 }
4651 
4652 /* The return value indicates if tlb flush on all vcpus is needed. */
4653 typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head *rmap_head);
4654 
4655 /* The caller should hold mmu-lock before calling this function. */
4656 static bool
4657 slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot,
4658                         slot_level_handler fn, int start_level, int end_level,
4659                         gfn_t start_gfn, gfn_t end_gfn, bool lock_flush_tlb)
4660 {
4661         struct slot_rmap_walk_iterator iterator;
4662         bool flush = false;
4663 
4664         for_each_slot_rmap_range(memslot, start_level, end_level, start_gfn,
4665                         end_gfn, &iterator) {
4666                 if (iterator.rmap)
4667                         flush |= fn(kvm, iterator.rmap);
4668 
4669                 if (need_resched() || spin_needbreak(&kvm->mmu_lock)) {
4670                         if (flush && lock_flush_tlb) {
4671                                 kvm_flush_remote_tlbs(kvm);
4672                                 flush = false;
4673                         }
4674                         cond_resched_lock(&kvm->mmu_lock);
4675                 }
4676         }
4677 
4678         if (flush && lock_flush_tlb) {
4679                 kvm_flush_remote_tlbs(kvm);
4680                 flush = false;
4681         }
4682 
4683         return flush;
4684 }
4685 
4686 static bool
4687 slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
4688                   slot_level_handler fn, int start_level, int end_level,
4689                   bool lock_flush_tlb)
4690 {
4691         return slot_handle_level_range(kvm, memslot, fn, start_level,
4692                         end_level, memslot->base_gfn,
4693                         memslot->base_gfn + memslot->npages - 1,
4694                         lock_flush_tlb);
4695 }
4696 
4697 static bool
4698 slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
4699                       slot_level_handler fn, bool lock_flush_tlb)
4700 {
4701         return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL,
4702                                  PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
4703 }
4704 
4705 static bool
4706 slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
4707                         slot_level_handler fn, bool lock_flush_tlb)
4708 {
4709         return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL + 1,
4710                                  PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
4711 }
4712 
4713 static bool
4714 slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot,
4715                  slot_level_handler fn, bool lock_flush_tlb)
4716 {
4717         return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL,
4718                                  PT_PAGE_TABLE_LEVEL, lock_flush_tlb);
4719 }
4720 
4721 void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
4722 {
4723         struct kvm_memslots *slots;
4724         struct kvm_memory_slot *memslot;
4725         int i;
4726 
4727         spin_lock(&kvm->mmu_lock);
4728         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
4729                 slots = __kvm_memslots(kvm, i);
4730                 kvm_for_each_memslot(memslot, slots) {
4731                         gfn_t start, end;
4732 
4733                         start = max(gfn_start, memslot->base_gfn);
4734                         end = min(gfn_end, memslot->base_gfn + memslot->npages);
4735                         if (start >= end)
4736                                 continue;
4737 
4738                         slot_handle_level_range(kvm, memslot, kvm_zap_rmapp,
4739                                                 PT_PAGE_TABLE_LEVEL, PT_MAX_HUGEPAGE_LEVEL,
4740                                                 start, end - 1, true);
4741                 }
4742         }
4743 
4744         spin_unlock(&kvm->mmu_lock);
4745 }
4746 
4747 static bool slot_rmap_write_protect(struct kvm *kvm,
4748                                     struct kvm_rmap_head *rmap_head)
4749 {
4750         return __rmap_write_protect(kvm, rmap_head, false);
4751 }
4752 
4753 void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
4754                                       struct kvm_memory_slot *memslot)
4755 {
4756         bool flush;
4757 
4758         spin_lock(&kvm->mmu_lock);
4759         flush = slot_handle_all_level(kvm, memslot, slot_rmap_write_protect,
4760                                       false);
4761         spin_unlock(&kvm->mmu_lock);
4762 
4763         /*
4764          * kvm_mmu_slot_remove_write_access() and kvm_vm_ioctl_get_dirty_log()
4765          * which do tlb flush out of mmu-lock should be serialized by
4766          * kvm->slots_lock otherwise tlb flush would be missed.
4767          */
4768         lockdep_assert_held(&kvm->slots_lock);
4769 
4770         /*
4771          * We can flush all the TLBs out of the mmu lock without TLB
4772          * corruption since we just change the spte from writable to
4773          * readonly so that we only need to care the case of changing
4774          * spte from present to present (changing the spte from present
4775          * to nonpresent will flush all the TLBs immediately), in other
4776          * words, the only case we care is mmu_spte_update() where we
4777          * haved checked SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE
4778          * instead of PT_WRITABLE_MASK, that means it does not depend
4779          * on PT_WRITABLE_MASK anymore.
4780          */
4781         if (flush)
4782                 kvm_flush_remote_tlbs(kvm);
4783 }
4784 
4785 static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm,
4786                                          struct kvm_rmap_head *rmap_head)
4787 {
4788         u64 *sptep;
4789         struct rmap_iterator iter;
4790         int need_tlb_flush = 0;
4791         kvm_pfn_t pfn;
4792         struct kvm_mmu_page *sp;
4793 
4794 restart:
4795         for_each_rmap_spte(rmap_head, &iter, sptep) {
4796                 sp = page_header(__pa(sptep));
4797                 pfn = spte_to_pfn(*sptep);
4798 
4799                 /*
4800                  * We cannot do huge page mapping for indirect shadow pages,
4801                  * which are found on the last rmap (level = 1) when not using
4802                  * tdp; such shadow pages are synced with the page table in
4803                  * the guest, and the guest page table is using 4K page size
4804                  * mapping if the indirect sp has level = 1.
4805                  */
4806                 if (sp->role.direct &&
4807                         !kvm_is_reserved_pfn(pfn) &&
4808                         PageTransCompoundMap(pfn_to_page(pfn))) {
4809                         drop_spte(kvm, sptep);
4810                         need_tlb_flush = 1;
4811                         goto restart;
4812                 }
4813         }
4814 
4815         return need_tlb_flush;
4816 }
4817 
4818 void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
4819                                    const struct kvm_memory_slot *memslot)
4820 {
4821         /* FIXME: const-ify all uses of struct kvm_memory_slot.  */
4822         spin_lock(&kvm->mmu_lock);
4823         slot_handle_leaf(kvm, (struct kvm_memory_slot *)memslot,
4824                          kvm_mmu_zap_collapsible_spte, true);
4825         spin_unlock(&kvm->mmu_lock);
4826 }
4827 
4828 void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
4829                                    struct kvm_memory_slot *memslot)
4830 {
4831         bool flush;
4832 
4833         spin_lock(&kvm->mmu_lock);
4834         flush = slot_handle_leaf(kvm, memslot, __rmap_clear_dirty, false);
4835         spin_unlock(&kvm->mmu_lock);
4836 
4837         lockdep_assert_held(&kvm->slots_lock);
4838 
4839         /*
4840          * It's also safe to flush TLBs out of mmu lock here as currently this
4841          * function is only used for dirty logging, in which case flushing TLB
4842          * out of mmu lock also guarantees no dirty pages will be lost in
4843          * dirty_bitmap.
4844          */
4845         if (flush)
4846                 kvm_flush_remote_tlbs(kvm);
4847 }
4848 EXPORT_SYMBOL_GPL(kvm_mmu_slot_leaf_clear_dirty);
4849 
4850 void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm,
4851                                         struct kvm_memory_slot *memslot)
4852 {
4853         bool flush;
4854 
4855         spin_lock(&kvm->mmu_lock);
4856         flush = slot_handle_large_level(kvm, memslot, slot_rmap_write_protect,
4857                                         false);
4858         spin_unlock(&kvm->mmu_lock);
4859 
4860         /* see kvm_mmu_slot_remove_write_access */
4861         lockdep_assert_held(&kvm->slots_lock);
4862 
4863         if (flush)
4864                 kvm_flush_remote_tlbs(kvm);
4865 }
4866 EXPORT_SYMBOL_GPL(kvm_mmu_slot_largepage_remove_write_access);
4867 
4868 void kvm_mmu_slot_set_dirty(struct kvm *kvm,
4869                             struct kvm_memory_slot *memslot)
4870 {
4871         bool flush;
4872 
4873         spin_lock(&kvm->mmu_lock);
4874         flush = slot_handle_all_level(kvm, memslot, __rmap_set_dirty, false);
4875         spin_unlock(&kvm->mmu_lock);
4876 
4877         lockdep_assert_held(&kvm->slots_lock);
4878 
4879         /* see kvm_mmu_slot_leaf_clear_dirty */
4880         if (flush)
4881                 kvm_flush_remote_tlbs(kvm);
4882 }
4883 EXPORT_SYMBOL_GPL(kvm_mmu_slot_set_dirty);
4884 
4885 #define BATCH_ZAP_PAGES 10
4886 static void kvm_zap_obsolete_pages(struct kvm *kvm)
4887 {
4888         struct kvm_mmu_page *sp, *node;
4889         int batch = 0;
4890 
4891 restart:
4892         list_for_each_entry_safe_reverse(sp, node,
4893               &kvm->arch.active_mmu_pages, link) {
4894                 int ret;
4895 
4896                 /*
4897                  * No obsolete page exists before new created page since
4898                  * active_mmu_pages is the FIFO list.
4899                  */
4900                 if (!is_obsolete_sp(kvm, sp))
4901                         break;
4902 
4903                 /*
4904                  * Since we are reversely walking the list and the invalid
4905                  * list will be moved to the head, skip the invalid page
4906                  * can help us to avoid the infinity list walking.
4907                  */
4908                 if (sp->role.invalid)
4909                         continue;
4910 
4911                 /*
4912                  * Need not flush tlb since we only zap the sp with invalid
4913                  * generation number.
4914                  */
4915                 if (batch >= BATCH_ZAP_PAGES &&
4916                       cond_resched_lock(&kvm->mmu_lock)) {
4917                         batch = 0;
4918                         goto restart;
4919                 }
4920 
4921                 ret = kvm_mmu_prepare_zap_page(kvm, sp,
4922                                 &kvm->arch.zapped_obsolete_pages);
4923                 batch += ret;
4924 
4925                 if (ret)
4926                         goto restart;
4927         }
4928 
4929         /*
4930          * Should flush tlb before free page tables since lockless-walking
4931          * may use the pages.
4932          */
4933         kvm_mmu_commit_zap_page(kvm, &kvm->arch.zapped_obsolete_pages);
4934 }
4935 
4936 /*
4937  * Fast invalidate all shadow pages and use lock-break technique
4938  * to zap obsolete pages.
4939  *
4940  * It's required when memslot is being deleted or VM is being
4941  * destroyed, in these cases, we should ensure that KVM MMU does
4942  * not use any resource of the being-deleted slot or all slots
4943  * after calling the function.
4944  */
4945 void kvm_mmu_invalidate_zap_all_pages(struct kvm *kvm)
4946 {
4947         spin_lock(&kvm->mmu_lock);
4948         trace_kvm_mmu_invalidate_zap_all_pages(kvm);
4949         kvm->arch.mmu_valid_gen++;
4950 
4951         /*
4952          * Notify all vcpus to reload its shadow page table
4953          * and flush TLB. Then all vcpus will switch to new
4954          * shadow page table with the new mmu_valid_gen.
4955          *
4956          * Note: we should do this under the protection of
4957          * mmu-lock, otherwise, vcpu would purge shadow page
4958          * but miss tlb flush.
4959          */
4960         kvm_reload_remote_mmus(kvm);
4961 
4962         kvm_zap_obsolete_pages(kvm);
4963         spin_unlock(&kvm->mmu_lock);
4964 }
4965 
4966 static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm)
4967 {
4968         return unlikely(!list_empty_careful(&kvm->arch.zapped_obsolete_pages));
4969 }
4970 
4971 void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, struct kvm_memslots *slots)
4972 {
4973         /*
4974          * The very rare case: if the generation-number is round,
4975          * zap all shadow pages.
4976          */
4977         if (unlikely((slots->generation & MMIO_GEN_MASK) == 0)) {
4978                 kvm_debug_ratelimited("kvm: zapping shadow pages for mmio generation wraparound\n");
4979                 kvm_mmu_invalidate_zap_all_pages(kvm);
4980         }
4981 }
4982 
4983 static unsigned long
4984 mmu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
4985 {
4986         struct kvm *kvm;
4987         int nr_to_scan = sc->nr_to_scan;
4988         unsigned long freed = 0;
4989 
4990         spin_lock(&kvm_lock);
4991 
4992         list_for_each_entry(kvm, &vm_list, vm_list) {
4993                 int idx;
4994                 LIST_HEAD(invalid_list);
4995 
4996                 /*
4997                  * Never scan more than sc->nr_to_scan VM instances.
4998                  * Will not hit this condition practically since we do not try
4999                  * to shrink more than one VM and it is very unlikely to see
5000                  * !n_used_mmu_pages so many times.
5001                  */
5002                 if (!nr_to_scan--)
5003                         break;
5004                 /*
5005                  * n_used_mmu_pages is accessed without holding kvm->mmu_lock
5006                  * here. We may skip a VM instance errorneosly, but we do not
5007                  * want to shrink a VM that only started to populate its MMU
5008                  * anyway.
5009                  */
5010                 if (!kvm->arch.n_used_mmu_pages &&
5011                       !kvm_has_zapped_obsolete_pages(kvm))
5012                         continue;
5013 
5014                 idx = srcu_read_lock(&kvm->srcu);
5015                 spin_lock(&kvm->mmu_lock);
5016 
5017                 if (kvm_has_zapped_obsolete_pages(kvm)) {
5018                         kvm_mmu_commit_zap_page(kvm,
5019                               &kvm->arch.zapped_obsolete_pages);
5020                         goto unlock;
5021                 }
5022 
5023                 if (prepare_zap_oldest_mmu_page(kvm, &invalid_list))
5024                         freed++;
5025                 kvm_mmu_commit_zap_page(kvm, &invalid_list);
5026 
5027 unlock:
5028                 spin_unlock(&kvm->mmu_lock);
5029                 srcu_read_unlock(&kvm->srcu, idx);
5030 
5031                 /*
5032                  * unfair on small ones
5033                  * per-vm shrinkers cry out
5034                  * sadness comes quickly
5035                  */
5036                 list_move_tail(&kvm->vm_list, &vm_list);
5037                 break;
5038         }
5039 
5040         spin_unlock(&kvm_lock);
5041         return freed;
5042 }
5043 
5044 static unsigned long
5045 mmu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
5046 {
5047         return percpu_counter_read_positive(&kvm_total_used_mmu_pages);
5048 }
5049 
5050 static struct shrinker mmu_shrinker = {
5051         .count_objects = mmu_shrink_count,
5052         .scan_objects = mmu_shrink_scan,
5053         .seeks = DEFAULT_SEEKS * 10,
5054 };
5055 
5056 static void mmu_destroy_caches(void)
5057 {
5058         if (pte_list_desc_cache)
5059                 kmem_cache_destroy(pte_list_desc_cache);
5060         if (mmu_page_header_cache)
5061                 kmem_cache_destroy(mmu_page_header_cache);
5062 }
5063 
5064 int kvm_mmu_module_init(void)
5065 {
5066         pte_list_desc_cache = kmem_cache_create("pte_list_desc",
5067                                             sizeof(struct pte_list_desc),
5068                                             0, 0, NULL);
5069         if (!pte_list_desc_cache)
5070                 goto nomem;
5071 
5072         mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
5073                                                   sizeof(struct kvm_mmu_page),
5074                                                   0, 0, NULL);
5075         if (!mmu_page_header_cache)
5076                 goto nomem;
5077 
5078         if (percpu_counter_init(&kvm_total_used_mmu_pages, 0, GFP_KERNEL))
5079                 goto nomem;
5080 
5081         register_shrinker(&mmu_shrinker);
5082 
5083         return 0;
5084 
5085 nomem:
5086         mmu_destroy_caches();
5087         return -ENOMEM;
5088 }
5089 
5090 /*
5091  * Caculate mmu pages needed for kvm.
5092  */
5093 unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
5094 {
5095         unsigned int nr_mmu_pages;
5096         unsigned int  nr_pages = 0;
5097         struct kvm_memslots *slots;
5098         struct kvm_memory_slot *memslot;
5099         int i;
5100 
5101         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
5102                 slots = __kvm_memslots(kvm, i);
5103 
5104                 kvm_for_each_memslot(memslot, slots)
5105                         nr_pages += memslot->npages;
5106         }
5107 
5108         nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
5109         nr_mmu_pages = max(nr_mmu_pages,
5110                            (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
5111 
5112         return nr_mmu_pages;
5113 }
5114 
5115 void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
5116 {
5117         kvm_mmu_unload(vcpu);
5118         free_mmu_pages(vcpu);
5119         mmu_free_memory_caches(vcpu);
5120 }
5121 
5122 void kvm_mmu_module_exit(void)
5123 {
5124         mmu_destroy_caches();
5125         percpu_counter_destroy(&kvm_total_used_mmu_pages);
5126         unregister_shrinker(&mmu_shrinker);
5127         mmu_audit_disable();
5128 }
5129 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp