~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/kvm/mmu.c

Version: ~ [ linux-5.4-rc3 ] ~ [ linux-5.3.6 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.79 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.149 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.196 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.196 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.75 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Kernel-based Virtual Machine driver for Linux
  3  *
  4  * This module enables machines with Intel VT-x extensions to run virtual
  5  * machines without emulation or binary translation.
  6  *
  7  * MMU support
  8  *
  9  * Copyright (C) 2006 Qumranet, Inc.
 10  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
 11  *
 12  * Authors:
 13  *   Yaniv Kamay  <yaniv@qumranet.com>
 14  *   Avi Kivity   <avi@qumranet.com>
 15  *
 16  * This work is licensed under the terms of the GNU GPL, version 2.  See
 17  * the COPYING file in the top-level directory.
 18  *
 19  */
 20 
 21 #include "irq.h"
 22 #include "mmu.h"
 23 #include "x86.h"
 24 #include "kvm_cache_regs.h"
 25 #include "cpuid.h"
 26 
 27 #include <linux/kvm_host.h>
 28 #include <linux/types.h>
 29 #include <linux/string.h>
 30 #include <linux/mm.h>
 31 #include <linux/highmem.h>
 32 #include <linux/moduleparam.h>
 33 #include <linux/export.h>
 34 #include <linux/swap.h>
 35 #include <linux/hugetlb.h>
 36 #include <linux/compiler.h>
 37 #include <linux/srcu.h>
 38 #include <linux/slab.h>
 39 #include <linux/uaccess.h>
 40 
 41 #include <asm/page.h>
 42 #include <asm/cmpxchg.h>
 43 #include <asm/io.h>
 44 #include <asm/vmx.h>
 45 #include <asm/kvm_page_track.h>
 46 
 47 /*
 48  * When setting this variable to true it enables Two-Dimensional-Paging
 49  * where the hardware walks 2 page tables:
 50  * 1. the guest-virtual to guest-physical
 51  * 2. while doing 1. it walks guest-physical to host-physical
 52  * If the hardware supports that we don't need to do shadow paging.
 53  */
 54 bool tdp_enabled = false;
 55 
 56 enum {
 57         AUDIT_PRE_PAGE_FAULT,
 58         AUDIT_POST_PAGE_FAULT,
 59         AUDIT_PRE_PTE_WRITE,
 60         AUDIT_POST_PTE_WRITE,
 61         AUDIT_PRE_SYNC,
 62         AUDIT_POST_SYNC
 63 };
 64 
 65 #undef MMU_DEBUG
 66 
 67 #ifdef MMU_DEBUG
 68 static bool dbg = 0;
 69 module_param(dbg, bool, 0644);
 70 
 71 #define pgprintk(x...) do { if (dbg) printk(x); } while (0)
 72 #define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
 73 #define MMU_WARN_ON(x) WARN_ON(x)
 74 #else
 75 #define pgprintk(x...) do { } while (0)
 76 #define rmap_printk(x...) do { } while (0)
 77 #define MMU_WARN_ON(x) do { } while (0)
 78 #endif
 79 
 80 #define PTE_PREFETCH_NUM                8
 81 
 82 #define PT_FIRST_AVAIL_BITS_SHIFT 10
 83 #define PT64_SECOND_AVAIL_BITS_SHIFT 52
 84 
 85 #define PT64_LEVEL_BITS 9
 86 
 87 #define PT64_LEVEL_SHIFT(level) \
 88                 (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
 89 
 90 #define PT64_INDEX(address, level)\
 91         (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
 92 
 93 
 94 #define PT32_LEVEL_BITS 10
 95 
 96 #define PT32_LEVEL_SHIFT(level) \
 97                 (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
 98 
 99 #define PT32_LVL_OFFSET_MASK(level) \
100         (PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
101                                                 * PT32_LEVEL_BITS))) - 1))
102 
103 #define PT32_INDEX(address, level)\
104         (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
105 
106 
107 #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
108 #define PT64_DIR_BASE_ADDR_MASK \
109         (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
110 #define PT64_LVL_ADDR_MASK(level) \
111         (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
112                                                 * PT64_LEVEL_BITS))) - 1))
113 #define PT64_LVL_OFFSET_MASK(level) \
114         (PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
115                                                 * PT64_LEVEL_BITS))) - 1))
116 
117 #define PT32_BASE_ADDR_MASK PAGE_MASK
118 #define PT32_DIR_BASE_ADDR_MASK \
119         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
120 #define PT32_LVL_ADDR_MASK(level) \
121         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
122                                             * PT32_LEVEL_BITS))) - 1))
123 
124 #define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | shadow_user_mask \
125                         | shadow_x_mask | shadow_nx_mask)
126 
127 #define ACC_EXEC_MASK    1
128 #define ACC_WRITE_MASK   PT_WRITABLE_MASK
129 #define ACC_USER_MASK    PT_USER_MASK
130 #define ACC_ALL          (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
131 
132 #include <trace/events/kvm.h>
133 
134 #define CREATE_TRACE_POINTS
135 #include "mmutrace.h"
136 
137 #define SPTE_HOST_WRITEABLE     (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
138 #define SPTE_MMU_WRITEABLE      (1ULL << (PT_FIRST_AVAIL_BITS_SHIFT + 1))
139 
140 #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
141 
142 /* make pte_list_desc fit well in cache line */
143 #define PTE_LIST_EXT 3
144 
145 struct pte_list_desc {
146         u64 *sptes[PTE_LIST_EXT];
147         struct pte_list_desc *more;
148 };
149 
150 struct kvm_shadow_walk_iterator {
151         u64 addr;
152         hpa_t shadow_addr;
153         u64 *sptep;
154         int level;
155         unsigned index;
156 };
157 
158 #define for_each_shadow_entry(_vcpu, _addr, _walker)    \
159         for (shadow_walk_init(&(_walker), _vcpu, _addr);        \
160              shadow_walk_okay(&(_walker));                      \
161              shadow_walk_next(&(_walker)))
162 
163 #define for_each_shadow_entry_lockless(_vcpu, _addr, _walker, spte)     \
164         for (shadow_walk_init(&(_walker), _vcpu, _addr);                \
165              shadow_walk_okay(&(_walker)) &&                            \
166                 ({ spte = mmu_spte_get_lockless(_walker.sptep); 1; });  \
167              __shadow_walk_next(&(_walker), spte))
168 
169 static struct kmem_cache *pte_list_desc_cache;
170 static struct kmem_cache *mmu_page_header_cache;
171 static struct percpu_counter kvm_total_used_mmu_pages;
172 
173 static u64 __read_mostly shadow_nx_mask;
174 static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
175 static u64 __read_mostly shadow_user_mask;
176 static u64 __read_mostly shadow_accessed_mask;
177 static u64 __read_mostly shadow_dirty_mask;
178 static u64 __read_mostly shadow_mmio_mask;
179 static u64 __read_mostly shadow_present_mask;
180 
181 static void mmu_spte_set(u64 *sptep, u64 spte);
182 static void mmu_free_roots(struct kvm_vcpu *vcpu);
183 
184 void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask)
185 {
186         shadow_mmio_mask = mmio_mask;
187 }
188 EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_mask);
189 
190 /*
191  * the low bit of the generation number is always presumed to be zero.
192  * This disables mmio caching during memslot updates.  The concept is
193  * similar to a seqcount but instead of retrying the access we just punt
194  * and ignore the cache.
195  *
196  * spte bits 3-11 are used as bits 1-9 of the generation number,
197  * the bits 52-61 are used as bits 10-19 of the generation number.
198  */
199 #define MMIO_SPTE_GEN_LOW_SHIFT         2
200 #define MMIO_SPTE_GEN_HIGH_SHIFT        52
201 
202 #define MMIO_GEN_SHIFT                  20
203 #define MMIO_GEN_LOW_SHIFT              10
204 #define MMIO_GEN_LOW_MASK               ((1 << MMIO_GEN_LOW_SHIFT) - 2)
205 #define MMIO_GEN_MASK                   ((1 << MMIO_GEN_SHIFT) - 1)
206 
207 static u64 generation_mmio_spte_mask(unsigned int gen)
208 {
209         u64 mask;
210 
211         WARN_ON(gen & ~MMIO_GEN_MASK);
212 
213         mask = (gen & MMIO_GEN_LOW_MASK) << MMIO_SPTE_GEN_LOW_SHIFT;
214         mask |= ((u64)gen >> MMIO_GEN_LOW_SHIFT) << MMIO_SPTE_GEN_HIGH_SHIFT;
215         return mask;
216 }
217 
218 static unsigned int get_mmio_spte_generation(u64 spte)
219 {
220         unsigned int gen;
221 
222         spte &= ~shadow_mmio_mask;
223 
224         gen = (spte >> MMIO_SPTE_GEN_LOW_SHIFT) & MMIO_GEN_LOW_MASK;
225         gen |= (spte >> MMIO_SPTE_GEN_HIGH_SHIFT) << MMIO_GEN_LOW_SHIFT;
226         return gen;
227 }
228 
229 static unsigned int kvm_current_mmio_generation(struct kvm_vcpu *vcpu)
230 {
231         return kvm_vcpu_memslots(vcpu)->generation & MMIO_GEN_MASK;
232 }
233 
234 static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn,
235                            unsigned access)
236 {
237         unsigned int gen = kvm_current_mmio_generation(vcpu);
238         u64 mask = generation_mmio_spte_mask(gen);
239 
240         access &= ACC_WRITE_MASK | ACC_USER_MASK;
241         mask |= shadow_mmio_mask | access | gfn << PAGE_SHIFT;
242 
243         trace_mark_mmio_spte(sptep, gfn, access, gen);
244         mmu_spte_set(sptep, mask);
245 }
246 
247 static bool is_mmio_spte(u64 spte)
248 {
249         return (spte & shadow_mmio_mask) == shadow_mmio_mask;
250 }
251 
252 static gfn_t get_mmio_spte_gfn(u64 spte)
253 {
254         u64 mask = generation_mmio_spte_mask(MMIO_GEN_MASK) | shadow_mmio_mask;
255         return (spte & ~mask) >> PAGE_SHIFT;
256 }
257 
258 static unsigned get_mmio_spte_access(u64 spte)
259 {
260         u64 mask = generation_mmio_spte_mask(MMIO_GEN_MASK) | shadow_mmio_mask;
261         return (spte & ~mask) & ~PAGE_MASK;
262 }
263 
264 static bool set_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
265                           kvm_pfn_t pfn, unsigned access)
266 {
267         if (unlikely(is_noslot_pfn(pfn))) {
268                 mark_mmio_spte(vcpu, sptep, gfn, access);
269                 return true;
270         }
271 
272         return false;
273 }
274 
275 static bool check_mmio_spte(struct kvm_vcpu *vcpu, u64 spte)
276 {
277         unsigned int kvm_gen, spte_gen;
278 
279         kvm_gen = kvm_current_mmio_generation(vcpu);
280         spte_gen = get_mmio_spte_generation(spte);
281 
282         trace_check_mmio_spte(spte, kvm_gen, spte_gen);
283         return likely(kvm_gen == spte_gen);
284 }
285 
286 void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
287                 u64 dirty_mask, u64 nx_mask, u64 x_mask, u64 p_mask)
288 {
289         shadow_user_mask = user_mask;
290         shadow_accessed_mask = accessed_mask;
291         shadow_dirty_mask = dirty_mask;
292         shadow_nx_mask = nx_mask;
293         shadow_x_mask = x_mask;
294         shadow_present_mask = p_mask;
295 }
296 EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);
297 
298 static int is_cpuid_PSE36(void)
299 {
300         return 1;
301 }
302 
303 static int is_nx(struct kvm_vcpu *vcpu)
304 {
305         return vcpu->arch.efer & EFER_NX;
306 }
307 
308 static int is_shadow_present_pte(u64 pte)
309 {
310         return (pte & 0xFFFFFFFFull) && !is_mmio_spte(pte);
311 }
312 
313 static int is_large_pte(u64 pte)
314 {
315         return pte & PT_PAGE_SIZE_MASK;
316 }
317 
318 static int is_last_spte(u64 pte, int level)
319 {
320         if (level == PT_PAGE_TABLE_LEVEL)
321                 return 1;
322         if (is_large_pte(pte))
323                 return 1;
324         return 0;
325 }
326 
327 static kvm_pfn_t spte_to_pfn(u64 pte)
328 {
329         return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
330 }
331 
332 static gfn_t pse36_gfn_delta(u32 gpte)
333 {
334         int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;
335 
336         return (gpte & PT32_DIR_PSE36_MASK) << shift;
337 }
338 
339 #ifdef CONFIG_X86_64
340 static void __set_spte(u64 *sptep, u64 spte)
341 {
342         WRITE_ONCE(*sptep, spte);
343 }
344 
345 static void __update_clear_spte_fast(u64 *sptep, u64 spte)
346 {
347         WRITE_ONCE(*sptep, spte);
348 }
349 
350 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
351 {
352         return xchg(sptep, spte);
353 }
354 
355 static u64 __get_spte_lockless(u64 *sptep)
356 {
357         return ACCESS_ONCE(*sptep);
358 }
359 #else
360 union split_spte {
361         struct {
362                 u32 spte_low;
363                 u32 spte_high;
364         };
365         u64 spte;
366 };
367 
368 static void count_spte_clear(u64 *sptep, u64 spte)
369 {
370         struct kvm_mmu_page *sp =  page_header(__pa(sptep));
371 
372         if (is_shadow_present_pte(spte))
373                 return;
374 
375         /* Ensure the spte is completely set before we increase the count */
376         smp_wmb();
377         sp->clear_spte_count++;
378 }
379 
380 static void __set_spte(u64 *sptep, u64 spte)
381 {
382         union split_spte *ssptep, sspte;
383 
384         ssptep = (union split_spte *)sptep;
385         sspte = (union split_spte)spte;
386 
387         ssptep->spte_high = sspte.spte_high;
388 
389         /*
390          * If we map the spte from nonpresent to present, We should store
391          * the high bits firstly, then set present bit, so cpu can not
392          * fetch this spte while we are setting the spte.
393          */
394         smp_wmb();
395 
396         WRITE_ONCE(ssptep->spte_low, sspte.spte_low);
397 }
398 
399 static void __update_clear_spte_fast(u64 *sptep, u64 spte)
400 {
401         union split_spte *ssptep, sspte;
402 
403         ssptep = (union split_spte *)sptep;
404         sspte = (union split_spte)spte;
405 
406         WRITE_ONCE(ssptep->spte_low, sspte.spte_low);
407 
408         /*
409          * If we map the spte from present to nonpresent, we should clear
410          * present bit firstly to avoid vcpu fetch the old high bits.
411          */
412         smp_wmb();
413 
414         ssptep->spte_high = sspte.spte_high;
415         count_spte_clear(sptep, spte);
416 }
417 
418 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
419 {
420         union split_spte *ssptep, sspte, orig;
421 
422         ssptep = (union split_spte *)sptep;
423         sspte = (union split_spte)spte;
424 
425         /* xchg acts as a barrier before the setting of the high bits */
426         orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low);
427         orig.spte_high = ssptep->spte_high;
428         ssptep->spte_high = sspte.spte_high;
429         count_spte_clear(sptep, spte);
430 
431         return orig.spte;
432 }
433 
434 /*
435  * The idea using the light way get the spte on x86_32 guest is from
436  * gup_get_pte(arch/x86/mm/gup.c).
437  *
438  * An spte tlb flush may be pending, because kvm_set_pte_rmapp
439  * coalesces them and we are running out of the MMU lock.  Therefore
440  * we need to protect against in-progress updates of the spte.
441  *
442  * Reading the spte while an update is in progress may get the old value
443  * for the high part of the spte.  The race is fine for a present->non-present
444  * change (because the high part of the spte is ignored for non-present spte),
445  * but for a present->present change we must reread the spte.
446  *
447  * All such changes are done in two steps (present->non-present and
448  * non-present->present), hence it is enough to count the number of
449  * present->non-present updates: if it changed while reading the spte,
450  * we might have hit the race.  This is done using clear_spte_count.
451  */
452 static u64 __get_spte_lockless(u64 *sptep)
453 {
454         struct kvm_mmu_page *sp =  page_header(__pa(sptep));
455         union split_spte spte, *orig = (union split_spte *)sptep;
456         int count;
457 
458 retry:
459         count = sp->clear_spte_count;
460         smp_rmb();
461 
462         spte.spte_low = orig->spte_low;
463         smp_rmb();
464 
465         spte.spte_high = orig->spte_high;
466         smp_rmb();
467 
468         if (unlikely(spte.spte_low != orig->spte_low ||
469               count != sp->clear_spte_count))
470                 goto retry;
471 
472         return spte.spte;
473 }
474 #endif
475 
476 static bool spte_is_locklessly_modifiable(u64 spte)
477 {
478         return (spte & (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE)) ==
479                 (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE);
480 }
481 
482 static bool spte_has_volatile_bits(u64 spte)
483 {
484         /*
485          * Always atomically update spte if it can be updated
486          * out of mmu-lock, it can ensure dirty bit is not lost,
487          * also, it can help us to get a stable is_writable_pte()
488          * to ensure tlb flush is not missed.
489          */
490         if (spte_is_locklessly_modifiable(spte))
491                 return true;
492 
493         if (!shadow_accessed_mask)
494                 return false;
495 
496         if (!is_shadow_present_pte(spte))
497                 return false;
498 
499         if ((spte & shadow_accessed_mask) &&
500               (!is_writable_pte(spte) || (spte & shadow_dirty_mask)))
501                 return false;
502 
503         return true;
504 }
505 
506 static bool spte_is_bit_cleared(u64 old_spte, u64 new_spte, u64 bit_mask)
507 {
508         return (old_spte & bit_mask) && !(new_spte & bit_mask);
509 }
510 
511 static bool spte_is_bit_changed(u64 old_spte, u64 new_spte, u64 bit_mask)
512 {
513         return (old_spte & bit_mask) != (new_spte & bit_mask);
514 }
515 
516 /* Rules for using mmu_spte_set:
517  * Set the sptep from nonpresent to present.
518  * Note: the sptep being assigned *must* be either not present
519  * or in a state where the hardware will not attempt to update
520  * the spte.
521  */
522 static void mmu_spte_set(u64 *sptep, u64 new_spte)
523 {
524         WARN_ON(is_shadow_present_pte(*sptep));
525         __set_spte(sptep, new_spte);
526 }
527 
528 /* Rules for using mmu_spte_update:
529  * Update the state bits, it means the mapped pfn is not changed.
530  *
531  * Whenever we overwrite a writable spte with a read-only one we
532  * should flush remote TLBs. Otherwise rmap_write_protect
533  * will find a read-only spte, even though the writable spte
534  * might be cached on a CPU's TLB, the return value indicates this
535  * case.
536  */
537 static bool mmu_spte_update(u64 *sptep, u64 new_spte)
538 {
539         u64 old_spte = *sptep;
540         bool ret = false;
541 
542         WARN_ON(!is_shadow_present_pte(new_spte));
543 
544         if (!is_shadow_present_pte(old_spte)) {
545                 mmu_spte_set(sptep, new_spte);
546                 return ret;
547         }
548 
549         if (!spte_has_volatile_bits(old_spte))
550                 __update_clear_spte_fast(sptep, new_spte);
551         else
552                 old_spte = __update_clear_spte_slow(sptep, new_spte);
553 
554         /*
555          * For the spte updated out of mmu-lock is safe, since
556          * we always atomically update it, see the comments in
557          * spte_has_volatile_bits().
558          */
559         if (spte_is_locklessly_modifiable(old_spte) &&
560               !is_writable_pte(new_spte))
561                 ret = true;
562 
563         if (!shadow_accessed_mask) {
564                 /*
565                  * We don't set page dirty when dropping non-writable spte.
566                  * So do it now if the new spte is becoming non-writable.
567                  */
568                 if (ret)
569                         kvm_set_pfn_dirty(spte_to_pfn(old_spte));
570                 return ret;
571         }
572 
573         /*
574          * Flush TLB when accessed/dirty bits are changed in the page tables,
575          * to guarantee consistency between TLB and page tables.
576          */
577         if (spte_is_bit_changed(old_spte, new_spte,
578                                 shadow_accessed_mask | shadow_dirty_mask))
579                 ret = true;
580 
581         if (spte_is_bit_cleared(old_spte, new_spte, shadow_accessed_mask))
582                 kvm_set_pfn_accessed(spte_to_pfn(old_spte));
583         if (spte_is_bit_cleared(old_spte, new_spte, shadow_dirty_mask))
584                 kvm_set_pfn_dirty(spte_to_pfn(old_spte));
585 
586         return ret;
587 }
588 
589 /*
590  * Rules for using mmu_spte_clear_track_bits:
591  * It sets the sptep from present to nonpresent, and track the
592  * state bits, it is used to clear the last level sptep.
593  */
594 static int mmu_spte_clear_track_bits(u64 *sptep)
595 {
596         kvm_pfn_t pfn;
597         u64 old_spte = *sptep;
598 
599         if (!spte_has_volatile_bits(old_spte))
600                 __update_clear_spte_fast(sptep, 0ull);
601         else
602                 old_spte = __update_clear_spte_slow(sptep, 0ull);
603 
604         if (!is_shadow_present_pte(old_spte))
605                 return 0;
606 
607         pfn = spte_to_pfn(old_spte);
608 
609         /*
610          * KVM does not hold the refcount of the page used by
611          * kvm mmu, before reclaiming the page, we should
612          * unmap it from mmu first.
613          */
614         WARN_ON(!kvm_is_reserved_pfn(pfn) && !page_count(pfn_to_page(pfn)));
615 
616         if (!shadow_accessed_mask || old_spte & shadow_accessed_mask)
617                 kvm_set_pfn_accessed(pfn);
618         if (old_spte & (shadow_dirty_mask ? shadow_dirty_mask :
619                                             PT_WRITABLE_MASK))
620                 kvm_set_pfn_dirty(pfn);
621         return 1;
622 }
623 
624 /*
625  * Rules for using mmu_spte_clear_no_track:
626  * Directly clear spte without caring the state bits of sptep,
627  * it is used to set the upper level spte.
628  */
629 static void mmu_spte_clear_no_track(u64 *sptep)
630 {
631         __update_clear_spte_fast(sptep, 0ull);
632 }
633 
634 static u64 mmu_spte_get_lockless(u64 *sptep)
635 {
636         return __get_spte_lockless(sptep);
637 }
638 
639 static void walk_shadow_page_lockless_begin(struct kvm_vcpu *vcpu)
640 {
641         /*
642          * Prevent page table teardown by making any free-er wait during
643          * kvm_flush_remote_tlbs() IPI to all active vcpus.
644          */
645         local_irq_disable();
646 
647         /*
648          * Make sure a following spte read is not reordered ahead of the write
649          * to vcpu->mode.
650          */
651         smp_store_mb(vcpu->mode, READING_SHADOW_PAGE_TABLES);
652 }
653 
654 static void walk_shadow_page_lockless_end(struct kvm_vcpu *vcpu)
655 {
656         /*
657          * Make sure the write to vcpu->mode is not reordered in front of
658          * reads to sptes.  If it does, kvm_commit_zap_page() can see us
659          * OUTSIDE_GUEST_MODE and proceed to free the shadow page table.
660          */
661         smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE);
662         local_irq_enable();
663 }
664 
665 static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
666                                   struct kmem_cache *base_cache, int min)
667 {
668         void *obj;
669 
670         if (cache->nobjs >= min)
671                 return 0;
672         while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
673                 obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
674                 if (!obj)
675                         return -ENOMEM;
676                 cache->objects[cache->nobjs++] = obj;
677         }
678         return 0;
679 }
680 
681 static int mmu_memory_cache_free_objects(struct kvm_mmu_memory_cache *cache)
682 {
683         return cache->nobjs;
684 }
685 
686 static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc,
687                                   struct kmem_cache *cache)
688 {
689         while (mc->nobjs)
690                 kmem_cache_free(cache, mc->objects[--mc->nobjs]);
691 }
692 
693 static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
694                                        int min)
695 {
696         void *page;
697 
698         if (cache->nobjs >= min)
699                 return 0;
700         while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
701                 page = (void *)__get_free_page(GFP_KERNEL);
702                 if (!page)
703                         return -ENOMEM;
704                 cache->objects[cache->nobjs++] = page;
705         }
706         return 0;
707 }
708 
709 static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
710 {
711         while (mc->nobjs)
712                 free_page((unsigned long)mc->objects[--mc->nobjs]);
713 }
714 
715 static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
716 {
717         int r;
718 
719         r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
720                                    pte_list_desc_cache, 8 + PTE_PREFETCH_NUM);
721         if (r)
722                 goto out;
723         r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
724         if (r)
725                 goto out;
726         r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
727                                    mmu_page_header_cache, 4);
728 out:
729         return r;
730 }
731 
732 static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
733 {
734         mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
735                                 pte_list_desc_cache);
736         mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
737         mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache,
738                                 mmu_page_header_cache);
739 }
740 
741 static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
742 {
743         void *p;
744 
745         BUG_ON(!mc->nobjs);
746         p = mc->objects[--mc->nobjs];
747         return p;
748 }
749 
750 static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu)
751 {
752         return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache);
753 }
754 
755 static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc)
756 {
757         kmem_cache_free(pte_list_desc_cache, pte_list_desc);
758 }
759 
760 static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
761 {
762         if (!sp->role.direct)
763                 return sp->gfns[index];
764 
765         return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
766 }
767 
768 static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
769 {
770         if (sp->role.direct)
771                 BUG_ON(gfn != kvm_mmu_page_get_gfn(sp, index));
772         else
773                 sp->gfns[index] = gfn;
774 }
775 
776 /*
777  * Return the pointer to the large page information for a given gfn,
778  * handling slots that are not large page aligned.
779  */
780 static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
781                                               struct kvm_memory_slot *slot,
782                                               int level)
783 {
784         unsigned long idx;
785 
786         idx = gfn_to_index(gfn, slot->base_gfn, level);
787         return &slot->arch.lpage_info[level - 2][idx];
788 }
789 
790 static void update_gfn_disallow_lpage_count(struct kvm_memory_slot *slot,
791                                             gfn_t gfn, int count)
792 {
793         struct kvm_lpage_info *linfo;
794         int i;
795 
796         for (i = PT_DIRECTORY_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) {
797                 linfo = lpage_info_slot(gfn, slot, i);
798                 linfo->disallow_lpage += count;
799                 WARN_ON(linfo->disallow_lpage < 0);
800         }
801 }
802 
803 void kvm_mmu_gfn_disallow_lpage(struct kvm_memory_slot *slot, gfn_t gfn)
804 {
805         update_gfn_disallow_lpage_count(slot, gfn, 1);
806 }
807 
808 void kvm_mmu_gfn_allow_lpage(struct kvm_memory_slot *slot, gfn_t gfn)
809 {
810         update_gfn_disallow_lpage_count(slot, gfn, -1);
811 }
812 
813 static void account_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
814 {
815         struct kvm_memslots *slots;
816         struct kvm_memory_slot *slot;
817         gfn_t gfn;
818 
819         kvm->arch.indirect_shadow_pages++;
820         gfn = sp->gfn;
821         slots = kvm_memslots_for_spte_role(kvm, sp->role);
822         slot = __gfn_to_memslot(slots, gfn);
823 
824         /* the non-leaf shadow pages are keeping readonly. */
825         if (sp->role.level > PT_PAGE_TABLE_LEVEL)
826                 return kvm_slot_page_track_add_page(kvm, slot, gfn,
827                                                     KVM_PAGE_TRACK_WRITE);
828 
829         kvm_mmu_gfn_disallow_lpage(slot, gfn);
830 }
831 
832 static void unaccount_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
833 {
834         struct kvm_memslots *slots;
835         struct kvm_memory_slot *slot;
836         gfn_t gfn;
837 
838         kvm->arch.indirect_shadow_pages--;
839         gfn = sp->gfn;
840         slots = kvm_memslots_for_spte_role(kvm, sp->role);
841         slot = __gfn_to_memslot(slots, gfn);
842         if (sp->role.level > PT_PAGE_TABLE_LEVEL)
843                 return kvm_slot_page_track_remove_page(kvm, slot, gfn,
844                                                        KVM_PAGE_TRACK_WRITE);
845 
846         kvm_mmu_gfn_allow_lpage(slot, gfn);
847 }
848 
849 static bool __mmu_gfn_lpage_is_disallowed(gfn_t gfn, int level,
850                                           struct kvm_memory_slot *slot)
851 {
852         struct kvm_lpage_info *linfo;
853 
854         if (slot) {
855                 linfo = lpage_info_slot(gfn, slot, level);
856                 return !!linfo->disallow_lpage;
857         }
858 
859         return true;
860 }
861 
862 static bool mmu_gfn_lpage_is_disallowed(struct kvm_vcpu *vcpu, gfn_t gfn,
863                                         int level)
864 {
865         struct kvm_memory_slot *slot;
866 
867         slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
868         return __mmu_gfn_lpage_is_disallowed(gfn, level, slot);
869 }
870 
871 static int host_mapping_level(struct kvm *kvm, gfn_t gfn)
872 {
873         unsigned long page_size;
874         int i, ret = 0;
875 
876         page_size = kvm_host_page_size(kvm, gfn);
877 
878         for (i = PT_PAGE_TABLE_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) {
879                 if (page_size >= KVM_HPAGE_SIZE(i))
880                         ret = i;
881                 else
882                         break;
883         }
884 
885         return ret;
886 }
887 
888 static inline bool memslot_valid_for_gpte(struct kvm_memory_slot *slot,
889                                           bool no_dirty_log)
890 {
891         if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
892                 return false;
893         if (no_dirty_log && slot->dirty_bitmap)
894                 return false;
895 
896         return true;
897 }
898 
899 static struct kvm_memory_slot *
900 gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t gfn,
901                             bool no_dirty_log)
902 {
903         struct kvm_memory_slot *slot;
904 
905         slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
906         if (!memslot_valid_for_gpte(slot, no_dirty_log))
907                 slot = NULL;
908 
909         return slot;
910 }
911 
912 static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn,
913                          bool *force_pt_level)
914 {
915         int host_level, level, max_level;
916         struct kvm_memory_slot *slot;
917 
918         if (unlikely(*force_pt_level))
919                 return PT_PAGE_TABLE_LEVEL;
920 
921         slot = kvm_vcpu_gfn_to_memslot(vcpu, large_gfn);
922         *force_pt_level = !memslot_valid_for_gpte(slot, true);
923         if (unlikely(*force_pt_level))
924                 return PT_PAGE_TABLE_LEVEL;
925 
926         host_level = host_mapping_level(vcpu->kvm, large_gfn);
927 
928         if (host_level == PT_PAGE_TABLE_LEVEL)
929                 return host_level;
930 
931         max_level = min(kvm_x86_ops->get_lpage_level(), host_level);
932 
933         for (level = PT_DIRECTORY_LEVEL; level <= max_level; ++level)
934                 if (__mmu_gfn_lpage_is_disallowed(large_gfn, level, slot))
935                         break;
936 
937         return level - 1;
938 }
939 
940 /*
941  * About rmap_head encoding:
942  *
943  * If the bit zero of rmap_head->val is clear, then it points to the only spte
944  * in this rmap chain. Otherwise, (rmap_head->val & ~1) points to a struct
945  * pte_list_desc containing more mappings.
946  */
947 
948 /*
949  * Returns the number of pointers in the rmap chain, not counting the new one.
950  */
951 static int pte_list_add(struct kvm_vcpu *vcpu, u64 *spte,
952                         struct kvm_rmap_head *rmap_head)
953 {
954         struct pte_list_desc *desc;
955         int i, count = 0;
956 
957         if (!rmap_head->val) {
958                 rmap_printk("pte_list_add: %p %llx 0->1\n", spte, *spte);
959                 rmap_head->val = (unsigned long)spte;
960         } else if (!(rmap_head->val & 1)) {
961                 rmap_printk("pte_list_add: %p %llx 1->many\n", spte, *spte);
962                 desc = mmu_alloc_pte_list_desc(vcpu);
963                 desc->sptes[0] = (u64 *)rmap_head->val;
964                 desc->sptes[1] = spte;
965                 rmap_head->val = (unsigned long)desc | 1;
966                 ++count;
967         } else {
968                 rmap_printk("pte_list_add: %p %llx many->many\n", spte, *spte);
969                 desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
970                 while (desc->sptes[PTE_LIST_EXT-1] && desc->more) {
971                         desc = desc->more;
972                         count += PTE_LIST_EXT;
973                 }
974                 if (desc->sptes[PTE_LIST_EXT-1]) {
975                         desc->more = mmu_alloc_pte_list_desc(vcpu);
976                         desc = desc->more;
977                 }
978                 for (i = 0; desc->sptes[i]; ++i)
979                         ++count;
980                 desc->sptes[i] = spte;
981         }
982         return count;
983 }
984 
985 static void
986 pte_list_desc_remove_entry(struct kvm_rmap_head *rmap_head,
987                            struct pte_list_desc *desc, int i,
988                            struct pte_list_desc *prev_desc)
989 {
990         int j;
991 
992         for (j = PTE_LIST_EXT - 1; !desc->sptes[j] && j > i; --j)
993                 ;
994         desc->sptes[i] = desc->sptes[j];
995         desc->sptes[j] = NULL;
996         if (j != 0)
997                 return;
998         if (!prev_desc && !desc->more)
999                 rmap_head->val = (unsigned long)desc->sptes[0];
1000         else
1001                 if (prev_desc)
1002                         prev_desc->more = desc->more;
1003                 else
1004                         rmap_head->val = (unsigned long)desc->more | 1;
1005         mmu_free_pte_list_desc(desc);
1006 }
1007 
1008 static void pte_list_remove(u64 *spte, struct kvm_rmap_head *rmap_head)
1009 {
1010         struct pte_list_desc *desc;
1011         struct pte_list_desc *prev_desc;
1012         int i;
1013 
1014         if (!rmap_head->val) {
1015                 printk(KERN_ERR "pte_list_remove: %p 0->BUG\n", spte);
1016                 BUG();
1017         } else if (!(rmap_head->val & 1)) {
1018                 rmap_printk("pte_list_remove:  %p 1->0\n", spte);
1019                 if ((u64 *)rmap_head->val != spte) {
1020                         printk(KERN_ERR "pte_list_remove:  %p 1->BUG\n", spte);
1021                         BUG();
1022                 }
1023                 rmap_head->val = 0;
1024         } else {
1025                 rmap_printk("pte_list_remove:  %p many->many\n", spte);
1026                 desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1027                 prev_desc = NULL;
1028                 while (desc) {
1029                         for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i) {
1030                                 if (desc->sptes[i] == spte) {
1031                                         pte_list_desc_remove_entry(rmap_head,
1032                                                         desc, i, prev_desc);
1033                                         return;
1034                                 }
1035                         }
1036                         prev_desc = desc;
1037                         desc = desc->more;
1038                 }
1039                 pr_err("pte_list_remove: %p many->many\n", spte);
1040                 BUG();
1041         }
1042 }
1043 
1044 static struct kvm_rmap_head *__gfn_to_rmap(gfn_t gfn, int level,
1045                                            struct kvm_memory_slot *slot)
1046 {
1047         unsigned long idx;
1048 
1049         idx = gfn_to_index(gfn, slot->base_gfn, level);
1050         return &slot->arch.rmap[level - PT_PAGE_TABLE_LEVEL][idx];
1051 }
1052 
1053 static struct kvm_rmap_head *gfn_to_rmap(struct kvm *kvm, gfn_t gfn,
1054                                          struct kvm_mmu_page *sp)
1055 {
1056         struct kvm_memslots *slots;
1057         struct kvm_memory_slot *slot;
1058 
1059         slots = kvm_memslots_for_spte_role(kvm, sp->role);
1060         slot = __gfn_to_memslot(slots, gfn);
1061         return __gfn_to_rmap(gfn, sp->role.level, slot);
1062 }
1063 
1064 static bool rmap_can_add(struct kvm_vcpu *vcpu)
1065 {
1066         struct kvm_mmu_memory_cache *cache;
1067 
1068         cache = &vcpu->arch.mmu_pte_list_desc_cache;
1069         return mmu_memory_cache_free_objects(cache);
1070 }
1071 
1072 static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
1073 {
1074         struct kvm_mmu_page *sp;
1075         struct kvm_rmap_head *rmap_head;
1076 
1077         sp = page_header(__pa(spte));
1078         kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
1079         rmap_head = gfn_to_rmap(vcpu->kvm, gfn, sp);
1080         return pte_list_add(vcpu, spte, rmap_head);
1081 }
1082 
1083 static void rmap_remove(struct kvm *kvm, u64 *spte)
1084 {
1085         struct kvm_mmu_page *sp;
1086         gfn_t gfn;
1087         struct kvm_rmap_head *rmap_head;
1088 
1089         sp = page_header(__pa(spte));
1090         gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
1091         rmap_head = gfn_to_rmap(kvm, gfn, sp);
1092         pte_list_remove(spte, rmap_head);
1093 }
1094 
1095 /*
1096  * Used by the following functions to iterate through the sptes linked by a
1097  * rmap.  All fields are private and not assumed to be used outside.
1098  */
1099 struct rmap_iterator {
1100         /* private fields */
1101         struct pte_list_desc *desc;     /* holds the sptep if not NULL */
1102         int pos;                        /* index of the sptep */
1103 };
1104 
1105 /*
1106  * Iteration must be started by this function.  This should also be used after
1107  * removing/dropping sptes from the rmap link because in such cases the
1108  * information in the itererator may not be valid.
1109  *
1110  * Returns sptep if found, NULL otherwise.
1111  */
1112 static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head,
1113                            struct rmap_iterator *iter)
1114 {
1115         u64 *sptep;
1116 
1117         if (!rmap_head->val)
1118                 return NULL;
1119 
1120         if (!(rmap_head->val & 1)) {
1121                 iter->desc = NULL;
1122                 sptep = (u64 *)rmap_head->val;
1123                 goto out;
1124         }
1125 
1126         iter->desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1127         iter->pos = 0;
1128         sptep = iter->desc->sptes[iter->pos];
1129 out:
1130         BUG_ON(!is_shadow_present_pte(*sptep));
1131         return sptep;
1132 }
1133 
1134 /*
1135  * Must be used with a valid iterator: e.g. after rmap_get_first().
1136  *
1137  * Returns sptep if found, NULL otherwise.
1138  */
1139 static u64 *rmap_get_next(struct rmap_iterator *iter)
1140 {
1141         u64 *sptep;
1142 
1143         if (iter->desc) {
1144                 if (iter->pos < PTE_LIST_EXT - 1) {
1145                         ++iter->pos;
1146                         sptep = iter->desc->sptes[iter->pos];
1147                         if (sptep)
1148                                 goto out;
1149                 }
1150 
1151                 iter->desc = iter->desc->more;
1152 
1153                 if (iter->desc) {
1154                         iter->pos = 0;
1155                         /* desc->sptes[0] cannot be NULL */
1156                         sptep = iter->desc->sptes[iter->pos];
1157                         goto out;
1158                 }
1159         }
1160 
1161         return NULL;
1162 out:
1163         BUG_ON(!is_shadow_present_pte(*sptep));
1164         return sptep;
1165 }
1166 
1167 #define for_each_rmap_spte(_rmap_head_, _iter_, _spte_)                 \
1168         for (_spte_ = rmap_get_first(_rmap_head_, _iter_);              \
1169              _spte_; _spte_ = rmap_get_next(_iter_))
1170 
1171 static void drop_spte(struct kvm *kvm, u64 *sptep)
1172 {
1173         if (mmu_spte_clear_track_bits(sptep))
1174                 rmap_remove(kvm, sptep);
1175 }
1176 
1177 
1178 static bool __drop_large_spte(struct kvm *kvm, u64 *sptep)
1179 {
1180         if (is_large_pte(*sptep)) {
1181                 WARN_ON(page_header(__pa(sptep))->role.level ==
1182                         PT_PAGE_TABLE_LEVEL);
1183                 drop_spte(kvm, sptep);
1184                 --kvm->stat.lpages;
1185                 return true;
1186         }
1187 
1188         return false;
1189 }
1190 
1191 static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
1192 {
1193         if (__drop_large_spte(vcpu->kvm, sptep))
1194                 kvm_flush_remote_tlbs(vcpu->kvm);
1195 }
1196 
1197 /*
1198  * Write-protect on the specified @sptep, @pt_protect indicates whether
1199  * spte write-protection is caused by protecting shadow page table.
1200  *
1201  * Note: write protection is difference between dirty logging and spte
1202  * protection:
1203  * - for dirty logging, the spte can be set to writable at anytime if
1204  *   its dirty bitmap is properly set.
1205  * - for spte protection, the spte can be writable only after unsync-ing
1206  *   shadow page.
1207  *
1208  * Return true if tlb need be flushed.
1209  */
1210 static bool spte_write_protect(struct kvm *kvm, u64 *sptep, bool pt_protect)
1211 {
1212         u64 spte = *sptep;
1213 
1214         if (!is_writable_pte(spte) &&
1215               !(pt_protect && spte_is_locklessly_modifiable(spte)))
1216                 return false;
1217 
1218         rmap_printk("rmap_write_protect: spte %p %llx\n", sptep, *sptep);
1219 
1220         if (pt_protect)
1221                 spte &= ~SPTE_MMU_WRITEABLE;
1222         spte = spte & ~PT_WRITABLE_MASK;
1223 
1224         return mmu_spte_update(sptep, spte);
1225 }
1226 
1227 static bool __rmap_write_protect(struct kvm *kvm,
1228                                  struct kvm_rmap_head *rmap_head,
1229                                  bool pt_protect)
1230 {
1231         u64 *sptep;
1232         struct rmap_iterator iter;
1233         bool flush = false;
1234 
1235         for_each_rmap_spte(rmap_head, &iter, sptep)
1236                 flush |= spte_write_protect(kvm, sptep, pt_protect);
1237 
1238         return flush;
1239 }
1240 
1241 static bool spte_clear_dirty(struct kvm *kvm, u64 *sptep)
1242 {
1243         u64 spte = *sptep;
1244 
1245         rmap_printk("rmap_clear_dirty: spte %p %llx\n", sptep, *sptep);
1246 
1247         spte &= ~shadow_dirty_mask;
1248 
1249         return mmu_spte_update(sptep, spte);
1250 }
1251 
1252 static bool __rmap_clear_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head)
1253 {
1254         u64 *sptep;
1255         struct rmap_iterator iter;
1256         bool flush = false;
1257 
1258         for_each_rmap_spte(rmap_head, &iter, sptep)
1259                 flush |= spte_clear_dirty(kvm, sptep);
1260 
1261         return flush;
1262 }
1263 
1264 static bool spte_set_dirty(struct kvm *kvm, u64 *sptep)
1265 {
1266         u64 spte = *sptep;
1267 
1268         rmap_printk("rmap_set_dirty: spte %p %llx\n", sptep, *sptep);
1269 
1270         spte |= shadow_dirty_mask;
1271 
1272         return mmu_spte_update(sptep, spte);
1273 }
1274 
1275 static bool __rmap_set_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head)
1276 {
1277         u64 *sptep;
1278         struct rmap_iterator iter;
1279         bool flush = false;
1280 
1281         for_each_rmap_spte(rmap_head, &iter, sptep)
1282                 flush |= spte_set_dirty(kvm, sptep);
1283 
1284         return flush;
1285 }
1286 
1287 /**
1288  * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages
1289  * @kvm: kvm instance
1290  * @slot: slot to protect
1291  * @gfn_offset: start of the BITS_PER_LONG pages we care about
1292  * @mask: indicates which pages we should protect
1293  *
1294  * Used when we do not need to care about huge page mappings: e.g. during dirty
1295  * logging we do not have any such mappings.
1296  */
1297 static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
1298                                      struct kvm_memory_slot *slot,
1299                                      gfn_t gfn_offset, unsigned long mask)
1300 {
1301         struct kvm_rmap_head *rmap_head;
1302 
1303         while (mask) {
1304                 rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
1305                                           PT_PAGE_TABLE_LEVEL, slot);
1306                 __rmap_write_protect(kvm, rmap_head, false);
1307 
1308                 /* clear the first set bit */
1309                 mask &= mask - 1;
1310         }
1311 }
1312 
1313 /**
1314  * kvm_mmu_clear_dirty_pt_masked - clear MMU D-bit for PT level pages
1315  * @kvm: kvm instance
1316  * @slot: slot to clear D-bit
1317  * @gfn_offset: start of the BITS_PER_LONG pages we care about
1318  * @mask: indicates which pages we should clear D-bit
1319  *
1320  * Used for PML to re-log the dirty GPAs after userspace querying dirty_bitmap.
1321  */
1322 void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm,
1323                                      struct kvm_memory_slot *slot,
1324                                      gfn_t gfn_offset, unsigned long mask)
1325 {
1326         struct kvm_rmap_head *rmap_head;
1327 
1328         while (mask) {
1329                 rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
1330                                           PT_PAGE_TABLE_LEVEL, slot);
1331                 __rmap_clear_dirty(kvm, rmap_head);
1332 
1333                 /* clear the first set bit */
1334                 mask &= mask - 1;
1335         }
1336 }
1337 EXPORT_SYMBOL_GPL(kvm_mmu_clear_dirty_pt_masked);
1338 
1339 /**
1340  * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected
1341  * PT level pages.
1342  *
1343  * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to
1344  * enable dirty logging for them.
1345  *
1346  * Used when we do not need to care about huge page mappings: e.g. during dirty
1347  * logging we do not have any such mappings.
1348  */
1349 void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
1350                                 struct kvm_memory_slot *slot,
1351                                 gfn_t gfn_offset, unsigned long mask)
1352 {
1353         if (kvm_x86_ops->enable_log_dirty_pt_masked)
1354                 kvm_x86_ops->enable_log_dirty_pt_masked(kvm, slot, gfn_offset,
1355                                 mask);
1356         else
1357                 kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
1358 }
1359 
1360 bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm,
1361                                     struct kvm_memory_slot *slot, u64 gfn)
1362 {
1363         struct kvm_rmap_head *rmap_head;
1364         int i;
1365         bool write_protected = false;
1366 
1367         for (i = PT_PAGE_TABLE_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) {
1368                 rmap_head = __gfn_to_rmap(gfn, i, slot);
1369                 write_protected |= __rmap_write_protect(kvm, rmap_head, true);
1370         }
1371 
1372         return write_protected;
1373 }
1374 
1375 static bool rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn)
1376 {
1377         struct kvm_memory_slot *slot;
1378 
1379         slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1380         return kvm_mmu_slot_gfn_write_protect(vcpu->kvm, slot, gfn);
1381 }
1382 
1383 static bool kvm_zap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head)
1384 {
1385         u64 *sptep;
1386         struct rmap_iterator iter;
1387         bool flush = false;
1388 
1389         while ((sptep = rmap_get_first(rmap_head, &iter))) {
1390                 rmap_printk("%s: spte %p %llx.\n", __func__, sptep, *sptep);
1391 
1392                 drop_spte(kvm, sptep);
1393                 flush = true;
1394         }
1395 
1396         return flush;
1397 }
1398 
1399 static int kvm_unmap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1400                            struct kvm_memory_slot *slot, gfn_t gfn, int level,
1401                            unsigned long data)
1402 {
1403         return kvm_zap_rmapp(kvm, rmap_head);
1404 }
1405 
1406 static int kvm_set_pte_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1407                              struct kvm_memory_slot *slot, gfn_t gfn, int level,
1408                              unsigned long data)
1409 {
1410         u64 *sptep;
1411         struct rmap_iterator iter;
1412         int need_flush = 0;
1413         u64 new_spte;
1414         pte_t *ptep = (pte_t *)data;
1415         kvm_pfn_t new_pfn;
1416 
1417         WARN_ON(pte_huge(*ptep));
1418         new_pfn = pte_pfn(*ptep);
1419 
1420 restart:
1421         for_each_rmap_spte(rmap_head, &iter, sptep) {
1422                 rmap_printk("kvm_set_pte_rmapp: spte %p %llx gfn %llx (%d)\n",
1423                              sptep, *sptep, gfn, level);
1424 
1425                 need_flush = 1;
1426 
1427                 if (pte_write(*ptep)) {
1428                         drop_spte(kvm, sptep);
1429                         goto restart;
1430                 } else {
1431                         new_spte = *sptep & ~PT64_BASE_ADDR_MASK;
1432                         new_spte |= (u64)new_pfn << PAGE_SHIFT;
1433 
1434                         new_spte &= ~PT_WRITABLE_MASK;
1435                         new_spte &= ~SPTE_HOST_WRITEABLE;
1436                         new_spte &= ~shadow_accessed_mask;
1437 
1438                         mmu_spte_clear_track_bits(sptep);
1439                         mmu_spte_set(sptep, new_spte);
1440                 }
1441         }
1442 
1443         if (need_flush)
1444                 kvm_flush_remote_tlbs(kvm);
1445 
1446         return 0;
1447 }
1448 
1449 struct slot_rmap_walk_iterator {
1450         /* input fields. */
1451         struct kvm_memory_slot *slot;
1452         gfn_t start_gfn;
1453         gfn_t end_gfn;
1454         int start_level;
1455         int end_level;
1456 
1457         /* output fields. */
1458         gfn_t gfn;
1459         struct kvm_rmap_head *rmap;
1460         int level;
1461 
1462         /* private field. */
1463         struct kvm_rmap_head *end_rmap;
1464 };
1465 
1466 static void
1467 rmap_walk_init_level(struct slot_rmap_walk_iterator *iterator, int level)
1468 {
1469         iterator->level = level;
1470         iterator->gfn = iterator->start_gfn;
1471         iterator->rmap = __gfn_to_rmap(iterator->gfn, level, iterator->slot);
1472         iterator->end_rmap = __gfn_to_rmap(iterator->end_gfn, level,
1473                                            iterator->slot);
1474 }
1475 
1476 static void
1477 slot_rmap_walk_init(struct slot_rmap_walk_iterator *iterator,
1478                     struct kvm_memory_slot *slot, int start_level,
1479                     int end_level, gfn_t start_gfn, gfn_t end_gfn)
1480 {
1481         iterator->slot = slot;
1482         iterator->start_level = start_level;
1483         iterator->end_level = end_level;
1484         iterator->start_gfn = start_gfn;
1485         iterator->end_gfn = end_gfn;
1486 
1487         rmap_walk_init_level(iterator, iterator->start_level);
1488 }
1489 
1490 static bool slot_rmap_walk_okay(struct slot_rmap_walk_iterator *iterator)
1491 {
1492         return !!iterator->rmap;
1493 }
1494 
1495 static void slot_rmap_walk_next(struct slot_rmap_walk_iterator *iterator)
1496 {
1497         if (++iterator->rmap <= iterator->end_rmap) {
1498                 iterator->gfn += (1UL << KVM_HPAGE_GFN_SHIFT(iterator->level));
1499                 return;
1500         }
1501 
1502         if (++iterator->level > iterator->end_level) {
1503                 iterator->rmap = NULL;
1504                 return;
1505         }
1506 
1507         rmap_walk_init_level(iterator, iterator->level);
1508 }
1509 
1510 #define for_each_slot_rmap_range(_slot_, _start_level_, _end_level_,    \
1511            _start_gfn, _end_gfn, _iter_)                                \
1512         for (slot_rmap_walk_init(_iter_, _slot_, _start_level_,         \
1513                                  _end_level_, _start_gfn, _end_gfn);    \
1514              slot_rmap_walk_okay(_iter_);                               \
1515              slot_rmap_walk_next(_iter_))
1516 
1517 static int kvm_handle_hva_range(struct kvm *kvm,
1518                                 unsigned long start,
1519                                 unsigned long end,
1520                                 unsigned long data,
1521                                 int (*handler)(struct kvm *kvm,
1522                                                struct kvm_rmap_head *rmap_head,
1523                                                struct kvm_memory_slot *slot,
1524                                                gfn_t gfn,
1525                                                int level,
1526                                                unsigned long data))
1527 {
1528         struct kvm_memslots *slots;
1529         struct kvm_memory_slot *memslot;
1530         struct slot_rmap_walk_iterator iterator;
1531         int ret = 0;
1532         int i;
1533 
1534         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
1535                 slots = __kvm_memslots(kvm, i);
1536                 kvm_for_each_memslot(memslot, slots) {
1537                         unsigned long hva_start, hva_end;
1538                         gfn_t gfn_start, gfn_end;
1539 
1540                         hva_start = max(start, memslot->userspace_addr);
1541                         hva_end = min(end, memslot->userspace_addr +
1542                                       (memslot->npages << PAGE_SHIFT));
1543                         if (hva_start >= hva_end)
1544                                 continue;
1545                         /*
1546                          * {gfn(page) | page intersects with [hva_start, hva_end)} =
1547                          * {gfn_start, gfn_start+1, ..., gfn_end-1}.
1548                          */
1549                         gfn_start = hva_to_gfn_memslot(hva_start, memslot);
1550                         gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
1551 
1552                         for_each_slot_rmap_range(memslot, PT_PAGE_TABLE_LEVEL,
1553                                                  PT_MAX_HUGEPAGE_LEVEL,
1554                                                  gfn_start, gfn_end - 1,
1555                                                  &iterator)
1556                                 ret |= handler(kvm, iterator.rmap, memslot,
1557                                                iterator.gfn, iterator.level, data);
1558                 }
1559         }
1560 
1561         return ret;
1562 }
1563 
1564 static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
1565                           unsigned long data,
1566                           int (*handler)(struct kvm *kvm,
1567                                          struct kvm_rmap_head *rmap_head,
1568                                          struct kvm_memory_slot *slot,
1569                                          gfn_t gfn, int level,
1570                                          unsigned long data))
1571 {
1572         return kvm_handle_hva_range(kvm, hva, hva + 1, data, handler);
1573 }
1574 
1575 int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
1576 {
1577         return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp);
1578 }
1579 
1580 int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
1581 {
1582         return kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp);
1583 }
1584 
1585 void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
1586 {
1587         kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
1588 }
1589 
1590 static int kvm_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1591                          struct kvm_memory_slot *slot, gfn_t gfn, int level,
1592                          unsigned long data)
1593 {
1594         u64 *sptep;
1595         struct rmap_iterator uninitialized_var(iter);
1596         int young = 0;
1597 
1598         BUG_ON(!shadow_accessed_mask);
1599 
1600         for_each_rmap_spte(rmap_head, &iter, sptep) {
1601                 if (*sptep & shadow_accessed_mask) {
1602                         young = 1;
1603                         clear_bit((ffs(shadow_accessed_mask) - 1),
1604                                  (unsigned long *)sptep);
1605                 }
1606         }
1607 
1608         trace_kvm_age_page(gfn, level, slot, young);
1609         return young;
1610 }
1611 
1612 static int kvm_test_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1613                               struct kvm_memory_slot *slot, gfn_t gfn,
1614                               int level, unsigned long data)
1615 {
1616         u64 *sptep;
1617         struct rmap_iterator iter;
1618         int young = 0;
1619 
1620         /*
1621          * If there's no access bit in the secondary pte set by the
1622          * hardware it's up to gup-fast/gup to set the access bit in
1623          * the primary pte or in the page structure.
1624          */
1625         if (!shadow_accessed_mask)
1626                 goto out;
1627 
1628         for_each_rmap_spte(rmap_head, &iter, sptep) {
1629                 if (*sptep & shadow_accessed_mask) {
1630                         young = 1;
1631                         break;
1632                 }
1633         }
1634 out:
1635         return young;
1636 }
1637 
1638 #define RMAP_RECYCLE_THRESHOLD 1000
1639 
1640 static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
1641 {
1642         struct kvm_rmap_head *rmap_head;
1643         struct kvm_mmu_page *sp;
1644 
1645         sp = page_header(__pa(spte));
1646 
1647         rmap_head = gfn_to_rmap(vcpu->kvm, gfn, sp);
1648 
1649         kvm_unmap_rmapp(vcpu->kvm, rmap_head, NULL, gfn, sp->role.level, 0);
1650         kvm_flush_remote_tlbs(vcpu->kvm);
1651 }
1652 
1653 int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
1654 {
1655         /*
1656          * In case of absence of EPT Access and Dirty Bits supports,
1657          * emulate the accessed bit for EPT, by checking if this page has
1658          * an EPT mapping, and clearing it if it does. On the next access,
1659          * a new EPT mapping will be established.
1660          * This has some overhead, but not as much as the cost of swapping
1661          * out actively used pages or breaking up actively used hugepages.
1662          */
1663         if (!shadow_accessed_mask) {
1664                 /*
1665                  * We are holding the kvm->mmu_lock, and we are blowing up
1666                  * shadow PTEs. MMU notifier consumers need to be kept at bay.
1667                  * This is correct as long as we don't decouple the mmu_lock
1668                  * protected regions (like invalidate_range_start|end does).
1669                  */
1670                 kvm->mmu_notifier_seq++;
1671                 return kvm_handle_hva_range(kvm, start, end, 0,
1672                                             kvm_unmap_rmapp);
1673         }
1674 
1675         return kvm_handle_hva_range(kvm, start, end, 0, kvm_age_rmapp);
1676 }
1677 
1678 int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
1679 {
1680         return kvm_handle_hva(kvm, hva, 0, kvm_test_age_rmapp);
1681 }
1682 
1683 #ifdef MMU_DEBUG
1684 static int is_empty_shadow_page(u64 *spt)
1685 {
1686         u64 *pos;
1687         u64 *end;
1688 
1689         for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
1690                 if (is_shadow_present_pte(*pos)) {
1691                         printk(KERN_ERR "%s: %p %llx\n", __func__,
1692                                pos, *pos);
1693                         return 0;
1694                 }
1695         return 1;
1696 }
1697 #endif
1698 
1699 /*
1700  * This value is the sum of all of the kvm instances's
1701  * kvm->arch.n_used_mmu_pages values.  We need a global,
1702  * aggregate version in order to make the slab shrinker
1703  * faster
1704  */
1705 static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, int nr)
1706 {
1707         kvm->arch.n_used_mmu_pages += nr;
1708         percpu_counter_add(&kvm_total_used_mmu_pages, nr);
1709 }
1710 
1711 static void kvm_mmu_free_page(struct kvm_mmu_page *sp)
1712 {
1713         MMU_WARN_ON(!is_empty_shadow_page(sp->spt));
1714         hlist_del(&sp->hash_link);
1715         list_del(&sp->link);
1716         free_page((unsigned long)sp->spt);
1717         if (!sp->role.direct)
1718                 free_page((unsigned long)sp->gfns);
1719         kmem_cache_free(mmu_page_header_cache, sp);
1720 }
1721 
1722 static unsigned kvm_page_table_hashfn(gfn_t gfn)
1723 {
1724         return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
1725 }
1726 
1727 static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
1728                                     struct kvm_mmu_page *sp, u64 *parent_pte)
1729 {
1730         if (!parent_pte)
1731                 return;
1732 
1733         pte_list_add(vcpu, parent_pte, &sp->parent_ptes);
1734 }
1735 
1736 static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
1737                                        u64 *parent_pte)
1738 {
1739         pte_list_remove(parent_pte, &sp->parent_ptes);
1740 }
1741 
1742 static void drop_parent_pte(struct kvm_mmu_page *sp,
1743                             u64 *parent_pte)
1744 {
1745         mmu_page_remove_parent_pte(sp, parent_pte);
1746         mmu_spte_clear_no_track(parent_pte);
1747 }
1748 
1749 static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, int direct)
1750 {
1751         struct kvm_mmu_page *sp;
1752 
1753         sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache);
1754         sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
1755         if (!direct)
1756                 sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
1757         set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
1758 
1759         /*
1760          * The active_mmu_pages list is the FIFO list, do not move the
1761          * page until it is zapped. kvm_zap_obsolete_pages depends on
1762          * this feature. See the comments in kvm_zap_obsolete_pages().
1763          */
1764         list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
1765         kvm_mod_used_mmu_pages(vcpu->kvm, +1);
1766         return sp;
1767 }
1768 
1769 static void mark_unsync(u64 *spte);
1770 static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
1771 {
1772         u64 *sptep;
1773         struct rmap_iterator iter;
1774 
1775         for_each_rmap_spte(&sp->parent_ptes, &iter, sptep) {
1776                 mark_unsync(sptep);
1777         }
1778 }
1779 
1780 static void mark_unsync(u64 *spte)
1781 {
1782         struct kvm_mmu_page *sp;
1783         unsigned int index;
1784 
1785         sp = page_header(__pa(spte));
1786         index = spte - sp->spt;
1787         if (__test_and_set_bit(index, sp->unsync_child_bitmap))
1788                 return;
1789         if (sp->unsync_children++)
1790                 return;
1791         kvm_mmu_mark_parents_unsync(sp);
1792 }
1793 
1794 static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
1795                                struct kvm_mmu_page *sp)
1796 {
1797         return 0;
1798 }
1799 
1800 static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
1801 {
1802 }
1803 
1804 static void nonpaging_update_pte(struct kvm_vcpu *vcpu,
1805                                  struct kvm_mmu_page *sp, u64 *spte,
1806                                  const void *pte)
1807 {
1808         WARN_ON(1);
1809 }
1810 
1811 #define KVM_PAGE_ARRAY_NR 16
1812 
1813 struct kvm_mmu_pages {
1814         struct mmu_page_and_offset {
1815                 struct kvm_mmu_page *sp;
1816                 unsigned int idx;
1817         } page[KVM_PAGE_ARRAY_NR];
1818         unsigned int nr;
1819 };
1820 
1821 static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
1822                          int idx)
1823 {
1824         int i;
1825 
1826         if (sp->unsync)
1827                 for (i=0; i < pvec->nr; i++)
1828                         if (pvec->page[i].sp == sp)
1829                                 return 0;
1830 
1831         pvec->page[pvec->nr].sp = sp;
1832         pvec->page[pvec->nr].idx = idx;
1833         pvec->nr++;
1834         return (pvec->nr == KVM_PAGE_ARRAY_NR);
1835 }
1836 
1837 static inline void clear_unsync_child_bit(struct kvm_mmu_page *sp, int idx)
1838 {
1839         --sp->unsync_children;
1840         WARN_ON((int)sp->unsync_children < 0);
1841         __clear_bit(idx, sp->unsync_child_bitmap);
1842 }
1843 
1844 static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
1845                            struct kvm_mmu_pages *pvec)
1846 {
1847         int i, ret, nr_unsync_leaf = 0;
1848 
1849         for_each_set_bit(i, sp->unsync_child_bitmap, 512) {
1850                 struct kvm_mmu_page *child;
1851                 u64 ent = sp->spt[i];
1852 
1853                 if (!is_shadow_present_pte(ent) || is_large_pte(ent)) {
1854                         clear_unsync_child_bit(sp, i);
1855                         continue;
1856                 }
1857 
1858                 child = page_header(ent & PT64_BASE_ADDR_MASK);
1859 
1860                 if (child->unsync_children) {
1861                         if (mmu_pages_add(pvec, child, i))
1862                                 return -ENOSPC;
1863 
1864                         ret = __mmu_unsync_walk(child, pvec);
1865                         if (!ret) {
1866                                 clear_unsync_child_bit(sp, i);
1867                                 continue;
1868                         } else if (ret > 0) {
1869                                 nr_unsync_leaf += ret;
1870                         } else
1871                                 return ret;
1872                 } else if (child->unsync) {
1873                         nr_unsync_leaf++;
1874                         if (mmu_pages_add(pvec, child, i))
1875                                 return -ENOSPC;
1876                 } else
1877                         clear_unsync_child_bit(sp, i);
1878         }
1879 
1880         return nr_unsync_leaf;
1881 }
1882 
1883 #define INVALID_INDEX (-1)
1884 
1885 static int mmu_unsync_walk(struct kvm_mmu_page *sp,
1886                            struct kvm_mmu_pages *pvec)
1887 {
1888         pvec->nr = 0;
1889         if (!sp->unsync_children)
1890                 return 0;
1891 
1892         mmu_pages_add(pvec, sp, INVALID_INDEX);
1893         return __mmu_unsync_walk(sp, pvec);
1894 }
1895 
1896 static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1897 {
1898         WARN_ON(!sp->unsync);
1899         trace_kvm_mmu_sync_page(sp);
1900         sp->unsync = 0;
1901         --kvm->stat.mmu_unsync;
1902 }
1903 
1904 static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
1905                                     struct list_head *invalid_list);
1906 static void kvm_mmu_commit_zap_page(struct kvm *kvm,
1907                                     struct list_head *invalid_list);
1908 
1909 /*
1910  * NOTE: we should pay more attention on the zapped-obsolete page
1911  * (is_obsolete_sp(sp) && sp->role.invalid) when you do hash list walk
1912  * since it has been deleted from active_mmu_pages but still can be found
1913  * at hast list.
1914  *
1915  * for_each_gfn_valid_sp() has skipped that kind of pages.
1916  */
1917 #define for_each_gfn_valid_sp(_kvm, _sp, _gfn)                          \
1918         hlist_for_each_entry(_sp,                                       \
1919           &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)], hash_link) \
1920                 if ((_sp)->gfn != (_gfn) || is_obsolete_sp((_kvm), (_sp)) \
1921                         || (_sp)->role.invalid) {} else
1922 
1923 #define for_each_gfn_indirect_valid_sp(_kvm, _sp, _gfn)                 \
1924         for_each_gfn_valid_sp(_kvm, _sp, _gfn)                          \
1925                 if ((_sp)->role.direct) {} else
1926 
1927 /* @sp->gfn should be write-protected at the call site */
1928 static bool __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
1929                             struct list_head *invalid_list)
1930 {
1931         if (sp->role.cr4_pae != !!is_pae(vcpu)) {
1932                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1933                 return false;
1934         }
1935 
1936         if (vcpu->arch.mmu.sync_page(vcpu, sp) == 0) {
1937                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1938                 return false;
1939         }
1940 
1941         return true;
1942 }
1943 
1944 static void kvm_mmu_flush_or_zap(struct kvm_vcpu *vcpu,
1945                                  struct list_head *invalid_list,
1946                                  bool remote_flush, bool local_flush)
1947 {
1948         if (!list_empty(invalid_list)) {
1949                 kvm_mmu_commit_zap_page(vcpu->kvm, invalid_list);
1950                 return;
1951         }
1952 
1953         if (remote_flush)
1954                 kvm_flush_remote_tlbs(vcpu->kvm);
1955         else if (local_flush)
1956                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1957 }
1958 
1959 #ifdef CONFIG_KVM_MMU_AUDIT
1960 #include "mmu_audit.c"
1961 #else
1962 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) { }
1963 static void mmu_audit_disable(void) { }
1964 #endif
1965 
1966 static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp)
1967 {
1968         return unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen);
1969 }
1970 
1971 static bool kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
1972                          struct list_head *invalid_list)
1973 {
1974         kvm_unlink_unsync_page(vcpu->kvm, sp);
1975         return __kvm_sync_page(vcpu, sp, invalid_list);
1976 }
1977 
1978 /* @gfn should be write-protected at the call site */
1979 static bool kvm_sync_pages(struct kvm_vcpu *vcpu, gfn_t gfn,
1980                            struct list_head *invalid_list)
1981 {
1982         struct kvm_mmu_page *s;
1983         bool ret = false;
1984 
1985         for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
1986                 if (!s->unsync)
1987                         continue;
1988 
1989                 WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
1990                 ret |= kvm_sync_page(vcpu, s, invalid_list);
1991         }
1992 
1993         return ret;
1994 }
1995 
1996 struct mmu_page_path {
1997         struct kvm_mmu_page *parent[PT64_ROOT_LEVEL];
1998         unsigned int idx[PT64_ROOT_LEVEL];
1999 };
2000 
2001 #define for_each_sp(pvec, sp, parents, i)                       \
2002                 for (i = mmu_pages_first(&pvec, &parents);      \
2003                         i < pvec.nr && ({ sp = pvec.page[i].sp; 1;});   \
2004                         i = mmu_pages_next(&pvec, &parents, i))
2005 
2006 static int mmu_pages_next(struct kvm_mmu_pages *pvec,
2007                           struct mmu_page_path *parents,
2008                           int i)
2009 {
2010         int n;
2011 
2012         for (n = i+1; n < pvec->nr; n++) {
2013                 struct kvm_mmu_page *sp = pvec->page[n].sp;
2014                 unsigned idx = pvec->page[n].idx;
2015                 int level = sp->role.level;
2016 
2017                 parents->idx[level-1] = idx;
2018                 if (level == PT_PAGE_TABLE_LEVEL)
2019                         break;
2020 
2021                 parents->parent[level-2] = sp;
2022         }
2023 
2024         return n;
2025 }
2026 
2027 static int mmu_pages_first(struct kvm_mmu_pages *pvec,
2028                            struct mmu_page_path *parents)
2029 {
2030         struct kvm_mmu_page *sp;
2031         int level;
2032 
2033         if (pvec->nr == 0)
2034                 return 0;
2035 
2036         WARN_ON(pvec->page[0].idx != INVALID_INDEX);
2037 
2038         sp = pvec->page[0].sp;
2039         level = sp->role.level;
2040         WARN_ON(level == PT_PAGE_TABLE_LEVEL);
2041 
2042         parents->parent[level-2] = sp;
2043 
2044         /* Also set up a sentinel.  Further entries in pvec are all
2045          * children of sp, so this element is never overwritten.
2046          */
2047         parents->parent[level-1] = NULL;
2048         return mmu_pages_next(pvec, parents, 0);
2049 }
2050 
2051 static void mmu_pages_clear_parents(struct mmu_page_path *parents)
2052 {
2053         struct kvm_mmu_page *sp;
2054         unsigned int level = 0;
2055 
2056         do {
2057                 unsigned int idx = parents->idx[level];
2058                 sp = parents->parent[level];
2059                 if (!sp)
2060                         return;
2061 
2062                 WARN_ON(idx == INVALID_INDEX);
2063                 clear_unsync_child_bit(sp, idx);
2064                 level++;
2065         } while (!sp->unsync_children);
2066 }
2067 
2068 static void mmu_sync_children(struct kvm_vcpu *vcpu,
2069                               struct kvm_mmu_page *parent)
2070 {
2071         int i;
2072         struct kvm_mmu_page *sp;
2073         struct mmu_page_path parents;
2074         struct kvm_mmu_pages pages;
2075         LIST_HEAD(invalid_list);
2076         bool flush = false;
2077 
2078         while (mmu_unsync_walk(parent, &pages)) {
2079                 bool protected = false;
2080 
2081                 for_each_sp(pages, sp, parents, i)
2082                         protected |= rmap_write_protect(vcpu, sp->gfn);
2083 
2084                 if (protected) {
2085                         kvm_flush_remote_tlbs(vcpu->kvm);
2086                         flush = false;
2087                 }
2088 
2089                 for_each_sp(pages, sp, parents, i) {
2090                         flush |= kvm_sync_page(vcpu, sp, &invalid_list);
2091                         mmu_pages_clear_parents(&parents);
2092                 }
2093                 if (need_resched() || spin_needbreak(&vcpu->kvm->mmu_lock)) {
2094                         kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush);
2095                         cond_resched_lock(&vcpu->kvm->mmu_lock);
2096                         flush = false;
2097                 }
2098         }
2099 
2100         kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush);
2101 }
2102 
2103 static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp)
2104 {
2105         atomic_set(&sp->write_flooding_count,  0);
2106 }
2107 
2108 static void clear_sp_write_flooding_count(u64 *spte)
2109 {
2110         struct kvm_mmu_page *sp =  page_header(__pa(spte));
2111 
2112         __clear_sp_write_flooding_count(sp);
2113 }
2114 
2115 static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
2116                                              gfn_t gfn,
2117                                              gva_t gaddr,
2118                                              unsigned level,
2119                                              int direct,
2120                                              unsigned access)
2121 {
2122         union kvm_mmu_page_role role;
2123         unsigned quadrant;
2124         struct kvm_mmu_page *sp;
2125         bool need_sync = false;
2126         bool flush = false;
2127         LIST_HEAD(invalid_list);
2128 
2129         role = vcpu->arch.mmu.base_role;
2130         role.level = level;
2131         role.direct = direct;
2132         if (role.direct)
2133                 role.cr4_pae = 0;
2134         role.access = access;
2135         if (!vcpu->arch.mmu.direct_map
2136             && vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
2137                 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
2138                 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
2139                 role.quadrant = quadrant;
2140         }
2141         for_each_gfn_valid_sp(vcpu->kvm, sp, gfn) {
2142                 if (!need_sync && sp->unsync)
2143                         need_sync = true;
2144 
2145                 if (sp->role.word != role.word)
2146                         continue;
2147 
2148                 if (sp->unsync) {
2149                         /* The page is good, but __kvm_sync_page might still end
2150                          * up zapping it.  If so, break in order to rebuild it.
2151                          */
2152                         if (!__kvm_sync_page(vcpu, sp, &invalid_list))
2153                                 break;
2154 
2155                         WARN_ON(!list_empty(&invalid_list));
2156                         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2157                 }
2158 
2159                 if (sp->unsync_children)
2160                         kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
2161 
2162                 __clear_sp_write_flooding_count(sp);
2163                 trace_kvm_mmu_get_page(sp, false);
2164                 return sp;
2165         }
2166 
2167         ++vcpu->kvm->stat.mmu_cache_miss;
2168 
2169         sp = kvm_mmu_alloc_page(vcpu, direct);
2170 
2171         sp->gfn = gfn;
2172         sp->role = role;
2173         hlist_add_head(&sp->hash_link,
2174                 &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]);
2175         if (!direct) {
2176                 /*
2177                  * we should do write protection before syncing pages
2178                  * otherwise the content of the synced shadow page may
2179                  * be inconsistent with guest page table.
2180                  */
2181                 account_shadowed(vcpu->kvm, sp);
2182                 if (level == PT_PAGE_TABLE_LEVEL &&
2183                       rmap_write_protect(vcpu, gfn))
2184                         kvm_flush_remote_tlbs(vcpu->kvm);
2185 
2186                 if (level > PT_PAGE_TABLE_LEVEL && need_sync)
2187                         flush |= kvm_sync_pages(vcpu, gfn, &invalid_list);
2188         }
2189         sp->mmu_valid_gen = vcpu->kvm->arch.mmu_valid_gen;
2190         clear_page(sp->spt);
2191         trace_kvm_mmu_get_page(sp, true);
2192 
2193         kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush);
2194         return sp;
2195 }
2196 
2197 static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
2198                              struct kvm_vcpu *vcpu, u64 addr)
2199 {
2200         iterator->addr = addr;
2201         iterator->shadow_addr = vcpu->arch.mmu.root_hpa;
2202         iterator->level = vcpu->arch.mmu.shadow_root_level;
2203 
2204         if (iterator->level == PT64_ROOT_LEVEL &&
2205             vcpu->arch.mmu.root_level < PT64_ROOT_LEVEL &&
2206             !vcpu->arch.mmu.direct_map)
2207                 --iterator->level;
2208 
2209         if (iterator->level == PT32E_ROOT_LEVEL) {
2210                 iterator->shadow_addr
2211                         = vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
2212                 iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
2213                 --iterator->level;
2214                 if (!iterator->shadow_addr)
2215                         iterator->level = 0;
2216         }
2217 }
2218 
2219 static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
2220 {
2221         if (iterator->level < PT_PAGE_TABLE_LEVEL)
2222                 return false;
2223 
2224         iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
2225         iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
2226         return true;
2227 }
2228 
2229 static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator,
2230                                u64 spte)
2231 {
2232         if (is_last_spte(spte, iterator->level)) {
2233                 iterator->level = 0;
2234                 return;
2235         }
2236 
2237         iterator->shadow_addr = spte & PT64_BASE_ADDR_MASK;
2238         --iterator->level;
2239 }
2240 
2241 static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
2242 {
2243         return __shadow_walk_next(iterator, *iterator->sptep);
2244 }
2245 
2246 static void link_shadow_page(struct kvm_vcpu *vcpu, u64 *sptep,
2247                              struct kvm_mmu_page *sp)
2248 {
2249         u64 spte;
2250 
2251         BUILD_BUG_ON(VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK);
2252 
2253         spte = __pa(sp->spt) | shadow_present_mask | PT_WRITABLE_MASK |
2254                shadow_user_mask | shadow_x_mask | shadow_accessed_mask;
2255 
2256         mmu_spte_set(sptep, spte);
2257 
2258         mmu_page_add_parent_pte(vcpu, sp, sptep);
2259 
2260         if (sp->unsync_children || sp->unsync)
2261                 mark_unsync(sptep);
2262 }
2263 
2264 static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2265                                    unsigned direct_access)
2266 {
2267         if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
2268                 struct kvm_mmu_page *child;
2269 
2270                 /*
2271                  * For the direct sp, if the guest pte's dirty bit
2272                  * changed form clean to dirty, it will corrupt the
2273                  * sp's access: allow writable in the read-only sp,
2274                  * so we should update the spte at this point to get
2275                  * a new sp with the correct access.
2276                  */
2277                 child = page_header(*sptep & PT64_BASE_ADDR_MASK);
2278                 if (child->role.access == direct_access)
2279                         return;
2280 
2281                 drop_parent_pte(child, sptep);
2282                 kvm_flush_remote_tlbs(vcpu->kvm);
2283         }
2284 }
2285 
2286 static bool mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
2287                              u64 *spte)
2288 {
2289         u64 pte;
2290         struct kvm_mmu_page *child;
2291 
2292         pte = *spte;
2293         if (is_shadow_present_pte(pte)) {
2294                 if (is_last_spte(pte, sp->role.level)) {
2295                         drop_spte(kvm, spte);
2296                         if (is_large_pte(pte))
2297                                 --kvm->stat.lpages;
2298                 } else {
2299                         child = page_header(pte & PT64_BASE_ADDR_MASK);
2300                         drop_parent_pte(child, spte);
2301                 }
2302                 return true;
2303         }
2304 
2305         if (is_mmio_spte(pte))
2306                 mmu_spte_clear_no_track(spte);
2307 
2308         return false;
2309 }
2310 
2311 static void kvm_mmu_page_unlink_children(struct kvm *kvm,
2312                                          struct kvm_mmu_page *sp)
2313 {
2314         unsigned i;
2315 
2316         for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
2317                 mmu_page_zap_pte(kvm, sp, sp->spt + i);
2318 }
2319 
2320 static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
2321 {
2322         u64 *sptep;
2323         struct rmap_iterator iter;
2324 
2325         while ((sptep = rmap_get_first(&sp->parent_ptes, &iter)))
2326                 drop_parent_pte(sp, sptep);
2327 }
2328 
2329 static int mmu_zap_unsync_children(struct kvm *kvm,
2330                                    struct kvm_mmu_page *parent,
2331                                    struct list_head *invalid_list)
2332 {
2333         int i, zapped = 0;
2334         struct mmu_page_path parents;
2335         struct kvm_mmu_pages pages;
2336 
2337         if (parent->role.level == PT_PAGE_TABLE_LEVEL)
2338                 return 0;
2339 
2340         while (mmu_unsync_walk(parent, &pages)) {
2341                 struct kvm_mmu_page *sp;
2342 
2343                 for_each_sp(pages, sp, parents, i) {
2344                         kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
2345                         mmu_pages_clear_parents(&parents);
2346                         zapped++;
2347                 }
2348         }
2349 
2350         return zapped;
2351 }
2352 
2353 static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
2354                                     struct list_head *invalid_list)
2355 {
2356         int ret;
2357 
2358         trace_kvm_mmu_prepare_zap_page(sp);
2359         ++kvm->stat.mmu_shadow_zapped;
2360         ret = mmu_zap_unsync_children(kvm, sp, invalid_list);
2361         kvm_mmu_page_unlink_children(kvm, sp);
2362         kvm_mmu_unlink_parents(kvm, sp);
2363 
2364         if (!sp->role.invalid && !sp->role.direct)
2365                 unaccount_shadowed(kvm, sp);
2366 
2367         if (sp->unsync)
2368                 kvm_unlink_unsync_page(kvm, sp);
2369         if (!sp->root_count) {
2370                 /* Count self */
2371                 ret++;
2372                 list_move(&sp->link, invalid_list);
2373                 kvm_mod_used_mmu_pages(kvm, -1);
2374         } else {
2375                 list_move(&sp->link, &kvm->arch.active_mmu_pages);
2376 
2377                 /*
2378                  * The obsolete pages can not be used on any vcpus.
2379                  * See the comments in kvm_mmu_invalidate_zap_all_pages().
2380                  */
2381                 if (!sp->role.invalid && !is_obsolete_sp(kvm, sp))
2382                         kvm_reload_remote_mmus(kvm);
2383         }
2384 
2385         sp->role.invalid = 1;
2386         return ret;
2387 }
2388 
2389 static void kvm_mmu_commit_zap_page(struct kvm *kvm,
2390                                     struct list_head *invalid_list)
2391 {
2392         struct kvm_mmu_page *sp, *nsp;
2393 
2394         if (list_empty(invalid_list))
2395                 return;
2396 
2397         /*
2398          * We need to make sure everyone sees our modifications to
2399          * the page tables and see changes to vcpu->mode here. The barrier
2400          * in the kvm_flush_remote_tlbs() achieves this. This pairs
2401          * with vcpu_enter_guest and walk_shadow_page_lockless_begin/end.
2402          *
2403          * In addition, kvm_flush_remote_tlbs waits for all vcpus to exit
2404          * guest mode and/or lockless shadow page table walks.
2405          */
2406         kvm_flush_remote_tlbs(kvm);
2407 
2408         list_for_each_entry_safe(sp, nsp, invalid_list, link) {
2409                 WARN_ON(!sp->role.invalid || sp->root_count);
2410                 kvm_mmu_free_page(sp);
2411         }
2412 }
2413 
2414 static bool prepare_zap_oldest_mmu_page(struct kvm *kvm,
2415                                         struct list_head *invalid_list)
2416 {
2417         struct kvm_mmu_page *sp;
2418 
2419         if (list_empty(&kvm->arch.active_mmu_pages))
2420                 return false;
2421 
2422         sp = list_last_entry(&kvm->arch.active_mmu_pages,
2423                              struct kvm_mmu_page, link);
2424         kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
2425 
2426         return true;
2427 }
2428 
2429 /*
2430  * Changing the number of mmu pages allocated to the vm
2431  * Note: if goal_nr_mmu_pages is too small, you will get dead lock
2432  */
2433 void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int goal_nr_mmu_pages)
2434 {
2435         LIST_HEAD(invalid_list);
2436 
2437         spin_lock(&kvm->mmu_lock);
2438 
2439         if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
2440                 /* Need to free some mmu pages to achieve the goal. */
2441                 while (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages)
2442                         if (!prepare_zap_oldest_mmu_page(kvm, &invalid_list))
2443                                 break;
2444 
2445                 kvm_mmu_commit_zap_page(kvm, &invalid_list);
2446                 goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
2447         }
2448 
2449         kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
2450 
2451         spin_unlock(&kvm->mmu_lock);
2452 }
2453 
2454 int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
2455 {
2456         struct kvm_mmu_page *sp;
2457         LIST_HEAD(invalid_list);
2458         int r;
2459 
2460         pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
2461         r = 0;
2462         spin_lock(&kvm->mmu_lock);
2463         for_each_gfn_indirect_valid_sp(kvm, sp, gfn) {
2464                 pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
2465                          sp->role.word);
2466                 r = 1;
2467                 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
2468         }
2469         kvm_mmu_commit_zap_page(kvm, &invalid_list);
2470         spin_unlock(&kvm->mmu_lock);
2471 
2472         return r;
2473 }
2474 EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page);
2475 
2476 static void kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
2477 {
2478         trace_kvm_mmu_unsync_page(sp);
2479         ++vcpu->kvm->stat.mmu_unsync;
2480         sp->unsync = 1;
2481 
2482         kvm_mmu_mark_parents_unsync(sp);
2483 }
2484 
2485 static bool mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
2486                                    bool can_unsync)
2487 {
2488         struct kvm_mmu_page *sp;
2489 
2490         if (kvm_page_track_is_active(vcpu, gfn, KVM_PAGE_TRACK_WRITE))
2491                 return true;
2492 
2493         for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) {
2494                 if (!can_unsync)
2495                         return true;
2496 
2497                 if (sp->unsync)
2498                         continue;
2499 
2500                 WARN_ON(sp->role.level != PT_PAGE_TABLE_LEVEL);
2501                 kvm_unsync_page(vcpu, sp);
2502         }
2503 
2504         return false;
2505 }
2506 
2507 static bool kvm_is_mmio_pfn(kvm_pfn_t pfn)
2508 {
2509         if (pfn_valid(pfn))
2510                 return !is_zero_pfn(pfn) && PageReserved(pfn_to_page(pfn));
2511 
2512         return true;
2513 }
2514 
2515 static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2516                     unsigned pte_access, int level,
2517                     gfn_t gfn, kvm_pfn_t pfn, bool speculative,
2518                     bool can_unsync, bool host_writable)
2519 {
2520         u64 spte = 0;
2521         int ret = 0;
2522 
2523         if (set_mmio_spte(vcpu, sptep, gfn, pfn, pte_access))
2524                 return 0;
2525 
2526         /*
2527          * For the EPT case, shadow_present_mask is 0 if hardware
2528          * supports exec-only page table entries.  In that case,
2529          * ACC_USER_MASK and shadow_user_mask are used to represent
2530          * read access.  See FNAME(gpte_access) in paging_tmpl.h.
2531          */
2532         spte |= shadow_present_mask;
2533         if (!speculative)
2534                 spte |= shadow_accessed_mask;
2535 
2536         if (pte_access & ACC_EXEC_MASK)
2537                 spte |= shadow_x_mask;
2538         else
2539                 spte |= shadow_nx_mask;
2540 
2541         if (pte_access & ACC_USER_MASK)
2542                 spte |= shadow_user_mask;
2543 
2544         if (level > PT_PAGE_TABLE_LEVEL)
2545                 spte |= PT_PAGE_SIZE_MASK;
2546         if (tdp_enabled)
2547                 spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
2548                         kvm_is_mmio_pfn(pfn));
2549 
2550         if (host_writable)
2551                 spte |= SPTE_HOST_WRITEABLE;
2552         else
2553                 pte_access &= ~ACC_WRITE_MASK;
2554 
2555         spte |= (u64)pfn << PAGE_SHIFT;
2556 
2557         if (pte_access & ACC_WRITE_MASK) {
2558 
2559                 /*
2560                  * Other vcpu creates new sp in the window between
2561                  * mapping_level() and acquiring mmu-lock. We can
2562                  * allow guest to retry the access, the mapping can
2563                  * be fixed if guest refault.
2564                  */
2565                 if (level > PT_PAGE_TABLE_LEVEL &&
2566                     mmu_gfn_lpage_is_disallowed(vcpu, gfn, level))
2567                         goto done;
2568 
2569                 spte |= PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE;
2570 
2571                 /*
2572                  * Optimization: for pte sync, if spte was writable the hash
2573                  * lookup is unnecessary (and expensive). Write protection
2574                  * is responsibility of mmu_get_page / kvm_sync_page.
2575                  * Same reasoning can be applied to dirty page accounting.
2576                  */
2577                 if (!can_unsync && is_writable_pte(*sptep))
2578                         goto set_pte;
2579 
2580                 if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
2581                         pgprintk("%s: found shadow page for %llx, marking ro\n",
2582                                  __func__, gfn);
2583                         ret = 1;
2584                         pte_access &= ~ACC_WRITE_MASK;
2585                         spte &= ~(PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE);
2586                 }
2587         }
2588 
2589         if (pte_access & ACC_WRITE_MASK) {
2590                 kvm_vcpu_mark_page_dirty(vcpu, gfn);
2591                 spte |= shadow_dirty_mask;
2592         }
2593 
2594 set_pte:
2595         if (mmu_spte_update(sptep, spte))
2596                 kvm_flush_remote_tlbs(vcpu->kvm);
2597 done:
2598         return ret;
2599 }
2600 
2601 static bool mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, unsigned pte_access,
2602                          int write_fault, int level, gfn_t gfn, kvm_pfn_t pfn,
2603                          bool speculative, bool host_writable)
2604 {
2605         int was_rmapped = 0;
2606         int rmap_count;
2607         bool emulate = false;
2608 
2609         pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__,
2610                  *sptep, write_fault, gfn);
2611 
2612         if (is_shadow_present_pte(*sptep)) {
2613                 /*
2614                  * If we overwrite a PTE page pointer with a 2MB PMD, unlink
2615                  * the parent of the now unreachable PTE.
2616                  */
2617                 if (level > PT_PAGE_TABLE_LEVEL &&
2618                     !is_large_pte(*sptep)) {
2619                         struct kvm_mmu_page *child;
2620                         u64 pte = *sptep;
2621 
2622                         child = page_header(pte & PT64_BASE_ADDR_MASK);
2623                         drop_parent_pte(child, sptep);
2624                         kvm_flush_remote_tlbs(vcpu->kvm);
2625                 } else if (pfn != spte_to_pfn(*sptep)) {
2626                         pgprintk("hfn old %llx new %llx\n",
2627                                  spte_to_pfn(*sptep), pfn);
2628                         drop_spte(vcpu->kvm, sptep);
2629                         kvm_flush_remote_tlbs(vcpu->kvm);
2630                 } else
2631                         was_rmapped = 1;
2632         }
2633 
2634         if (set_spte(vcpu, sptep, pte_access, level, gfn, pfn, speculative,
2635               true, host_writable)) {
2636                 if (write_fault)
2637                         emulate = true;
2638                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2639         }
2640 
2641         if (unlikely(is_mmio_spte(*sptep)))
2642                 emulate = true;
2643 
2644         pgprintk("%s: setting spte %llx\n", __func__, *sptep);
2645         pgprintk("instantiating %s PTE (%s) at %llx (%llx) addr %p\n",
2646                  is_large_pte(*sptep)? "2MB" : "4kB",
2647                  *sptep & PT_PRESENT_MASK ?"RW":"R", gfn,
2648                  *sptep, sptep);
2649         if (!was_rmapped && is_large_pte(*sptep))
2650                 ++vcpu->kvm->stat.lpages;
2651 
2652         if (is_shadow_present_pte(*sptep)) {
2653                 if (!was_rmapped) {
2654                         rmap_count = rmap_add(vcpu, sptep, gfn);
2655                         if (rmap_count > RMAP_RECYCLE_THRESHOLD)
2656                                 rmap_recycle(vcpu, sptep, gfn);
2657                 }
2658         }
2659 
2660         kvm_release_pfn_clean(pfn);
2661 
2662         return emulate;
2663 }
2664 
2665 static kvm_pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
2666                                      bool no_dirty_log)
2667 {
2668         struct kvm_memory_slot *slot;
2669 
2670         slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log);
2671         if (!slot)
2672                 return KVM_PFN_ERR_FAULT;
2673 
2674         return gfn_to_pfn_memslot_atomic(slot, gfn);
2675 }
2676 
2677 static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
2678                                     struct kvm_mmu_page *sp,
2679                                     u64 *start, u64 *end)
2680 {
2681         struct page *pages[PTE_PREFETCH_NUM];
2682         struct kvm_memory_slot *slot;
2683         unsigned access = sp->role.access;
2684         int i, ret;
2685         gfn_t gfn;
2686 
2687         gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
2688         slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK);
2689         if (!slot)
2690                 return -1;
2691 
2692         ret = gfn_to_page_many_atomic(slot, gfn, pages, end - start);
2693         if (ret <= 0)
2694                 return -1;
2695 
2696         for (i = 0; i < ret; i++, gfn++, start++)
2697                 mmu_set_spte(vcpu, start, access, 0, sp->role.level, gfn,
2698                              page_to_pfn(pages[i]), true, true);
2699 
2700         return 0;
2701 }
2702 
2703 static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
2704                                   struct kvm_mmu_page *sp, u64 *sptep)
2705 {
2706         u64 *spte, *start = NULL;
2707         int i;
2708 
2709         WARN_ON(!sp->role.direct);
2710 
2711         i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
2712         spte = sp->spt + i;
2713 
2714         for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
2715                 if (is_shadow_present_pte(*spte) || spte == sptep) {
2716                         if (!start)
2717                                 continue;
2718                         if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
2719                                 break;
2720                         start = NULL;
2721                 } else if (!start)
2722                         start = spte;
2723         }
2724 }
2725 
2726 static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
2727 {
2728         struct kvm_mmu_page *sp;
2729 
2730         /*
2731          * Since it's no accessed bit on EPT, it's no way to
2732          * distinguish between actually accessed translations
2733          * and prefetched, so disable pte prefetch if EPT is
2734          * enabled.
2735          */
2736         if (!shadow_accessed_mask)
2737                 return;
2738 
2739         sp = page_header(__pa(sptep));
2740         if (sp->role.level > PT_PAGE_TABLE_LEVEL)
2741                 return;
2742 
2743         __direct_pte_prefetch(vcpu, sp, sptep);
2744 }
2745 
2746 static int __direct_map(struct kvm_vcpu *vcpu, int write, int map_writable,
2747                         int level, gfn_t gfn, kvm_pfn_t pfn, bool prefault)
2748 {
2749         struct kvm_shadow_walk_iterator iterator;
2750         struct kvm_mmu_page *sp;
2751         int emulate = 0;
2752         gfn_t pseudo_gfn;
2753 
2754         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
2755                 return 0;
2756 
2757         for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
2758                 if (iterator.level == level) {
2759                         emulate = mmu_set_spte(vcpu, iterator.sptep, ACC_ALL,
2760                                                write, level, gfn, pfn, prefault,
2761                                                map_writable);
2762                         direct_pte_prefetch(vcpu, iterator.sptep);
2763                         ++vcpu->stat.pf_fixed;
2764                         break;
2765                 }
2766 
2767                 drop_large_spte(vcpu, iterator.sptep);
2768                 if (!is_shadow_present_pte(*iterator.sptep)) {
2769                         u64 base_addr = iterator.addr;
2770 
2771                         base_addr &= PT64_LVL_ADDR_MASK(iterator.level);
2772                         pseudo_gfn = base_addr >> PAGE_SHIFT;
2773                         sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
2774                                               iterator.level - 1, 1, ACC_ALL);
2775 
2776                         link_shadow_page(vcpu, iterator.sptep, sp);
2777                 }
2778         }
2779         return emulate;
2780 }
2781 
2782 static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk)
2783 {
2784         siginfo_t info;
2785 
2786         info.si_signo   = SIGBUS;
2787         info.si_errno   = 0;
2788         info.si_code    = BUS_MCEERR_AR;
2789         info.si_addr    = (void __user *)address;
2790         info.si_addr_lsb = PAGE_SHIFT;
2791 
2792         send_sig_info(SIGBUS, &info, tsk);
2793 }
2794 
2795 static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn)
2796 {
2797         /*
2798          * Do not cache the mmio info caused by writing the readonly gfn
2799          * into the spte otherwise read access on readonly gfn also can
2800          * caused mmio page fault and treat it as mmio access.
2801          * Return 1 to tell kvm to emulate it.
2802          */
2803         if (pfn == KVM_PFN_ERR_RO_FAULT)
2804                 return 1;
2805 
2806         if (pfn == KVM_PFN_ERR_HWPOISON) {
2807                 kvm_send_hwpoison_signal(kvm_vcpu_gfn_to_hva(vcpu, gfn), current);
2808                 return 0;
2809         }
2810 
2811         return -EFAULT;
2812 }
2813 
2814 static void transparent_hugepage_adjust(struct kvm_vcpu *vcpu,
2815                                         gfn_t *gfnp, kvm_pfn_t *pfnp,
2816                                         int *levelp)
2817 {
2818         kvm_pfn_t pfn = *pfnp;
2819         gfn_t gfn = *gfnp;
2820         int level = *levelp;
2821 
2822         /*
2823          * Check if it's a transparent hugepage. If this would be an
2824          * hugetlbfs page, level wouldn't be set to
2825          * PT_PAGE_TABLE_LEVEL and there would be no adjustment done
2826          * here.
2827          */
2828         if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn) &&
2829             level == PT_PAGE_TABLE_LEVEL &&
2830             PageTransCompoundMap(pfn_to_page(pfn)) &&
2831             !mmu_gfn_lpage_is_disallowed(vcpu, gfn, PT_DIRECTORY_LEVEL)) {
2832                 unsigned long mask;
2833                 /*
2834                  * mmu_notifier_retry was successful and we hold the
2835                  * mmu_lock here, so the pmd can't become splitting
2836                  * from under us, and in turn
2837                  * __split_huge_page_refcount() can't run from under
2838                  * us and we can safely transfer the refcount from
2839                  * PG_tail to PG_head as we switch the pfn to tail to
2840                  * head.
2841                  */
2842                 *levelp = level = PT_DIRECTORY_LEVEL;
2843                 mask = KVM_PAGES_PER_HPAGE(level) - 1;
2844                 VM_BUG_ON((gfn & mask) != (pfn & mask));
2845                 if (pfn & mask) {
2846                         gfn &= ~mask;
2847                         *gfnp = gfn;
2848                         kvm_release_pfn_clean(pfn);
2849                         pfn &= ~mask;
2850                         kvm_get_pfn(pfn);
2851                         *pfnp = pfn;
2852                 }
2853         }
2854 }
2855 
2856 static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn,
2857                                 kvm_pfn_t pfn, unsigned access, int *ret_val)
2858 {
2859         /* The pfn is invalid, report the error! */
2860         if (unlikely(is_error_pfn(pfn))) {
2861                 *ret_val = kvm_handle_bad_page(vcpu, gfn, pfn);
2862                 return true;
2863         }
2864 
2865         if (unlikely(is_noslot_pfn(pfn)))
2866                 vcpu_cache_mmio_info(vcpu, gva, gfn, access);
2867 
2868         return false;
2869 }
2870 
2871 static bool page_fault_can_be_fast(u32 error_code)
2872 {
2873         /*
2874          * Do not fix the mmio spte with invalid generation number which
2875          * need to be updated by slow page fault path.
2876          */
2877         if (unlikely(error_code & PFERR_RSVD_MASK))
2878                 return false;
2879 
2880         /*
2881          * #PF can be fast only if the shadow page table is present and it
2882          * is caused by write-protect, that means we just need change the
2883          * W bit of the spte which can be done out of mmu-lock.
2884          */
2885         if (!(error_code & PFERR_PRESENT_MASK) ||
2886               !(error_code & PFERR_WRITE_MASK))
2887                 return false;
2888 
2889         return true;
2890 }
2891 
2892 static bool
2893 fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
2894                         u64 *sptep, u64 spte)
2895 {
2896         gfn_t gfn;
2897 
2898         WARN_ON(!sp->role.direct);
2899 
2900         /*
2901          * The gfn of direct spte is stable since it is calculated
2902          * by sp->gfn.
2903          */
2904         gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt);
2905 
2906         /*
2907          * Theoretically we could also set dirty bit (and flush TLB) here in
2908          * order to eliminate unnecessary PML logging. See comments in
2909          * set_spte. But fast_page_fault is very unlikely to happen with PML
2910          * enabled, so we do not do this. This might result in the same GPA
2911          * to be logged in PML buffer again when the write really happens, and
2912          * eventually to be called by mark_page_dirty twice. But it's also no
2913          * harm. This also avoids the TLB flush needed after setting dirty bit
2914          * so non-PML cases won't be impacted.
2915          *
2916          * Compare with set_spte where instead shadow_dirty_mask is set.
2917          */
2918         if (cmpxchg64(sptep, spte, spte | PT_WRITABLE_MASK) == spte)
2919                 kvm_vcpu_mark_page_dirty(vcpu, gfn);
2920 
2921         return true;
2922 }
2923 
2924 /*
2925  * Return value:
2926  * - true: let the vcpu to access on the same address again.
2927  * - false: let the real page fault path to fix it.
2928  */
2929 static bool fast_page_fault(struct kvm_vcpu *vcpu, gva_t gva, int level,
2930                             u32 error_code)
2931 {
2932         struct kvm_shadow_walk_iterator iterator;
2933         struct kvm_mmu_page *sp;
2934         bool ret = false;
2935         u64 spte = 0ull;
2936 
2937         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
2938                 return false;
2939 
2940         if (!page_fault_can_be_fast(error_code))
2941                 return false;
2942 
2943         walk_shadow_page_lockless_begin(vcpu);
2944         for_each_shadow_entry_lockless(vcpu, gva, iterator, spte)
2945                 if (!is_shadow_present_pte(spte) || iterator.level < level)
2946                         break;
2947 
2948         /*
2949          * If the mapping has been changed, let the vcpu fault on the
2950          * same address again.
2951          */
2952         if (!is_shadow_present_pte(spte)) {
2953                 ret = true;
2954                 goto exit;
2955         }
2956 
2957         sp = page_header(__pa(iterator.sptep));
2958         if (!is_last_spte(spte, sp->role.level))
2959                 goto exit;
2960 
2961         /*
2962          * Check if it is a spurious fault caused by TLB lazily flushed.
2963          *
2964          * Need not check the access of upper level table entries since
2965          * they are always ACC_ALL.
2966          */
2967          if (is_writable_pte(spte)) {
2968                 ret = true;
2969                 goto exit;
2970         }
2971 
2972         /*
2973          * Currently, to simplify the code, only the spte write-protected
2974          * by dirty-log can be fast fixed.
2975          */
2976         if (!spte_is_locklessly_modifiable(spte))
2977                 goto exit;
2978 
2979         /*
2980          * Do not fix write-permission on the large spte since we only dirty
2981          * the first page into the dirty-bitmap in fast_pf_fix_direct_spte()
2982          * that means other pages are missed if its slot is dirty-logged.
2983          *
2984          * Instead, we let the slow page fault path create a normal spte to
2985          * fix the access.
2986          *
2987          * See the comments in kvm_arch_commit_memory_region().
2988          */
2989         if (sp->role.level > PT_PAGE_TABLE_LEVEL)
2990                 goto exit;
2991 
2992         /*
2993          * Currently, fast page fault only works for direct mapping since
2994          * the gfn is not stable for indirect shadow page.
2995          * See Documentation/virtual/kvm/locking.txt to get more detail.
2996          */
2997         ret = fast_pf_fix_direct_spte(vcpu, sp, iterator.sptep, spte);
2998 exit:
2999         trace_fast_page_fault(vcpu, gva, error_code, iterator.sptep,
3000                               spte, ret);
3001         walk_shadow_page_lockless_end(vcpu);
3002 
3003         return ret;
3004 }
3005 
3006 static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
3007                          gva_t gva, kvm_pfn_t *pfn, bool write, bool *writable);
3008 static void make_mmu_pages_available(struct kvm_vcpu *vcpu);
3009 
3010 static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, u32 error_code,
3011                          gfn_t gfn, bool prefault)
3012 {
3013         int r;
3014         int level;
3015         bool force_pt_level = false;
3016         kvm_pfn_t pfn;
3017         unsigned long mmu_seq;
3018         bool map_writable, write = error_code & PFERR_WRITE_MASK;
3019 
3020         level = mapping_level(vcpu, gfn, &force_pt_level);
3021         if (likely(!force_pt_level)) {
3022                 /*
3023                  * This path builds a PAE pagetable - so we can map
3024                  * 2mb pages at maximum. Therefore check if the level
3025                  * is larger than that.
3026                  */
3027                 if (level > PT_DIRECTORY_LEVEL)
3028                         level = PT_DIRECTORY_LEVEL;
3029 
3030                 gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
3031         }
3032 
3033         if (fast_page_fault(vcpu, v, level, error_code))
3034                 return 0;
3035 
3036         mmu_seq = vcpu->kvm->mmu_notifier_seq;
3037         smp_rmb();
3038 
3039         if (try_async_pf(vcpu, prefault, gfn, v, &pfn, write, &map_writable))
3040                 return 0;
3041 
3042         if (handle_abnormal_pfn(vcpu, v, gfn, pfn, ACC_ALL, &r))
3043                 return r;
3044 
3045         spin_lock(&vcpu->kvm->mmu_lock);
3046         if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
3047                 goto out_unlock;
3048         make_mmu_pages_available(vcpu);
3049         if (likely(!force_pt_level))
3050                 transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
3051         r = __direct_map(vcpu, write, map_writable, level, gfn, pfn, prefault);
3052         spin_unlock(&vcpu->kvm->mmu_lock);
3053 
3054         return r;
3055 
3056 out_unlock:
3057         spin_unlock(&vcpu->kvm->mmu_lock);
3058         kvm_release_pfn_clean(pfn);
3059         return 0;
3060 }
3061 
3062 
3063 static void mmu_free_roots(struct kvm_vcpu *vcpu)
3064 {
3065         int i;
3066         struct kvm_mmu_page *sp;
3067         LIST_HEAD(invalid_list);
3068 
3069         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
3070                 return;
3071 
3072         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL &&
3073             (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL ||
3074              vcpu->arch.mmu.direct_map)) {
3075                 hpa_t root = vcpu->arch.mmu.root_hpa;
3076 
3077                 spin_lock(&vcpu->kvm->mmu_lock);
3078                 sp = page_header(root);
3079                 --sp->root_count;
3080                 if (!sp->root_count && sp->role.invalid) {
3081                         kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
3082                         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
3083                 }
3084                 spin_unlock(&vcpu->kvm->mmu_lock);
3085                 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
3086                 return;
3087         }
3088 
3089         spin_lock(&vcpu->kvm->mmu_lock);
3090         for (i = 0; i < 4; ++i) {
3091                 hpa_t root = vcpu->arch.mmu.pae_root[i];
3092 
3093                 if (root) {
3094                         root &= PT64_BASE_ADDR_MASK;
3095                         sp = page_header(root);
3096                         --sp->root_count;
3097                         if (!sp->root_count && sp->role.invalid)
3098                                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
3099                                                          &invalid_list);
3100                 }
3101                 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
3102         }
3103         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
3104         spin_unlock(&vcpu->kvm->mmu_lock);
3105         vcpu->arch.mmu.root_hpa = INVALID_PAGE;
3106 }
3107 
3108 static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
3109 {
3110         int ret = 0;
3111 
3112         if (!kvm_is_visible_gfn(vcpu->kvm, root_gfn)) {
3113                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
3114                 ret = 1;
3115         }
3116 
3117         return ret;
3118 }
3119 
3120 static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
3121 {
3122         struct kvm_mmu_page *sp;
3123         unsigned i;
3124 
3125         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
3126                 spin_lock(&vcpu->kvm->mmu_lock);
3127                 make_mmu_pages_available(vcpu);
3128                 sp = kvm_mmu_get_page(vcpu, 0, 0, PT64_ROOT_LEVEL, 1, ACC_ALL);
3129                 ++sp->root_count;
3130                 spin_unlock(&vcpu->kvm->mmu_lock);
3131                 vcpu->arch.mmu.root_hpa = __pa(sp->spt);
3132         } else if (vcpu->arch.mmu.shadow_root_level == PT32E_ROOT_LEVEL) {
3133                 for (i = 0; i < 4; ++i) {
3134                         hpa_t root = vcpu->arch.mmu.pae_root[i];
3135 
3136                         MMU_WARN_ON(VALID_PAGE(root));
3137                         spin_lock(&vcpu->kvm->mmu_lock);
3138                         make_mmu_pages_available(vcpu);
3139                         sp = kvm_mmu_get_page(vcpu, i << (30 - PAGE_SHIFT),
3140                                         i << 30, PT32_ROOT_LEVEL, 1, ACC_ALL);
3141                         root = __pa(sp->spt);
3142                         ++sp->root_count;
3143                         spin_unlock(&vcpu->kvm->mmu_lock);
3144                         vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
3145                 }
3146                 vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
3147         } else
3148                 BUG();
3149 
3150         return 0;
3151 }
3152 
3153 static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
3154 {
3155         struct kvm_mmu_page *sp;
3156         u64 pdptr, pm_mask;
3157         gfn_t root_gfn;
3158         int i;
3159 
3160         root_gfn = vcpu->arch.mmu.get_cr3(vcpu) >> PAGE_SHIFT;
3161 
3162         if (mmu_check_root(vcpu, root_gfn))
3163                 return 1;
3164 
3165         /*
3166          * Do we shadow a long mode page table? If so we need to
3167          * write-protect the guests page table root.
3168          */
3169         if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
3170                 hpa_t root = vcpu->arch.mmu.root_hpa;
3171 
3172                 MMU_WARN_ON(VALID_PAGE(root));
3173 
3174                 spin_lock(&vcpu->kvm->mmu_lock);
3175                 make_mmu_pages_available(vcpu);
3176                 sp = kvm_mmu_get_page(vcpu, root_gfn, 0, PT64_ROOT_LEVEL,
3177                                       0, ACC_ALL);
3178                 root = __pa(sp->spt);
3179                 ++sp->root_count;
3180                 spin_unlock(&vcpu->kvm->mmu_lock);
3181                 vcpu->arch.mmu.root_hpa = root;
3182                 return 0;
3183         }
3184 
3185         /*
3186          * We shadow a 32 bit page table. This may be a legacy 2-level
3187          * or a PAE 3-level page table. In either case we need to be aware that
3188          * the shadow page table may be a PAE or a long mode page table.
3189          */
3190         pm_mask = PT_PRESENT_MASK;
3191         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL)
3192                 pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;
3193 
3194         for (i = 0; i < 4; ++i) {
3195                 hpa_t root = vcpu->arch.mmu.pae_root[i];
3196 
3197                 MMU_WARN_ON(VALID_PAGE(root));
3198                 if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
3199                         pdptr = vcpu->arch.mmu.get_pdptr(vcpu, i);
3200                         if (!(pdptr & PT_PRESENT_MASK)) {
3201                                 vcpu->arch.mmu.pae_root[i] = 0;
3202                                 continue;
3203                         }
3204                         root_gfn = pdptr >> PAGE_SHIFT;
3205                         if (mmu_check_root(vcpu, root_gfn))
3206                                 return 1;
3207                 }
3208                 spin_lock(&vcpu->kvm->mmu_lock);
3209                 make_mmu_pages_available(vcpu);
3210                 sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30, PT32_ROOT_LEVEL,
3211                                       0, ACC_ALL);
3212                 root = __pa(sp->spt);
3213                 ++sp->root_count;
3214                 spin_unlock(&vcpu->kvm->mmu_lock);
3215 
3216                 vcpu->arch.mmu.pae_root[i] = root | pm_mask;
3217         }
3218         vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
3219 
3220         /*
3221          * If we shadow a 32 bit page table with a long mode page
3222          * table we enter this path.
3223          */
3224         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
3225                 if (vcpu->arch.mmu.lm_root == NULL) {
3226                         /*
3227                          * The additional page necessary for this is only
3228                          * allocated on demand.
3229                          */
3230 
3231                         u64 *lm_root;
3232 
3233                         lm_root = (void*)get_zeroed_page(GFP_KERNEL);
3234                         if (lm_root == NULL)
3235                                 return 1;
3236 
3237                         lm_root[0] = __pa(vcpu->arch.mmu.pae_root) | pm_mask;
3238 
3239                         vcpu->arch.mmu.lm_root = lm_root;
3240                 }
3241 
3242                 vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.lm_root);
3243         }
3244 
3245         return 0;
3246 }
3247 
3248 static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
3249 {
3250         if (vcpu->arch.mmu.direct_map)
3251                 return mmu_alloc_direct_roots(vcpu);
3252         else
3253                 return mmu_alloc_shadow_roots(vcpu);
3254 }
3255 
3256 static void mmu_sync_roots(struct kvm_vcpu *vcpu)
3257 {
3258         int i;
3259         struct kvm_mmu_page *sp;
3260 
3261         if (vcpu->arch.mmu.direct_map)
3262                 return;
3263 
3264         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
3265                 return;
3266 
3267         vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
3268         kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
3269         if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
3270                 hpa_t root = vcpu->arch.mmu.root_hpa;
3271                 sp = page_header(root);
3272                 mmu_sync_children(vcpu, sp);
3273                 kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3274                 return;
3275         }
3276         for (i = 0; i < 4; ++i) {
3277                 hpa_t root = vcpu->arch.mmu.pae_root[i];
3278 
3279                 if (root && VALID_PAGE(root)) {
3280                         root &= PT64_BASE_ADDR_MASK;
3281                         sp = page_header(root);
3282                         mmu_sync_children(vcpu, sp);
3283                 }
3284         }
3285         kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3286 }
3287 
3288 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
3289 {
3290         spin_lock(&vcpu->kvm->mmu_lock);
3291         mmu_sync_roots(vcpu);
3292         spin_unlock(&vcpu->kvm->mmu_lock);
3293 }
3294 EXPORT_SYMBOL_GPL(kvm_mmu_sync_roots);
3295 
3296 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr,
3297                                   u32 access, struct x86_exception *exception)
3298 {
3299         if (exception)
3300                 exception->error_code = 0;
3301         return vaddr;
3302 }
3303 
3304 static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gva_t vaddr,
3305                                          u32 access,
3306                                          struct x86_exception *exception)
3307 {
3308         if (exception)
3309                 exception->error_code = 0;
3310         return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access, exception);
3311 }
3312 
3313 static bool
3314 __is_rsvd_bits_set(struct rsvd_bits_validate *rsvd_check, u64 pte, int level)
3315 {
3316         int bit7 = (pte >> 7) & 1, low6 = pte & 0x3f;
3317 
3318         return (pte & rsvd_check->rsvd_bits_mask[bit7][level-1]) |
3319                 ((rsvd_check->bad_mt_xwr & (1ull << low6)) != 0);
3320 }
3321 
3322 static bool is_rsvd_bits_set(struct kvm_mmu *mmu, u64 gpte, int level)
3323 {
3324         return __is_rsvd_bits_set(&mmu->guest_rsvd_check, gpte, level);
3325 }
3326 
3327 static bool is_shadow_zero_bits_set(struct kvm_mmu *mmu, u64 spte, int level)
3328 {
3329         return __is_rsvd_bits_set(&mmu->shadow_zero_check, spte, level);
3330 }
3331 
3332 static bool mmio_info_in_cache(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3333 {
3334         if (direct)
3335                 return vcpu_match_mmio_gpa(vcpu, addr);
3336 
3337         return vcpu_match_mmio_gva(vcpu, addr);
3338 }
3339 
3340 /* return true if reserved bit is detected on spte. */
3341 static bool
3342 walk_shadow_page_get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep)
3343 {
3344         struct kvm_shadow_walk_iterator iterator;
3345         u64 sptes[PT64_ROOT_LEVEL], spte = 0ull;
3346         int root, leaf;
3347         bool reserved = false;
3348 
3349         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
3350                 goto exit;
3351 
3352         walk_shadow_page_lockless_begin(vcpu);
3353 
3354         for (shadow_walk_init(&iterator, vcpu, addr),
3355                  leaf = root = iterator.level;
3356              shadow_walk_okay(&iterator);
3357              __shadow_walk_next(&iterator, spte)) {
3358                 spte = mmu_spte_get_lockless(iterator.sptep);
3359 
3360                 sptes[leaf - 1] = spte;
3361                 leaf--;
3362 
3363                 if (!is_shadow_present_pte(spte))
3364                         break;
3365 
3366                 reserved |= is_shadow_zero_bits_set(&vcpu->arch.mmu, spte,
3367                                                     iterator.level);
3368         }
3369 
3370         walk_shadow_page_lockless_end(vcpu);
3371 
3372         if (reserved) {
3373                 pr_err("%s: detect reserved bits on spte, addr 0x%llx, dump hierarchy:\n",
3374                        __func__, addr);
3375                 while (root > leaf) {
3376                         pr_err("------ spte 0x%llx level %d.\n",
3377                                sptes[root - 1], root);
3378                         root--;
3379                 }
3380         }
3381 exit:
3382         *sptep = spte;
3383         return reserved;
3384 }
3385 
3386 int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3387 {
3388         u64 spte;
3389         bool reserved;
3390 
3391         if (mmio_info_in_cache(vcpu, addr, direct))
3392                 return RET_MMIO_PF_EMULATE;
3393 
3394         reserved = walk_shadow_page_get_mmio_spte(vcpu, addr, &spte);
3395         if (WARN_ON(reserved))
3396                 return RET_MMIO_PF_BUG;
3397 
3398         if (is_mmio_spte(spte)) {
3399                 gfn_t gfn = get_mmio_spte_gfn(spte);
3400                 unsigned access = get_mmio_spte_access(spte);
3401 
3402                 if (!check_mmio_spte(vcpu, spte))
3403                         return RET_MMIO_PF_INVALID;
3404 
3405                 if (direct)
3406                         addr = 0;
3407 
3408                 trace_handle_mmio_page_fault(addr, gfn, access);
3409                 vcpu_cache_mmio_info(vcpu, addr, gfn, access);
3410                 return RET_MMIO_PF_EMULATE;
3411         }
3412 
3413         /*
3414          * If the page table is zapped by other cpus, let CPU fault again on
3415          * the address.
3416          */
3417         return RET_MMIO_PF_RETRY;
3418 }
3419 EXPORT_SYMBOL_GPL(handle_mmio_page_fault);
3420 
3421 static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu,
3422                                          u32 error_code, gfn_t gfn)
3423 {
3424         if (unlikely(error_code & PFERR_RSVD_MASK))
3425                 return false;
3426 
3427         if (!(error_code & PFERR_PRESENT_MASK) ||
3428               !(error_code & PFERR_WRITE_MASK))
3429                 return false;
3430 
3431         /*
3432          * guest is writing the page which is write tracked which can
3433          * not be fixed by page fault handler.
3434          */
3435         if (kvm_page_track_is_active(vcpu, gfn, KVM_PAGE_TRACK_WRITE))
3436                 return true;
3437 
3438         return false;
3439 }
3440 
3441 static void shadow_page_table_clear_flood(struct kvm_vcpu *vcpu, gva_t addr)
3442 {
3443         struct kvm_shadow_walk_iterator iterator;
3444         u64 spte;
3445 
3446         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
3447                 return;
3448 
3449         walk_shadow_page_lockless_begin(vcpu);
3450         for_each_shadow_entry_lockless(vcpu, addr, iterator, spte) {
3451                 clear_sp_write_flooding_count(iterator.sptep);
3452                 if (!is_shadow_present_pte(spte))
3453                         break;
3454         }
3455         walk_shadow_page_lockless_end(vcpu);
3456 }
3457 
3458 static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
3459                                 u32 error_code, bool prefault)
3460 {
3461         gfn_t gfn = gva >> PAGE_SHIFT;
3462         int r;
3463 
3464         pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
3465 
3466         if (page_fault_handle_page_track(vcpu, error_code, gfn))
3467                 return 1;
3468 
3469         r = mmu_topup_memory_caches(vcpu);
3470         if (r)
3471                 return r;
3472 
3473         MMU_WARN_ON(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
3474 
3475 
3476         return nonpaging_map(vcpu, gva & PAGE_MASK,
3477                              error_code, gfn, prefault);
3478 }
3479 
3480 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn)
3481 {
3482         struct kvm_arch_async_pf arch;
3483 
3484         arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
3485         arch.gfn = gfn;
3486         arch.direct_map = vcpu->arch.mmu.direct_map;
3487         arch.cr3 = vcpu->arch.mmu.get_cr3(vcpu);
3488 
3489         return kvm_setup_async_pf(vcpu, gva, kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch);
3490 }
3491 
3492 static bool can_do_async_pf(struct kvm_vcpu *vcpu)
3493 {
3494         if (unlikely(!lapic_in_kernel(vcpu) ||
3495                      kvm_event_needs_reinjection(vcpu)))
3496                 return false;
3497 
3498         return kvm_x86_ops->interrupt_allowed(vcpu);
3499 }
3500 
3501 static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
3502                          gva_t gva, kvm_pfn_t *pfn, bool write, bool *writable)
3503 {
3504         struct kvm_memory_slot *slot;
3505         bool async;
3506 
3507         slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
3508         async = false;
3509         *pfn = __gfn_to_pfn_memslot(slot, gfn, false, &async, write, writable);
3510         if (!async)
3511                 return false; /* *pfn has correct page already */
3512 
3513         if (!prefault && can_do_async_pf(vcpu)) {
3514                 trace_kvm_try_async_get_page(gva, gfn);
3515                 if (kvm_find_async_pf_gfn(vcpu, gfn)) {
3516                         trace_kvm_async_pf_doublefault(gva, gfn);
3517                         kvm_make_request(KVM_REQ_APF_HALT, vcpu);
3518                         return true;
3519                 } else if (kvm_arch_setup_async_pf(vcpu, gva, gfn))
3520                         return true;
3521         }
3522 
3523         *pfn = __gfn_to_pfn_memslot(slot, gfn, false, NULL, write, writable);
3524         return false;
3525 }
3526 
3527 static bool
3528 check_hugepage_cache_consistency(struct kvm_vcpu *vcpu, gfn_t gfn, int level)
3529 {
3530         int page_num = KVM_PAGES_PER_HPAGE(level);
3531 
3532         gfn &= ~(page_num - 1);
3533 
3534         return kvm_mtrr_check_gfn_range_consistency(vcpu, gfn, page_num);
3535 }
3536 
3537 static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code,
3538                           bool prefault)
3539 {
3540         kvm_pfn_t pfn;
3541         int r;
3542         int level;
3543         bool force_pt_level;
3544         gfn_t gfn = gpa >> PAGE_SHIFT;
3545         unsigned long mmu_seq;
3546         int write = error_code & PFERR_WRITE_MASK;
3547         bool map_writable;
3548 
3549         MMU_WARN_ON(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
3550 
3551         if (page_fault_handle_page_track(vcpu, error_code, gfn))
3552                 return 1;
3553 
3554         r = mmu_topup_memory_caches(vcpu);
3555         if (r)
3556                 return r;
3557 
3558         force_pt_level = !check_hugepage_cache_consistency(vcpu, gfn,
3559                                                            PT_DIRECTORY_LEVEL);
3560         level = mapping_level(vcpu, gfn, &force_pt_level);
3561         if (likely(!force_pt_level)) {
3562                 if (level > PT_DIRECTORY_LEVEL &&
3563                     !check_hugepage_cache_consistency(vcpu, gfn, level))
3564                         level = PT_DIRECTORY_LEVEL;
3565                 gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
3566         }
3567 
3568         if (fast_page_fault(vcpu, gpa, level, error_code))
3569                 return 0;
3570 
3571         mmu_seq = vcpu->kvm->mmu_notifier_seq;
3572         smp_rmb();
3573 
3574         if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable))
3575                 return 0;
3576 
3577         if (handle_abnormal_pfn(vcpu, 0, gfn, pfn, ACC_ALL, &r))
3578                 return r;
3579 
3580         spin_lock(&vcpu->kvm->mmu_lock);
3581         if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
3582                 goto out_unlock;
3583         make_mmu_pages_available(vcpu);
3584         if (likely(!force_pt_level))
3585                 transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
3586         r = __direct_map(vcpu, write, map_writable, level, gfn, pfn, prefault);
3587         spin_unlock(&vcpu->kvm->mmu_lock);
3588 
3589         return r;
3590 
3591 out_unlock:
3592         spin_unlock(&vcpu->kvm->mmu_lock);
3593         kvm_release_pfn_clean(pfn);
3594         return 0;
3595 }
3596 
3597 static void nonpaging_init_context(struct kvm_vcpu *vcpu,
3598                                    struct kvm_mmu *context)
3599 {
3600         context->page_fault = nonpaging_page_fault;
3601         context->gva_to_gpa = nonpaging_gva_to_gpa;
3602         context->sync_page = nonpaging_sync_page;
3603         context->invlpg = nonpaging_invlpg;
3604         context->update_pte = nonpaging_update_pte;
3605         context->root_level = 0;
3606         context->shadow_root_level = PT32E_ROOT_LEVEL;
3607         context->root_hpa = INVALID_PAGE;
3608         context->direct_map = true;
3609         context->nx = false;
3610 }
3611 
3612 void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu)
3613 {
3614         mmu_free_roots(vcpu);
3615 }
3616 
3617 static unsigned long get_cr3(struct kvm_vcpu *vcpu)
3618 {
3619         return kvm_read_cr3(vcpu);
3620 }
3621 
3622 static void inject_page_fault(struct kvm_vcpu *vcpu,
3623                               struct x86_exception *fault)
3624 {
3625         vcpu->arch.mmu.inject_page_fault(vcpu, fault);
3626 }
3627 
3628 static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
3629                            unsigned access, int *nr_present)
3630 {
3631         if (unlikely(is_mmio_spte(*sptep))) {
3632                 if (gfn != get_mmio_spte_gfn(*sptep)) {
3633                         mmu_spte_clear_no_track(sptep);
3634                         return true;
3635                 }
3636 
3637                 (*nr_present)++;
3638                 mark_mmio_spte(vcpu, sptep, gfn, access);
3639                 return true;
3640         }
3641 
3642         return false;
3643 }
3644 
3645 static inline bool is_last_gpte(struct kvm_mmu *mmu,
3646                                 unsigned level, unsigned gpte)
3647 {
3648         /*
3649          * PT_PAGE_TABLE_LEVEL always terminates.  The RHS has bit 7 set
3650          * iff level <= PT_PAGE_TABLE_LEVEL, which for our purpose means
3651          * level == PT_PAGE_TABLE_LEVEL; set PT_PAGE_SIZE_MASK in gpte then.
3652          */
3653         gpte |= level - PT_PAGE_TABLE_LEVEL - 1;
3654 
3655         /*
3656          * The RHS has bit 7 set iff level < mmu->last_nonleaf_level.
3657          * If it is clear, there are no large pages at this level, so clear
3658          * PT_PAGE_SIZE_MASK in gpte if that is the case.
3659          */
3660         gpte &= level - mmu->last_nonleaf_level;
3661 
3662         return gpte & PT_PAGE_SIZE_MASK;
3663 }
3664 
3665 #define PTTYPE_EPT 18 /* arbitrary */
3666 #define PTTYPE PTTYPE_EPT
3667 #include "paging_tmpl.h"
3668 #undef PTTYPE
3669 
3670 #define PTTYPE 64
3671 #include "paging_tmpl.h"
3672 #undef PTTYPE
3673 
3674 #define PTTYPE 32
3675 #include "paging_tmpl.h"
3676 #undef PTTYPE
3677 
3678 static void
3679 __reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
3680                         struct rsvd_bits_validate *rsvd_check,
3681                         int maxphyaddr, int level, bool nx, bool gbpages,
3682                         bool pse, bool amd)
3683 {
3684         u64 exb_bit_rsvd = 0;
3685         u64 gbpages_bit_rsvd = 0;
3686         u64 nonleaf_bit8_rsvd = 0;
3687 
3688         rsvd_check->bad_mt_xwr = 0;
3689 
3690         if (!nx)
3691                 exb_bit_rsvd = rsvd_bits(63, 63);
3692         if (!gbpages)
3693                 gbpages_bit_rsvd = rsvd_bits(7, 7);
3694 
3695         /*
3696          * Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for
3697          * leaf entries) on AMD CPUs only.
3698          */
3699         if (amd)
3700                 nonleaf_bit8_rsvd = rsvd_bits(8, 8);
3701 
3702         switch (level) {
3703         case PT32_ROOT_LEVEL:
3704                 /* no rsvd bits for 2 level 4K page table entries */
3705                 rsvd_check->rsvd_bits_mask[0][1] = 0;
3706                 rsvd_check->rsvd_bits_mask[0][0] = 0;
3707                 rsvd_check->rsvd_bits_mask[1][0] =
3708                         rsvd_check->rsvd_bits_mask[0][0];
3709 
3710                 if (!pse) {
3711                         rsvd_check->rsvd_bits_mask[1][1] = 0;
3712                         break;
3713                 }
3714 
3715                 if (is_cpuid_PSE36())
3716                         /* 36bits PSE 4MB page */
3717                         rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
3718                 else
3719                         /* 32 bits PSE 4MB page */
3720                         rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
3721                 break;
3722         case PT32E_ROOT_LEVEL:
3723                 rsvd_check->rsvd_bits_mask[0][2] =
3724                         rsvd_bits(maxphyaddr, 63) |
3725                         rsvd_bits(5, 8) | rsvd_bits(1, 2);      /* PDPTE */
3726                 rsvd_check->rsvd_bits_mask[0][1] = exb_bit_rsvd |
3727                         rsvd_bits(maxphyaddr, 62);      /* PDE */
3728                 rsvd_check->rsvd_bits_mask[0][0] = exb_bit_rsvd |
3729                         rsvd_bits(maxphyaddr, 62);      /* PTE */
3730                 rsvd_check->rsvd_bits_mask[1][1] = exb_bit_rsvd |
3731                         rsvd_bits(maxphyaddr, 62) |
3732                         rsvd_bits(13, 20);              /* large page */
3733                 rsvd_check->rsvd_bits_mask[1][0] =
3734                         rsvd_check->rsvd_bits_mask[0][0];
3735                 break;
3736         case PT64_ROOT_LEVEL:
3737                 rsvd_check->rsvd_bits_mask[0][3] = exb_bit_rsvd |
3738                         nonleaf_bit8_rsvd | rsvd_bits(7, 7) |
3739                         rsvd_bits(maxphyaddr, 51);
3740                 rsvd_check->rsvd_bits_mask[0][2] = exb_bit_rsvd |
3741                         nonleaf_bit8_rsvd | gbpages_bit_rsvd |
3742                         rsvd_bits(maxphyaddr, 51);
3743                 rsvd_check->rsvd_bits_mask[0][1] = exb_bit_rsvd |
3744                         rsvd_bits(maxphyaddr, 51);
3745                 rsvd_check->rsvd_bits_mask[0][0] = exb_bit_rsvd |
3746                         rsvd_bits(maxphyaddr, 51);
3747                 rsvd_check->rsvd_bits_mask[1][3] =
3748                         rsvd_check->rsvd_bits_mask[0][3];
3749                 rsvd_check->rsvd_bits_mask[1][2] = exb_bit_rsvd |
3750                         gbpages_bit_rsvd | rsvd_bits(maxphyaddr, 51) |
3751                         rsvd_bits(13, 29);
3752                 rsvd_check->rsvd_bits_mask[1][1] = exb_bit_rsvd |
3753                         rsvd_bits(maxphyaddr, 51) |
3754                         rsvd_bits(13, 20);              /* large page */
3755                 rsvd_check->rsvd_bits_mask[1][0] =
3756                         rsvd_check->rsvd_bits_mask[0][0];
3757                 break;
3758         }
3759 }
3760 
3761 static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
3762                                   struct kvm_mmu *context)
3763 {
3764         __reset_rsvds_bits_mask(vcpu, &context->guest_rsvd_check,
3765                                 cpuid_maxphyaddr(vcpu), context->root_level,
3766                                 context->nx, guest_cpuid_has_gbpages(vcpu),
3767                                 is_pse(vcpu), guest_cpuid_is_amd(vcpu));
3768 }
3769 
3770 static void
3771 __reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
3772                             int maxphyaddr, bool execonly)
3773 {
3774         u64 bad_mt_xwr;
3775 
3776         rsvd_check->rsvd_bits_mask[0][3] =
3777                 rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 7);
3778         rsvd_check->rsvd_bits_mask[0][2] =
3779                 rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 6);
3780         rsvd_check->rsvd_bits_mask[0][1] =
3781                 rsvd_bits(maxphyaddr, 51) | rsvd_bits(3, 6);
3782         rsvd_check->rsvd_bits_mask[0][0] = rsvd_bits(maxphyaddr, 51);
3783 
3784         /* large page */
3785         rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3];
3786         rsvd_check->rsvd_bits_mask[1][2] =
3787                 rsvd_bits(maxphyaddr, 51) | rsvd_bits(12, 29);
3788         rsvd_check->rsvd_bits_mask[1][1] =
3789                 rsvd_bits(maxphyaddr, 51) | rsvd_bits(12, 20);
3790         rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0];
3791 
3792         bad_mt_xwr = 0xFFull << (2 * 8);        /* bits 3..5 must not be 2 */
3793         bad_mt_xwr |= 0xFFull << (3 * 8);       /* bits 3..5 must not be 3 */
3794         bad_mt_xwr |= 0xFFull << (7 * 8);       /* bits 3..5 must not be 7 */
3795         bad_mt_xwr |= REPEAT_BYTE(1ull << 2);   /* bits 0..2 must not be 010 */
3796         bad_mt_xwr |= REPEAT_BYTE(1ull << 6);   /* bits 0..2 must not be 110 */
3797         if (!execonly) {
3798                 /* bits 0..2 must not be 100 unless VMX capabilities allow it */
3799                 bad_mt_xwr |= REPEAT_BYTE(1ull << 4);
3800         }
3801         rsvd_check->bad_mt_xwr = bad_mt_xwr;
3802 }
3803 
3804 static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu,
3805                 struct kvm_mmu *context, bool execonly)
3806 {
3807         __reset_rsvds_bits_mask_ept(&context->guest_rsvd_check,
3808                                     cpuid_maxphyaddr(vcpu), execonly);
3809 }
3810 
3811 /*
3812  * the page table on host is the shadow page table for the page
3813  * table in guest or amd nested guest, its mmu features completely
3814  * follow the features in guest.
3815  */
3816 void
3817 reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, struct kvm_mmu *context)
3818 {
3819         bool uses_nx = context->nx || context->base_role.smep_andnot_wp;
3820 
3821         /*
3822          * Passing "true" to the last argument is okay; it adds a check
3823          * on bit 8 of the SPTEs which KVM doesn't use anyway.
3824          */
3825         __reset_rsvds_bits_mask(vcpu, &context->shadow_zero_check,
3826                                 boot_cpu_data.x86_phys_bits,
3827                                 context->shadow_root_level, uses_nx,
3828                                 guest_cpuid_has_gbpages(vcpu), is_pse(vcpu),
3829                                 true);
3830 }
3831 EXPORT_SYMBOL_GPL(reset_shadow_zero_bits_mask);
3832 
3833 static inline bool boot_cpu_is_amd(void)
3834 {
3835         WARN_ON_ONCE(!tdp_enabled);
3836         return shadow_x_mask == 0;
3837 }
3838 
3839 /*
3840  * the direct page table on host, use as much mmu features as
3841  * possible, however, kvm currently does not do execution-protection.
3842  */
3843 static void
3844 reset_tdp_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
3845                                 struct kvm_mmu *context)
3846 {
3847         if (boot_cpu_is_amd())
3848                 __reset_rsvds_bits_mask(vcpu, &context->shadow_zero_check,
3849                                         boot_cpu_data.x86_phys_bits,
3850                                         context->shadow_root_level, false,
3851                                         boot_cpu_has(X86_FEATURE_GBPAGES),
3852                                         true, true);
3853         else
3854                 __reset_rsvds_bits_mask_ept(&context->shadow_zero_check,
3855                                             boot_cpu_data.x86_phys_bits,
3856                                             false);
3857 
3858 }
3859 
3860 /*
3861  * as the comments in reset_shadow_zero_bits_mask() except it
3862  * is the shadow page table for intel nested guest.
3863  */
3864 static void
3865 reset_ept_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
3866                                 struct kvm_mmu *context, bool execonly)
3867 {
3868         __reset_rsvds_bits_mask_ept(&context->shadow_zero_check,
3869                                     boot_cpu_data.x86_phys_bits, execonly);
3870 }
3871 
3872 static void update_permission_bitmask(struct kvm_vcpu *vcpu,
3873                                       struct kvm_mmu *mmu, bool ept)
3874 {
3875         unsigned bit, byte, pfec;
3876         u8 map;
3877         bool fault, x, w, u, wf, uf, ff, smapf, cr4_smap, cr4_smep, smap = 0;
3878 
3879         cr4_smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
3880         cr4_smap = kvm_read_cr4_bits(vcpu, X86_CR4_SMAP);
3881         for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {
3882                 pfec = byte << 1;
3883                 map = 0;
3884                 wf = pfec & PFERR_WRITE_MASK;
3885                 uf = pfec & PFERR_USER_MASK;
3886                 ff = pfec & PFERR_FETCH_MASK;
3887                 /*
3888                  * PFERR_RSVD_MASK bit is set in PFEC if the access is not
3889                  * subject to SMAP restrictions, and cleared otherwise. The
3890                  * bit is only meaningful if the SMAP bit is set in CR4.
3891                  */
3892                 smapf = !(pfec & PFERR_RSVD_MASK);
3893                 for (bit = 0; bit < 8; ++bit) {
3894                         x = bit & ACC_EXEC_MASK;
3895                         w = bit & ACC_WRITE_MASK;
3896                         u = bit & ACC_USER_MASK;
3897 
3898                         if (!ept) {
3899                                 /* Not really needed: !nx will cause pte.nx to fault */
3900                                 x |= !mmu->nx;
3901                                 /* Allow supervisor writes if !cr0.wp */
3902                                 w |= !is_write_protection(vcpu) && !uf;
3903                                 /* Disallow supervisor fetches of user code if cr4.smep */
3904                                 x &= !(cr4_smep && u && !uf);
3905 
3906                                 /*
3907                                  * SMAP:kernel-mode data accesses from user-mode
3908                                  * mappings should fault. A fault is considered
3909                                  * as a SMAP violation if all of the following
3910                                  * conditions are ture:
3911                                  *   - X86_CR4_SMAP is set in CR4
3912                                  *   - An user page is accessed
3913                                  *   - Page fault in kernel mode
3914                                  *   - if CPL = 3 or X86_EFLAGS_AC is clear
3915                                  *
3916                                  *   Here, we cover the first three conditions.
3917                                  *   The fourth is computed dynamically in
3918                                  *   permission_fault() and is in smapf.
3919                                  *
3920                                  *   Also, SMAP does not affect instruction
3921                                  *   fetches, add the !ff check here to make it
3922                                  *   clearer.
3923                                  */
3924                                 smap = cr4_smap && u && !uf && !ff;
3925                         }
3926 
3927                         fault = (ff && !x) || (uf && !u) || (wf && !w) ||
3928                                 (smapf && smap);
3929                         map |= fault << bit;
3930                 }
3931                 mmu->permissions[byte] = map;
3932         }
3933 }
3934 
3935 /*
3936 * PKU is an additional mechanism by which the paging controls access to
3937 * user-mode addresses based on the value in the PKRU register.  Protection
3938 * key violations are reported through a bit in the page fault error code.
3939 * Unlike other bits of the error code, the PK bit is not known at the
3940 * call site of e.g. gva_to_gpa; it must be computed directly in
3941 * permission_fault based on two bits of PKRU, on some machine state (CR4,
3942 * CR0, EFER, CPL), and on other bits of the error code and the page tables.
3943 *
3944 * In particular the following conditions come from the error code, the
3945 * page tables and the machine state:
3946 * - PK is always zero unless CR4.PKE=1 and EFER.LMA=1
3947 * - PK is always zero if RSVD=1 (reserved bit set) or F=1 (instruction fetch)
3948 * - PK is always zero if U=0 in the page tables
3949 * - PKRU.WD is ignored if CR0.WP=0 and the access is a supervisor access.
3950 *
3951 * The PKRU bitmask caches the result of these four conditions.  The error
3952 * code (minus the P bit) and the page table's U bit form an index into the
3953 * PKRU bitmask.  Two bits of the PKRU bitmask are then extracted and ANDed
3954 * with the two bits of the PKRU register corresponding to the protection key.
3955 * For the first three conditions above the bits will be 00, thus masking
3956 * away both AD and WD.  For all reads or if the last condition holds, WD
3957 * only will be masked away.
3958 */
3959 static void update_pkru_bitmask(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
3960                                 bool ept)
3961 {
3962         unsigned bit;
3963         bool wp;
3964 
3965         if (ept) {
3966                 mmu->pkru_mask = 0;
3967                 return;
3968         }
3969 
3970         /* PKEY is enabled only if CR4.PKE and EFER.LMA are both set. */
3971         if (!kvm_read_cr4_bits(vcpu, X86_CR4_PKE) || !is_long_mode(vcpu)) {
3972                 mmu->pkru_mask = 0;
3973                 return;
3974         }
3975 
3976         wp = is_write_protection(vcpu);
3977 
3978         for (bit = 0; bit < ARRAY_SIZE(mmu->permissions); ++bit) {
3979                 unsigned pfec, pkey_bits;
3980                 bool check_pkey, check_write, ff, uf, wf, pte_user;
3981 
3982                 pfec = bit << 1;
3983                 ff = pfec & PFERR_FETCH_MASK;
3984                 uf = pfec & PFERR_USER_MASK;
3985                 wf = pfec & PFERR_WRITE_MASK;
3986 
3987                 /* PFEC.RSVD is replaced by ACC_USER_MASK. */
3988                 pte_user = pfec & PFERR_RSVD_MASK;
3989 
3990                 /*
3991                  * Only need to check the access which is not an
3992                  * instruction fetch and is to a user page.
3993                  */
3994                 check_pkey = (!ff && pte_user);
3995                 /*
3996                  * write access is controlled by PKRU if it is a
3997                  * user access or CR0.WP = 1.
3998                  */
3999                 check_write = check_pkey && wf && (uf || wp);
4000 
4001                 /* PKRU.AD stops both read and write access. */
4002                 pkey_bits = !!check_pkey;
4003                 /* PKRU.WD stops write access. */
4004                 pkey_bits |= (!!check_write) << 1;
4005 
4006                 mmu->pkru_mask |= (pkey_bits & 3) << pfec;
4007         }
4008 }
4009 
4010 static void update_last_nonleaf_level(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
4011 {
4012         unsigned root_level = mmu->root_level;
4013 
4014         mmu->last_nonleaf_level = root_level;
4015         if (root_level == PT32_ROOT_LEVEL && is_pse(vcpu))
4016                 mmu->last_nonleaf_level++;
4017 }
4018 
4019 static void paging64_init_context_common(struct kvm_vcpu *vcpu,
4020                                          struct kvm_mmu *context,
4021                                          int level)
4022 {
4023         context->nx = is_nx(vcpu);
4024         context->root_level = level;
4025 
4026         reset_rsvds_bits_mask(vcpu, context);
4027         update_permission_bitmask(vcpu, context, false);
4028         update_pkru_bitmask(vcpu, context, false);
4029         update_last_nonleaf_level(vcpu, context);
4030 
4031         MMU_WARN_ON(!is_pae(vcpu));
4032         context->page_fault = paging64_page_fault;
4033         context->gva_to_gpa = paging64_gva_to_gpa;
4034         context->sync_page = paging64_sync_page;
4035         context->invlpg = paging64_invlpg;
4036         context->update_pte = paging64_update_pte;
4037         context->shadow_root_level = level;
4038         context->root_hpa = INVALID_PAGE;
4039         context->direct_map = false;
4040 }
4041 
4042 static void paging64_init_context(struct kvm_vcpu *vcpu,
4043                                   struct kvm_mmu *context)
4044 {
4045         paging64_init_context_common(vcpu, context, PT64_ROOT_LEVEL);
4046 }
4047 
4048 static void paging32_init_context(struct kvm_vcpu *vcpu,
4049                                   struct kvm_mmu *context)
4050 {
4051         context->nx = false;
4052         context->root_level = PT32_ROOT_LEVEL;
4053 
4054         reset_rsvds_bits_mask(vcpu, context);
4055         update_permission_bitmask(vcpu, context, false);
4056         update_pkru_bitmask(vcpu, context, false);
4057         update_last_nonleaf_level(vcpu, context);
4058 
4059         context->page_fault = paging32_page_fault;
4060         context->gva_to_gpa = paging32_gva_to_gpa;
4061         context->sync_page = paging32_sync_page;
4062         context->invlpg = paging32_invlpg;
4063         context->update_pte = paging32_update_pte;
4064         context->shadow_root_level = PT32E_ROOT_LEVEL;
4065         context->root_hpa = INVALID_PAGE;
4066         context->direct_map = false;
4067 }
4068 
4069 static void paging32E_init_context(struct kvm_vcpu *vcpu,
4070                                    struct kvm_mmu *context)
4071 {
4072         paging64_init_context_common(vcpu, context, PT32E_ROOT_LEVEL);
4073 }
4074 
4075 static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
4076 {
4077         struct kvm_mmu *context = &vcpu->arch.mmu;
4078 
4079         context->base_role.word = 0;
4080         context->base_role.smm = is_smm(vcpu);
4081         context->page_fault = tdp_page_fault;
4082         context->sync_page = nonpaging_sync_page;
4083         context->invlpg = nonpaging_invlpg;
4084         context->update_pte = nonpaging_update_pte;
4085         context->shadow_root_level = kvm_x86_ops->get_tdp_level();
4086         context->root_hpa = INVALID_PAGE;
4087         context->direct_map = true;
4088         context->set_cr3 = kvm_x86_ops->set_tdp_cr3;
4089         context->get_cr3 = get_cr3;
4090         context->get_pdptr = kvm_pdptr_read;
4091         context->inject_page_fault = kvm_inject_page_fault;
4092 
4093         if (!is_paging(vcpu)) {
4094                 context->nx = false;
4095                 context->gva_to_gpa = nonpaging_gva_to_gpa;
4096                 context->root_level = 0;
4097         } else if (is_long_mode(vcpu)) {
4098                 context->nx = is_nx(vcpu);
4099                 context->root_level = PT64_ROOT_LEVEL;
4100                 reset_rsvds_bits_mask(vcpu, context);
4101                 context->gva_to_gpa = paging64_gva_to_gpa;
4102         } else if (is_pae(vcpu)) {
4103                 context->nx = is_nx(vcpu);
4104                 context->root_level = PT32E_ROOT_LEVEL;
4105                 reset_rsvds_bits_mask(vcpu, context);
4106                 context->gva_to_gpa = paging64_gva_to_gpa;
4107         } else {
4108                 context->nx = false;
4109                 context->root_level = PT32_ROOT_LEVEL;
4110                 reset_rsvds_bits_mask(vcpu, context);
4111                 context->gva_to_gpa = paging32_gva_to_gpa;
4112         }
4113 
4114         update_permission_bitmask(vcpu, context, false);
4115         update_pkru_bitmask(vcpu, context, false);
4116         update_last_nonleaf_level(vcpu, context);
4117         reset_tdp_shadow_zero_bits_mask(vcpu, context);
4118 }
4119 
4120 void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu)
4121 {
4122         bool smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
4123         bool smap = kvm_read_cr4_bits(vcpu, X86_CR4_SMAP);
4124         struct kvm_mmu *context = &vcpu->arch.mmu;
4125 
4126         MMU_WARN_ON(VALID_PAGE(context->root_hpa));
4127 
4128         if (!is_paging(vcpu))
4129                 nonpaging_init_context(vcpu, context);
4130         else if (is_long_mode(vcpu))
4131                 paging64_init_context(vcpu, context);
4132         else if (is_pae(vcpu))
4133                 paging32E_init_context(vcpu, context);
4134         else
4135                 paging32_init_context(vcpu, context);
4136 
4137         context->base_role.nxe = is_nx(vcpu);
4138         context->base_role.cr4_pae = !!is_pae(vcpu);
4139         context->base_role.cr0_wp  = is_write_protection(vcpu);
4140         context->base_role.smep_andnot_wp
4141                 = smep && !is_write_protection(vcpu);
4142         context->base_role.smap_andnot_wp
4143                 = smap && !is_write_protection(vcpu);
4144         context->base_role.smm = is_smm(vcpu);
4145         reset_shadow_zero_bits_mask(vcpu, context);
4146 }
4147 EXPORT_SYMBOL_GPL(kvm_init_shadow_mmu);
4148 
4149 void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly)
4150 {
4151         struct kvm_mmu *context = &vcpu->arch.mmu;
4152 
4153         MMU_WARN_ON(VALID_PAGE(context->root_hpa));
4154 
4155         context->shadow_root_level = kvm_x86_ops->get_tdp_level();
4156 
4157         context->nx = true;
4158         context->page_fault = ept_page_fault;
4159         context->gva_to_gpa = ept_gva_to_gpa;
4160         context->sync_page = ept_sync_page;
4161         context->invlpg = ept_invlpg;
4162         context->update_pte = ept_update_pte;
4163         context->root_level = context->shadow_root_level;
4164         context->root_hpa = INVALID_PAGE;
4165         context->direct_map = false;
4166 
4167         update_permission_bitmask(vcpu, context, true);
4168         update_pkru_bitmask(vcpu, context, true);
4169         reset_rsvds_bits_mask_ept(vcpu, context, execonly);
4170         reset_ept_shadow_zero_bits_mask(vcpu, context, execonly);
4171 }
4172 EXPORT_SYMBOL_GPL(kvm_init_shadow_ept_mmu);
4173 
4174 static void init_kvm_softmmu(struct kvm_vcpu *vcpu)
4175 {
4176         struct kvm_mmu *context = &vcpu->arch.mmu;
4177 
4178         kvm_init_shadow_mmu(vcpu);
4179         context->set_cr3           = kvm_x86_ops->set_cr3;
4180         context->get_cr3           = get_cr3;
4181         context->get_pdptr         = kvm_pdptr_read;
4182         context->inject_page_fault = kvm_inject_page_fault;
4183 }
4184 
4185 static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
4186 {
4187         struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;
4188 
4189         g_context->get_cr3           = get_cr3;
4190         g_context->get_pdptr         = kvm_pdptr_read;
4191         g_context->inject_page_fault = kvm_inject_page_fault;
4192 
4193         /*
4194          * Note that arch.mmu.gva_to_gpa translates l2_gpa to l1_gpa using
4195          * L1's nested page tables (e.g. EPT12). The nested translation
4196          * of l2_gva to l1_gpa is done by arch.nested_mmu.gva_to_gpa using
4197          * L2's page tables as the first level of translation and L1's
4198          * nested page tables as the second level of translation. Basically
4199          * the gva_to_gpa functions between mmu and nested_mmu are swapped.
4200          */
4201         if (!is_paging(vcpu)) {
4202                 g_context->nx = false;
4203                 g_context->root_level = 0;
4204                 g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested;
4205         } else if (is_long_mode(vcpu)) {
4206                 g_context->nx = is_nx(vcpu);
4207                 g_context->root_level = PT64_ROOT_LEVEL;
4208                 reset_rsvds_bits_mask(vcpu, g_context);
4209                 g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
4210         } else if (is_pae(vcpu)) {
4211                 g_context->nx = is_nx(vcpu);
4212                 g_context->root_level = PT32E_ROOT_LEVEL;
4213                 reset_rsvds_bits_mask(vcpu, g_context);
4214                 g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
4215         } else {
4216                 g_context->nx = false;
4217                 g_context->root_level = PT32_ROOT_LEVEL;
4218                 reset_rsvds_bits_mask(vcpu, g_context);
4219                 g_context->gva_to_gpa = paging32_gva_to_gpa_nested;
4220         }
4221 
4222         update_permission_bitmask(vcpu, g_context, false);
4223         update_pkru_bitmask(vcpu, g_context, false);
4224         update_last_nonleaf_level(vcpu, g_context);
4225 }
4226 
4227 static void init_kvm_mmu(struct kvm_vcpu *vcpu)
4228 {
4229         if (mmu_is_nested(vcpu))
4230                 init_kvm_nested_mmu(vcpu);
4231         else if (tdp_enabled)
4232                 init_kvm_tdp_mmu(vcpu);
4233         else
4234                 init_kvm_softmmu(vcpu);
4235 }
4236 
4237 void kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
4238 {
4239         kvm_mmu_unload(vcpu);
4240         init_kvm_mmu(vcpu);
4241 }
4242 EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
4243 
4244 int kvm_mmu_load(struct kvm_vcpu *vcpu)
4245 {
4246         int r;
4247 
4248         r = mmu_topup_memory_caches(vcpu);
4249         if (r)
4250                 goto out;
4251         r = mmu_alloc_roots(vcpu);
4252         kvm_mmu_sync_roots(vcpu);
4253         if (r)
4254                 goto out;
4255         /* set_cr3() should ensure TLB has been flushed */
4256         vcpu->arch.mmu.set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
4257 out:
4258         return r;
4259 }
4260 EXPORT_SYMBOL_GPL(kvm_mmu_load);
4261 
4262 void kvm_mmu_unload(struct kvm_vcpu *vcpu)
4263 {
4264         mmu_free_roots(vcpu);
4265         WARN_ON(VALID_PAGE(vcpu->arch.mmu.root_hpa));
4266 }
4267 EXPORT_SYMBOL_GPL(kvm_mmu_unload);
4268 
4269 static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
4270                                   struct kvm_mmu_page *sp, u64 *spte,
4271                                   const void *new)
4272 {
4273         if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
4274                 ++vcpu->kvm->stat.mmu_pde_zapped;
4275                 return;
4276         }
4277 
4278         ++vcpu->kvm->stat.mmu_pte_updated;
4279         vcpu->arch.mmu.update_pte(vcpu, sp, spte, new);
4280 }
4281 
4282 static bool need_remote_flush(u64 old, u64 new)
4283 {
4284         if (!is_shadow_present_pte(old))
4285                 return false;
4286         if (!is_shadow_present_pte(new))
4287                 return true;
4288         if ((old ^ new) & PT64_BASE_ADDR_MASK)
4289                 return true;
4290         old ^= shadow_nx_mask;
4291         new ^= shadow_nx_mask;
4292         return (old & ~new & PT64_PERM_MASK) != 0;
4293 }
4294 
4295 static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa,
4296                                     const u8 *new, int *bytes)
4297 {
4298         u64 gentry;
4299         int r;
4300 
4301         /*
4302          * Assume that the pte write on a page table of the same type
4303          * as the current vcpu paging mode since we update the sptes only
4304          * when they have the same mode.
4305          */
4306         if (is_pae(vcpu) && *bytes == 4) {
4307                 /* Handle a 32-bit guest writing two halves of a 64-bit gpte */
4308                 *gpa &= ~(gpa_t)7;
4309                 *bytes = 8;
4310                 r = kvm_vcpu_read_guest(vcpu, *gpa, &gentry, 8);
4311                 if (r)
4312                         gentry = 0;
4313                 new = (const u8 *)&gentry;
4314         }
4315 
4316         switch (*bytes) {
4317         case 4:
4318                 gentry = *(const u32 *)new;
4319                 break;
4320         case 8:
4321                 gentry = *(const u64 *)new;
4322                 break;
4323         default:
4324                 gentry = 0;
4325                 break;
4326         }
4327 
4328         return gentry;
4329 }
4330 
4331 /*
4332  * If we're seeing too many writes to a page, it may no longer be a page table,
4333  * or we may be forking, in which case it is better to unmap the page.
4334  */
4335 static bool detect_write_flooding(struct kvm_mmu_page *sp)
4336 {
4337         /*
4338          * Skip write-flooding detected for the sp whose level is 1, because
4339          * it can become unsync, then the guest page is not write-protected.
4340          */
4341         if (sp->role.level == PT_PAGE_TABLE_LEVEL)
4342                 return false;
4343 
4344         atomic_inc(&sp->write_flooding_count);
4345         return atomic_read(&sp->write_flooding_count) >= 3;
4346 }
4347 
4348 /*
4349  * Misaligned accesses are too much trouble to fix up; also, they usually
4350  * indicate a page is not used as a page table.
4351  */
4352 static bool detect_write_misaligned(struct kvm_mmu_page *sp, gpa_t gpa,
4353                                     int bytes)
4354 {
4355         unsigned offset, pte_size, misaligned;
4356 
4357         pgprintk("misaligned: gpa %llx bytes %d role %x\n",
4358                  gpa, bytes, sp->role.word);
4359 
4360         offset = offset_in_page(gpa);
4361         pte_size = sp->role.cr4_pae ? 8 : 4;
4362 
4363         /*
4364          * Sometimes, the OS only writes the last one bytes to update status
4365          * bits, for example, in linux, andb instruction is used in clear_bit().
4366          */
4367         if (!(offset & (pte_size - 1)) && bytes == 1)
4368                 return false;
4369 
4370         misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
4371         misaligned |= bytes < 4;
4372 
4373         return misaligned;
4374 }
4375 
4376 static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte)
4377 {
4378         unsigned page_offset, quadrant;
4379         u64 *spte;
4380         int level;
4381 
4382         page_offset = offset_in_page(gpa);
4383         level = sp->role.level;
4384         *nspte = 1;
4385         if (!sp->role.cr4_pae) {
4386                 page_offset <<= 1;      /* 32->64 */
4387                 /*
4388                  * A 32-bit pde maps 4MB while the shadow pdes map
4389                  * only 2MB.  So we need to double the offset again
4390                  * and zap two pdes instead of one.
4391                  */
4392                 if (level == PT32_ROOT_LEVEL) {
4393                         page_offset &= ~7; /* kill rounding error */
4394                         page_offset <<= 1;
4395                         *nspte = 2;
4396                 }
4397                 quadrant = page_offset >> PAGE_SHIFT;
4398                 page_offset &= ~PAGE_MASK;
4399                 if (quadrant != sp->role.quadrant)
4400                         return NULL;
4401         }
4402 
4403         spte = &sp->spt[page_offset / sizeof(*spte)];
4404         return spte;
4405 }
4406 
4407 static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
4408                               const u8 *new, int bytes)
4409 {
4410         gfn_t gfn = gpa >> PAGE_SHIFT;
4411         struct kvm_mmu_page *sp;
4412         LIST_HEAD(invalid_list);
4413         u64 entry, gentry, *spte;
4414         int npte;
4415         bool remote_flush, local_flush;
4416         union kvm_mmu_page_role mask = { };
4417 
4418         mask.cr0_wp = 1;
4419         mask.cr4_pae = 1;
4420         mask.nxe = 1;
4421         mask.smep_andnot_wp = 1;
4422         mask.smap_andnot_wp = 1;
4423         mask.smm = 1;
4424 
4425         /*
4426          * If we don't have indirect shadow pages, it means no page is
4427          * write-protected, so we can exit simply.
4428          */
4429         if (!ACCESS_ONCE(vcpu->kvm->arch.indirect_shadow_pages))
4430                 return;
4431 
4432         remote_flush = local_flush = false;
4433 
4434         pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
4435 
4436         gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, new, &bytes);
4437 
4438         /*
4439          * No need to care whether allocation memory is successful
4440          * or not since pte prefetch is skiped if it does not have
4441          * enough objects in the cache.
4442          */
4443         mmu_topup_memory_caches(vcpu);
4444 
4445         spin_lock(&vcpu->kvm->mmu_lock);
4446         ++vcpu->kvm->stat.mmu_pte_write;
4447         kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE);
4448 
4449         for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) {
4450                 if (detect_write_misaligned(sp, gpa, bytes) ||
4451                       detect_write_flooding(sp)) {
4452                         kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
4453                         ++vcpu->kvm->stat.mmu_flooded;
4454                         continue;
4455                 }
4456 
4457                 spte = get_written_sptes(sp, gpa, &npte);
4458                 if (!spte)
4459                         continue;
4460 
4461                 local_flush = true;
4462                 while (npte--) {
4463                         entry = *spte;
4464                         mmu_page_zap_pte(vcpu->kvm, sp, spte);
4465                         if (gentry &&
4466                               !((sp->role.word ^ vcpu->arch.mmu.base_role.word)
4467                               & mask.word) && rmap_can_add(vcpu))
4468                                 mmu_pte_write_new_pte(vcpu, sp, spte, &gentry);
4469                         if (need_remote_flush(entry, *spte))
4470                                 remote_flush = true;
4471                         ++spte;
4472                 }
4473         }
4474         kvm_mmu_flush_or_zap(vcpu, &invalid_list, remote_flush, local_flush);
4475         kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE);
4476         spin_unlock(&vcpu->kvm->mmu_lock);
4477 }
4478 
4479 int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
4480 {
4481         gpa_t gpa;
4482         int r;
4483 
4484         if (vcpu->arch.mmu.direct_map)
4485                 return 0;
4486 
4487         gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);
4488 
4489         r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
4490 
4491         return r;
4492 }
4493 EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
4494 
4495 static void make_mmu_pages_available(struct kvm_vcpu *vcpu)
4496 {
4497         LIST_HEAD(invalid_list);
4498 
4499         if (likely(kvm_mmu_available_pages(vcpu->kvm) >= KVM_MIN_FREE_MMU_PAGES))
4500                 return;
4501 
4502         while (kvm_mmu_available_pages(vcpu->kvm) < KVM_REFILL_PAGES) {
4503                 if (!prepare_zap_oldest_mmu_page(vcpu->kvm, &invalid_list))
4504                         break;
4505 
4506                 ++vcpu->kvm->stat.mmu_recycled;
4507         }
4508         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
4509 }
4510 
4511 int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code,
4512                        void *insn, int insn_len)
4513 {
4514         int r, emulation_type = EMULTYPE_RETRY;
4515         enum emulation_result er;
4516         bool direct = vcpu->arch.mmu.direct_map || mmu_is_nested(vcpu);
4517 
4518         if (unlikely(error_code & PFERR_RSVD_MASK)) {
4519                 r = handle_mmio_page_fault(vcpu, cr2, direct);
4520                 if (r == RET_MMIO_PF_EMULATE) {
4521                         emulation_type = 0;
4522                         goto emulate;
4523                 }
4524                 if (r == RET_MMIO_PF_RETRY)
4525                         return 1;
4526                 if (r < 0)
4527                         return r;
4528         }
4529 
4530         r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code, false);
4531         if (r < 0)
4532                 return r;
4533         if (!r)
4534                 return 1;
4535 
4536         if (mmio_info_in_cache(vcpu, cr2, direct))
4537                 emulation_type = 0;
4538 emulate:
4539         er = x86_emulate_instruction(vcpu, cr2, emulation_type, insn, insn_len);
4540 
4541         switch (er) {
4542         case EMULATE_DONE:
4543                 return 1;
4544         case EMULATE_USER_EXIT:
4545                 ++vcpu->stat.mmio_exits;
4546                 /* fall through */
4547         case EMULATE_FAIL:
4548                 return 0;
4549         default:
4550                 BUG();
4551         }
4552 }
4553 EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
4554 
4555 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
4556 {
4557         vcpu->arch.mmu.invlpg(vcpu, gva);
4558         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
4559         ++vcpu->stat.invlpg;
4560 }
4561 EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
4562 
4563 void kvm_enable_tdp(void)
4564 {
4565         tdp_enabled = true;
4566 }
4567 EXPORT_SYMBOL_GPL(kvm_enable_tdp);
4568 
4569 void kvm_disable_tdp(void)
4570 {
4571         tdp_enabled = false;
4572 }
4573 EXPORT_SYMBOL_GPL(kvm_disable_tdp);
4574 
4575 static void free_mmu_pages(struct kvm_vcpu *vcpu)
4576 {
4577         free_page((unsigned long)vcpu->arch.mmu.pae_root);
4578         if (vcpu->arch.mmu.lm_root != NULL)
4579                 free_page((unsigned long)vcpu->arch.mmu.lm_root);
4580 }
4581 
4582 static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
4583 {
4584         struct page *page;
4585         int i;
4586 
4587         /*
4588          * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
4589          * Therefore we need to allocate shadow page tables in the first
4590          * 4GB of memory, which happens to fit the DMA32 zone.
4591          */
4592         page = alloc_page(GFP_KERNEL | __GFP_DMA32);
4593         if (!page)
4594                 return -ENOMEM;
4595 
4596         vcpu->arch.mmu.pae_root = page_address(page);
4597         for (i = 0; i < 4; ++i)
4598                 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
4599 
4600         return 0;
4601 }
4602 
4603 int kvm_mmu_create(struct kvm_vcpu *vcpu)
4604 {
4605         vcpu->arch.walk_mmu = &vcpu->arch.mmu;
4606         vcpu->arch.mmu.root_hpa = INVALID_PAGE;
4607         vcpu->arch.mmu.translate_gpa = translate_gpa;
4608         vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;
4609 
4610         return alloc_mmu_pages(vcpu);
4611 }
4612 
4613 void kvm_mmu_setup(struct kvm_vcpu *vcpu)
4614 {
4615         MMU_WARN_ON(VALID_PAGE(vcpu->arch.mmu.root_hpa));
4616 
4617         init_kvm_mmu(vcpu);
4618 }
4619 
4620 void kvm_mmu_init_vm(struct kvm *kvm)
4621 {
4622         struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker;
4623 
4624         node->track_write = kvm_mmu_pte_write;
4625         kvm_page_track_register_notifier(kvm, node);
4626 }
4627 
4628 void kvm_mmu_uninit_vm(struct kvm *kvm)
4629 {
4630         struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker;
4631 
4632         kvm_page_track_unregister_notifier(kvm, node);
4633 }
4634 
4635 /* The return value indicates if tlb flush on all vcpus is needed. */
4636 typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head *rmap_head);
4637 
4638 /* The caller should hold mmu-lock before calling this function. */
4639 static bool
4640 slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot,
4641                         slot_level_handler fn, int start_level, int end_level,
4642                         gfn_t start_gfn, gfn_t end_gfn, bool lock_flush_tlb)
4643 {
4644         struct slot_rmap_walk_iterator iterator;
4645         bool flush = false;
4646 
4647         for_each_slot_rmap_range(memslot, start_level, end_level, start_gfn,
4648                         end_gfn, &iterator) {
4649                 if (iterator.rmap)
4650                         flush |= fn(kvm, iterator.rmap);
4651 
4652                 if (need_resched() || spin_needbreak(&kvm->mmu_lock)) {
4653                         if (flush && lock_flush_tlb) {
4654                                 kvm_flush_remote_tlbs(kvm);
4655                                 flush = false;
4656                         }
4657                         cond_resched_lock(&kvm->mmu_lock);
4658                 }
4659         }
4660 
4661         if (flush && lock_flush_tlb) {
4662                 kvm_flush_remote_tlbs(kvm);
4663                 flush = false;
4664         }
4665 
4666         return flush;
4667 }
4668 
4669 static bool
4670 slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
4671                   slot_level_handler fn, int start_level, int end_level,
4672                   bool lock_flush_tlb)
4673 {
4674         return slot_handle_level_range(kvm, memslot, fn, start_level,
4675                         end_level, memslot->base_gfn,
4676                         memslot->base_gfn + memslot->npages - 1,
4677                         lock_flush_tlb);
4678 }
4679 
4680 static bool
4681 slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
4682                       slot_level_handler fn, bool lock_flush_tlb)
4683 {
4684         return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL,
4685                                  PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
4686 }
4687 
4688 static bool
4689 slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
4690                         slot_level_handler fn, bool lock_flush_tlb)
4691 {
4692         return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL + 1,
4693                                  PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
4694 }
4695 
4696 static bool
4697 slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot,
4698                  slot_level_handler fn, bool lock_flush_tlb)
4699 {
4700         return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL,
4701                                  PT_PAGE_TABLE_LEVEL, lock_flush_tlb);
4702 }
4703 
4704 void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
4705 {
4706         struct kvm_memslots *slots;
4707         struct kvm_memory_slot *memslot;
4708         int i;
4709 
4710         spin_lock(&kvm->mmu_lock);
4711         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
4712                 slots = __kvm_memslots(kvm, i);
4713                 kvm_for_each_memslot(memslot, slots) {
4714                         gfn_t start, end;
4715 
4716                         start = max(gfn_start, memslot->base_gfn);
4717                         end = min(gfn_end, memslot->base_gfn + memslot->npages);
4718                         if (start >= end)
4719                                 continue;
4720 
4721                         slot_handle_level_range(kvm, memslot, kvm_zap_rmapp,
4722                                                 PT_PAGE_TABLE_LEVEL, PT_MAX_HUGEPAGE_LEVEL,
4723                                                 start, end - 1, true);
4724                 }
4725         }
4726 
4727         spin_unlock(&kvm->mmu_lock);
4728 }
4729 
4730 static bool slot_rmap_write_protect(struct kvm *kvm,
4731                                     struct kvm_rmap_head *rmap_head)
4732 {
4733         return __rmap_write_protect(kvm, rmap_head, false);
4734 }
4735 
4736 void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
4737                                       struct kvm_memory_slot *memslot)
4738 {
4739         bool flush;
4740 
4741         spin_lock(&kvm->mmu_lock);
4742         flush = slot_handle_all_level(kvm, memslot, slot_rmap_write_protect,
4743                                       false);
4744         spin_unlock(&kvm->mmu_lock);
4745 
4746         /*
4747          * kvm_mmu_slot_remove_write_access() and kvm_vm_ioctl_get_dirty_log()
4748          * which do tlb flush out of mmu-lock should be serialized by
4749          * kvm->slots_lock otherwise tlb flush would be missed.
4750          */
4751         lockdep_assert_held(&kvm->slots_lock);
4752 
4753         /*
4754          * We can flush all the TLBs out of the mmu lock without TLB
4755          * corruption since we just change the spte from writable to
4756          * readonly so that we only need to care the case of changing
4757          * spte from present to present (changing the spte from present
4758          * to nonpresent will flush all the TLBs immediately), in other
4759          * words, the only case we care is mmu_spte_update() where we
4760          * haved checked SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE
4761          * instead of PT_WRITABLE_MASK, that means it does not depend
4762          * on PT_WRITABLE_MASK anymore.
4763          */
4764         if (flush)
4765                 kvm_flush_remote_tlbs(kvm);
4766 }
4767 
4768 static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm,
4769                                          struct kvm_rmap_head *rmap_head)
4770 {
4771         u64 *sptep;
4772         struct rmap_iterator iter;
4773         int need_tlb_flush = 0;
4774         kvm_pfn_t pfn;
4775         struct kvm_mmu_page *sp;
4776 
4777 restart:
4778         for_each_rmap_spte(rmap_head, &iter, sptep) {
4779                 sp = page_header(__pa(sptep));
4780                 pfn = spte_to_pfn(*sptep);
4781 
4782                 /*
4783                  * We cannot do huge page mapping for indirect shadow pages,
4784                  * which are found on the last rmap (level = 1) when not using
4785                  * tdp; such shadow pages are synced with the page table in
4786                  * the guest, and the guest page table is using 4K page size
4787                  * mapping if the indirect sp has level = 1.
4788                  */
4789                 if (sp->role.direct &&
4790                         !kvm_is_reserved_pfn(pfn) &&
4791                         PageTransCompoundMap(pfn_to_page(pfn))) {
4792                         drop_spte(kvm, sptep);
4793                         need_tlb_flush = 1;
4794                         goto restart;
4795                 }
4796         }
4797 
4798         return need_tlb_flush;
4799 }
4800 
4801 void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
4802                                    const struct kvm_memory_slot *memslot)
4803 {
4804         /* FIXME: const-ify all uses of struct kvm_memory_slot.  */
4805         spin_lock(&kvm->mmu_lock);
4806         slot_handle_leaf(kvm, (struct kvm_memory_slot *)memslot,
4807                          kvm_mmu_zap_collapsible_spte, true);
4808         spin_unlock(&kvm->mmu_lock);
4809 }
4810 
4811 void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
4812                                    struct kvm_memory_slot *memslot)
4813 {
4814         bool flush;
4815 
4816         spin_lock(&kvm->mmu_lock);
4817         flush = slot_handle_leaf(kvm, memslot, __rmap_clear_dirty, false);
4818         spin_unlock(&kvm->mmu_lock);
4819 
4820         lockdep_assert_held(&kvm->slots_lock);
4821 
4822         /*
4823          * It's also safe to flush TLBs out of mmu lock here as currently this
4824          * function is only used for dirty logging, in which case flushing TLB
4825          * out of mmu lock also guarantees no dirty pages will be lost in
4826          * dirty_bitmap.
4827          */
4828         if (flush)
4829                 kvm_flush_remote_tlbs(kvm);
4830 }
4831 EXPORT_SYMBOL_GPL(kvm_mmu_slot_leaf_clear_dirty);
4832 
4833 void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm,
4834                                         struct kvm_memory_slot *memslot)
4835 {
4836         bool flush;
4837 
4838         spin_lock(&kvm->mmu_lock);
4839         flush = slot_handle_large_level(kvm, memslot, slot_rmap_write_protect,
4840                                         false);
4841         spin_unlock(&kvm->mmu_lock);
4842 
4843         /* see kvm_mmu_slot_remove_write_access */
4844         lockdep_assert_held(&kvm->slots_lock);
4845 
4846         if (flush)
4847                 kvm_flush_remote_tlbs(kvm);
4848 }
4849 EXPORT_SYMBOL_GPL(kvm_mmu_slot_largepage_remove_write_access);
4850 
4851 void kvm_mmu_slot_set_dirty(struct kvm *kvm,
4852                             struct kvm_memory_slot *memslot)
4853 {
4854         bool flush;
4855 
4856         spin_lock(&kvm->mmu_lock);
4857         flush = slot_handle_all_level(kvm, memslot, __rmap_set_dirty, false);
4858         spin_unlock(&kvm->mmu_lock);
4859 
4860         lockdep_assert_held(&kvm->slots_lock);
4861 
4862         /* see kvm_mmu_slot_leaf_clear_dirty */
4863         if (flush)
4864                 kvm_flush_remote_tlbs(kvm);
4865 }
4866 EXPORT_SYMBOL_GPL(kvm_mmu_slot_set_dirty);
4867 
4868 #define BATCH_ZAP_PAGES 10
4869 static void kvm_zap_obsolete_pages(struct kvm *kvm)
4870 {
4871         struct kvm_mmu_page *sp, *node;
4872         int batch = 0;
4873 
4874 restart:
4875         list_for_each_entry_safe_reverse(sp, node,
4876               &kvm->arch.active_mmu_pages, link) {
4877                 int ret;
4878 
4879                 /*
4880                  * No obsolete page exists before new created page since
4881                  * active_mmu_pages is the FIFO list.
4882                  */
4883                 if (!is_obsolete_sp(kvm, sp))
4884                         break;
4885 
4886                 /*
4887                  * Since we are reversely walking the list and the invalid
4888                  * list will be moved to the head, skip the invalid page
4889                  * can help us to avoid the infinity list walking.
4890                  */
4891                 if (sp->role.invalid)
4892                         continue;
4893 
4894                 /*
4895                  * Need not flush tlb since we only zap the sp with invalid
4896                  * generation number.
4897                  */
4898                 if (batch >= BATCH_ZAP_PAGES &&
4899                       cond_resched_lock(&kvm->mmu_lock)) {
4900                         batch = 0;
4901                         goto restart;
4902                 }
4903 
4904                 ret = kvm_mmu_prepare_zap_page(kvm, sp,
4905                                 &kvm->arch.zapped_obsolete_pages);
4906                 batch += ret;
4907 
4908                 if (ret)
4909                         goto restart;
4910         }
4911 
4912         /*
4913          * Should flush tlb before free page tables since lockless-walking
4914          * may use the pages.
4915          */
4916         kvm_mmu_commit_zap_page(kvm, &kvm->arch.zapped_obsolete_pages);
4917 }
4918 
4919 /*
4920  * Fast invalidate all shadow pages and use lock-break technique
4921  * to zap obsolete pages.
4922  *
4923  * It's required when memslot is being deleted or VM is being
4924  * destroyed, in these cases, we should ensure that KVM MMU does
4925  * not use any resource of the being-deleted slot or all slots
4926  * after calling the function.
4927  */
4928 void kvm_mmu_invalidate_zap_all_pages(struct kvm *kvm)
4929 {
4930         spin_lock(&kvm->mmu_lock);
4931         trace_kvm_mmu_invalidate_zap_all_pages(kvm);
4932         kvm->arch.mmu_valid_gen++;
4933 
4934         /*
4935          * Notify all vcpus to reload its shadow page table
4936          * and flush TLB. Then all vcpus will switch to new
4937          * shadow page table with the new mmu_valid_gen.
4938          *
4939          * Note: we should do this under the protection of
4940          * mmu-lock, otherwise, vcpu would purge shadow page
4941          * but miss tlb flush.
4942          */
4943         kvm_reload_remote_mmus(kvm);
4944 
4945         kvm_zap_obsolete_pages(kvm);
4946         spin_unlock(&kvm->mmu_lock);
4947 }
4948 
4949 static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm)
4950 {
4951         return unlikely(!list_empty_careful(&kvm->arch.zapped_obsolete_pages));
4952 }
4953 
4954 void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, struct kvm_memslots *slots)
4955 {
4956         /*
4957          * The very rare case: if the generation-number is round,
4958          * zap all shadow pages.
4959          */
4960         if (unlikely((slots->generation & MMIO_GEN_MASK) == 0)) {
4961                 printk_ratelimited(KERN_DEBUG "kvm: zapping shadow pages for mmio generation wraparound\n");
4962                 kvm_mmu_invalidate_zap_all_pages(kvm);
4963         }
4964 }
4965 
4966 static unsigned long
4967 mmu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
4968 {
4969         struct kvm *kvm;
4970         int nr_to_scan = sc->nr_to_scan;
4971         unsigned long freed = 0;
4972 
4973         spin_lock(&kvm_lock);
4974 
4975         list_for_each_entry(kvm, &vm_list, vm_list) {
4976                 int idx;
4977                 LIST_HEAD(invalid_list);
4978 
4979                 /*
4980                  * Never scan more than sc->nr_to_scan VM instances.
4981                  * Will not hit this condition practically since we do not try
4982                  * to shrink more than one VM and it is very unlikely to see
4983                  * !n_used_mmu_pages so many times.
4984                  */
4985                 if (!nr_to_scan--)
4986                         break;
4987                 /*
4988                  * n_used_mmu_pages is accessed without holding kvm->mmu_lock
4989                  * here. We may skip a VM instance errorneosly, but we do not
4990                  * want to shrink a VM that only started to populate its MMU
4991                  * anyway.
4992                  */
4993                 if (!kvm->arch.n_used_mmu_pages &&
4994                       !kvm_has_zapped_obsolete_pages(kvm))
4995                         continue;
4996 
4997                 idx = srcu_read_lock(&kvm->srcu);
4998                 spin_lock(&kvm->mmu_lock);
4999 
5000                 if (kvm_has_zapped_obsolete_pages(kvm)) {
5001                         kvm_mmu_commit_zap_page(kvm,
5002                               &kvm->arch.zapped_obsolete_pages);
5003                         goto unlock;
5004                 }
5005 
5006                 if (prepare_zap_oldest_mmu_page(kvm, &invalid_list))
5007                         freed++;
5008                 kvm_mmu_commit_zap_page(kvm, &invalid_list);
5009 
5010 unlock:
5011                 spin_unlock(&kvm->mmu_lock);
5012                 srcu_read_unlock(&kvm->srcu, idx);
5013 
5014                 /*
5015                  * unfair on small ones
5016                  * per-vm shrinkers cry out
5017                  * sadness comes quickly
5018                  */
5019                 list_move_tail(&kvm->vm_list, &vm_list);
5020                 break;
5021         }
5022 
5023         spin_unlock(&kvm_lock);
5024         return freed;
5025 }
5026 
5027 static unsigned long
5028 mmu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
5029 {
5030         return percpu_counter_read_positive(&kvm_total_used_mmu_pages);
5031 }
5032 
5033 static struct shrinker mmu_shrinker = {
5034         .count_objects = mmu_shrink_count,
5035         .scan_objects = mmu_shrink_scan,
5036         .seeks = DEFAULT_SEEKS * 10,
5037 };
5038 
5039 static void mmu_destroy_caches(void)
5040 {
5041         if (pte_list_desc_cache)
5042                 kmem_cache_destroy(pte_list_desc_cache);
5043         if (mmu_page_header_cache)
5044                 kmem_cache_destroy(mmu_page_header_cache);
5045 }
5046 
5047 int kvm_mmu_module_init(void)
5048 {
5049         pte_list_desc_cache = kmem_cache_create("pte_list_desc",
5050                                             sizeof(struct pte_list_desc),
5051                                             0, 0, NULL);
5052         if (!pte_list_desc_cache)
5053                 goto nomem;
5054 
5055         mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
5056                                                   sizeof(struct kvm_mmu_page),
5057                                                   0, 0, NULL);
5058         if (!mmu_page_header_cache)
5059                 goto nomem;
5060 
5061         if (percpu_counter_init(&kvm_total_used_mmu_pages, 0, GFP_KERNEL))
5062                 goto nomem;
5063 
5064         register_shrinker(&mmu_shrinker);
5065 
5066         return 0;
5067 
5068 nomem:
5069         mmu_destroy_caches();
5070         return -ENOMEM;
5071 }
5072 
5073 /*
5074  * Caculate mmu pages needed for kvm.
5075  */
5076 unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
5077 {
5078         unsigned int nr_mmu_pages;
5079         unsigned int  nr_pages = 0;
5080         struct kvm_memslots *slots;
5081         struct kvm_memory_slot *memslot;
5082         int i;
5083 
5084         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
5085                 slots = __kvm_memslots(kvm, i);
5086 
5087                 kvm_for_each_memslot(memslot, slots)
5088                         nr_pages += memslot->npages;
5089         }
5090 
5091         nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
5092         nr_mmu_pages = max(nr_mmu_pages,
5093                            (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
5094 
5095         return nr_mmu_pages;
5096 }
5097 
5098 void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
5099 {
5100         kvm_mmu_unload(vcpu);
5101         free_mmu_pages(vcpu);
5102         mmu_free_memory_caches(vcpu);
5103 }
5104 
5105 void kvm_mmu_module_exit(void)
5106 {
5107         mmu_destroy_caches();
5108         percpu_counter_destroy(&kvm_total_used_mmu_pages);
5109         unregister_shrinker(&mmu_shrinker);
5110         mmu_audit_disable();
5111 }
5112 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp