~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/kvm/vmx/nested.c

Version: ~ [ linux-5.8-rc4 ] ~ [ linux-5.7.7 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.50 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.131 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.187 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.229 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.229 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 
  3 #include <linux/frame.h>
  4 #include <linux/percpu.h>
  5 
  6 #include <asm/debugreg.h>
  7 #include <asm/mmu_context.h>
  8 
  9 #include "cpuid.h"
 10 #include "hyperv.h"
 11 #include "mmu.h"
 12 #include "nested.h"
 13 #include "trace.h"
 14 #include "x86.h"
 15 
 16 static bool __read_mostly enable_shadow_vmcs = 1;
 17 module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO);
 18 
 19 static bool __read_mostly nested_early_check = 0;
 20 module_param(nested_early_check, bool, S_IRUGO);
 21 
 22 /*
 23  * Hyper-V requires all of these, so mark them as supported even though
 24  * they are just treated the same as all-context.
 25  */
 26 #define VMX_VPID_EXTENT_SUPPORTED_MASK          \
 27         (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT |  \
 28         VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT |    \
 29         VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT |    \
 30         VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT)
 31 
 32 #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5
 33 
 34 enum {
 35         VMX_VMREAD_BITMAP,
 36         VMX_VMWRITE_BITMAP,
 37         VMX_BITMAP_NR
 38 };
 39 static unsigned long *vmx_bitmap[VMX_BITMAP_NR];
 40 
 41 #define vmx_vmread_bitmap                    (vmx_bitmap[VMX_VMREAD_BITMAP])
 42 #define vmx_vmwrite_bitmap                   (vmx_bitmap[VMX_VMWRITE_BITMAP])
 43 
 44 struct shadow_vmcs_field {
 45         u16     encoding;
 46         u16     offset;
 47 };
 48 static struct shadow_vmcs_field shadow_read_only_fields[] = {
 49 #define SHADOW_FIELD_RO(x, y) { x, offsetof(struct vmcs12, y) },
 50 #include "vmcs_shadow_fields.h"
 51 };
 52 static int max_shadow_read_only_fields =
 53         ARRAY_SIZE(shadow_read_only_fields);
 54 
 55 static struct shadow_vmcs_field shadow_read_write_fields[] = {
 56 #define SHADOW_FIELD_RW(x, y) { x, offsetof(struct vmcs12, y) },
 57 #include "vmcs_shadow_fields.h"
 58 };
 59 static int max_shadow_read_write_fields =
 60         ARRAY_SIZE(shadow_read_write_fields);
 61 
 62 static void init_vmcs_shadow_fields(void)
 63 {
 64         int i, j;
 65 
 66         memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE);
 67         memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE);
 68 
 69         for (i = j = 0; i < max_shadow_read_only_fields; i++) {
 70                 struct shadow_vmcs_field entry = shadow_read_only_fields[i];
 71                 u16 field = entry.encoding;
 72 
 73                 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 &&
 74                     (i + 1 == max_shadow_read_only_fields ||
 75                      shadow_read_only_fields[i + 1].encoding != field + 1))
 76                         pr_err("Missing field from shadow_read_only_field %x\n",
 77                                field + 1);
 78 
 79                 clear_bit(field, vmx_vmread_bitmap);
 80                 if (field & 1)
 81 #ifdef CONFIG_X86_64
 82                         continue;
 83 #else
 84                         entry.offset += sizeof(u32);
 85 #endif
 86                 shadow_read_only_fields[j++] = entry;
 87         }
 88         max_shadow_read_only_fields = j;
 89 
 90         for (i = j = 0; i < max_shadow_read_write_fields; i++) {
 91                 struct shadow_vmcs_field entry = shadow_read_write_fields[i];
 92                 u16 field = entry.encoding;
 93 
 94                 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 &&
 95                     (i + 1 == max_shadow_read_write_fields ||
 96                      shadow_read_write_fields[i + 1].encoding != field + 1))
 97                         pr_err("Missing field from shadow_read_write_field %x\n",
 98                                field + 1);
 99 
100                 WARN_ONCE(field >= GUEST_ES_AR_BYTES &&
101                           field <= GUEST_TR_AR_BYTES,
102                           "Update vmcs12_write_any() to drop reserved bits from AR_BYTES");
103 
104                 /*
105                  * PML and the preemption timer can be emulated, but the
106                  * processor cannot vmwrite to fields that don't exist
107                  * on bare metal.
108                  */
109                 switch (field) {
110                 case GUEST_PML_INDEX:
111                         if (!cpu_has_vmx_pml())
112                                 continue;
113                         break;
114                 case VMX_PREEMPTION_TIMER_VALUE:
115                         if (!cpu_has_vmx_preemption_timer())
116                                 continue;
117                         break;
118                 case GUEST_INTR_STATUS:
119                         if (!cpu_has_vmx_apicv())
120                                 continue;
121                         break;
122                 default:
123                         break;
124                 }
125 
126                 clear_bit(field, vmx_vmwrite_bitmap);
127                 clear_bit(field, vmx_vmread_bitmap);
128                 if (field & 1)
129 #ifdef CONFIG_X86_64
130                         continue;
131 #else
132                         entry.offset += sizeof(u32);
133 #endif
134                 shadow_read_write_fields[j++] = entry;
135         }
136         max_shadow_read_write_fields = j;
137 }
138 
139 /*
140  * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(),
141  * set the success or error code of an emulated VMX instruction (as specified
142  * by Vol 2B, VMX Instruction Reference, "Conventions"), and skip the emulated
143  * instruction.
144  */
145 static int nested_vmx_succeed(struct kvm_vcpu *vcpu)
146 {
147         vmx_set_rflags(vcpu, vmx_get_rflags(vcpu)
148                         & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
149                             X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF));
150         return kvm_skip_emulated_instruction(vcpu);
151 }
152 
153 static int nested_vmx_failInvalid(struct kvm_vcpu *vcpu)
154 {
155         vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
156                         & ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
157                             X86_EFLAGS_SF | X86_EFLAGS_OF))
158                         | X86_EFLAGS_CF);
159         return kvm_skip_emulated_instruction(vcpu);
160 }
161 
162 static int nested_vmx_failValid(struct kvm_vcpu *vcpu,
163                                 u32 vm_instruction_error)
164 {
165         struct vcpu_vmx *vmx = to_vmx(vcpu);
166 
167         /*
168          * failValid writes the error number to the current VMCS, which
169          * can't be done if there isn't a current VMCS.
170          */
171         if (vmx->nested.current_vmptr == -1ull && !vmx->nested.hv_evmcs)
172                 return nested_vmx_failInvalid(vcpu);
173 
174         vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
175                         & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
176                             X86_EFLAGS_SF | X86_EFLAGS_OF))
177                         | X86_EFLAGS_ZF);
178         get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error;
179         /*
180          * We don't need to force a shadow sync because
181          * VM_INSTRUCTION_ERROR is not shadowed
182          */
183         return kvm_skip_emulated_instruction(vcpu);
184 }
185 
186 static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator)
187 {
188         /* TODO: not to reset guest simply here. */
189         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
190         pr_debug_ratelimited("kvm: nested vmx abort, indicator %d\n", indicator);
191 }
192 
193 static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx)
194 {
195         secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_SHADOW_VMCS);
196         vmcs_write64(VMCS_LINK_POINTER, -1ull);
197         vmx->nested.need_vmcs12_to_shadow_sync = false;
198 }
199 
200 static inline void nested_release_evmcs(struct kvm_vcpu *vcpu)
201 {
202         struct vcpu_vmx *vmx = to_vmx(vcpu);
203 
204         if (!vmx->nested.hv_evmcs)
205                 return;
206 
207         kvm_vcpu_unmap(vcpu, &vmx->nested.hv_evmcs_map, true);
208         vmx->nested.hv_evmcs_vmptr = -1ull;
209         vmx->nested.hv_evmcs = NULL;
210 }
211 
212 /*
213  * Free whatever needs to be freed from vmx->nested when L1 goes down, or
214  * just stops using VMX.
215  */
216 static void free_nested(struct kvm_vcpu *vcpu)
217 {
218         struct vcpu_vmx *vmx = to_vmx(vcpu);
219 
220         if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon)
221                 return;
222 
223         kvm_clear_request(KVM_REQ_GET_VMCS12_PAGES, vcpu);
224 
225         vmx->nested.vmxon = false;
226         vmx->nested.smm.vmxon = false;
227         free_vpid(vmx->nested.vpid02);
228         vmx->nested.posted_intr_nv = -1;
229         vmx->nested.current_vmptr = -1ull;
230         if (enable_shadow_vmcs) {
231                 vmx_disable_shadow_vmcs(vmx);
232                 vmcs_clear(vmx->vmcs01.shadow_vmcs);
233                 free_vmcs(vmx->vmcs01.shadow_vmcs);
234                 vmx->vmcs01.shadow_vmcs = NULL;
235         }
236         kfree(vmx->nested.cached_vmcs12);
237         vmx->nested.cached_vmcs12 = NULL;
238         kfree(vmx->nested.cached_shadow_vmcs12);
239         vmx->nested.cached_shadow_vmcs12 = NULL;
240         /* Unpin physical memory we referred to in the vmcs02 */
241         if (vmx->nested.apic_access_page) {
242                 kvm_release_page_dirty(vmx->nested.apic_access_page);
243                 vmx->nested.apic_access_page = NULL;
244         }
245         kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true);
246         kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true);
247         vmx->nested.pi_desc = NULL;
248 
249         kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
250 
251         nested_release_evmcs(vcpu);
252 
253         free_loaded_vmcs(&vmx->nested.vmcs02);
254 }
255 
256 static void vmx_sync_vmcs_host_state(struct vcpu_vmx *vmx,
257                                      struct loaded_vmcs *prev)
258 {
259         struct vmcs_host_state *dest, *src;
260 
261         if (unlikely(!vmx->guest_state_loaded))
262                 return;
263 
264         src = &prev->host_state;
265         dest = &vmx->loaded_vmcs->host_state;
266 
267         vmx_set_host_fs_gs(dest, src->fs_sel, src->gs_sel, src->fs_base, src->gs_base);
268         dest->ldt_sel = src->ldt_sel;
269 #ifdef CONFIG_X86_64
270         dest->ds_sel = src->ds_sel;
271         dest->es_sel = src->es_sel;
272 #endif
273 }
274 
275 static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs)
276 {
277         struct vcpu_vmx *vmx = to_vmx(vcpu);
278         struct loaded_vmcs *prev;
279         int cpu;
280 
281         if (vmx->loaded_vmcs == vmcs)
282                 return;
283 
284         cpu = get_cpu();
285         prev = vmx->loaded_vmcs;
286         vmx->loaded_vmcs = vmcs;
287         vmx_vcpu_load_vmcs(vcpu, cpu);
288         vmx_sync_vmcs_host_state(vmx, prev);
289         put_cpu();
290 
291         vmx_segment_cache_clear(vmx);
292 }
293 
294 /*
295  * Ensure that the current vmcs of the logical processor is the
296  * vmcs01 of the vcpu before calling free_nested().
297  */
298 void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu)
299 {
300         vcpu_load(vcpu);
301         vmx_leave_nested(vcpu);
302         vmx_switch_vmcs(vcpu, &to_vmx(vcpu)->vmcs01);
303         free_nested(vcpu);
304         vcpu_put(vcpu);
305 }
306 
307 static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
308                 struct x86_exception *fault)
309 {
310         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
311         struct vcpu_vmx *vmx = to_vmx(vcpu);
312         u32 exit_reason;
313         unsigned long exit_qualification = vcpu->arch.exit_qualification;
314 
315         if (vmx->nested.pml_full) {
316                 exit_reason = EXIT_REASON_PML_FULL;
317                 vmx->nested.pml_full = false;
318                 exit_qualification &= INTR_INFO_UNBLOCK_NMI;
319         } else if (fault->error_code & PFERR_RSVD_MASK)
320                 exit_reason = EXIT_REASON_EPT_MISCONFIG;
321         else
322                 exit_reason = EXIT_REASON_EPT_VIOLATION;
323 
324         nested_vmx_vmexit(vcpu, exit_reason, 0, exit_qualification);
325         vmcs12->guest_physical_address = fault->address;
326 }
327 
328 static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
329 {
330         WARN_ON(mmu_is_nested(vcpu));
331 
332         vcpu->arch.mmu = &vcpu->arch.guest_mmu;
333         kvm_init_shadow_ept_mmu(vcpu,
334                         to_vmx(vcpu)->nested.msrs.ept_caps &
335                         VMX_EPT_EXECUTE_ONLY_BIT,
336                         nested_ept_ad_enabled(vcpu),
337                         nested_ept_get_cr3(vcpu));
338         vcpu->arch.mmu->set_cr3           = vmx_set_cr3;
339         vcpu->arch.mmu->get_cr3           = nested_ept_get_cr3;
340         vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault;
341         vcpu->arch.mmu->get_pdptr         = kvm_pdptr_read;
342 
343         vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
344 }
345 
346 static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu)
347 {
348         vcpu->arch.mmu = &vcpu->arch.root_mmu;
349         vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
350 }
351 
352 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,
353                                             u16 error_code)
354 {
355         bool inequality, bit;
356 
357         bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0;
358         inequality =
359                 (error_code & vmcs12->page_fault_error_code_mask) !=
360                  vmcs12->page_fault_error_code_match;
361         return inequality ^ bit;
362 }
363 
364 
365 /*
366  * KVM wants to inject page-faults which it got to the guest. This function
367  * checks whether in a nested guest, we need to inject them to L1 or L2.
368  */
369 static int nested_vmx_check_exception(struct kvm_vcpu *vcpu, unsigned long *exit_qual)
370 {
371         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
372         unsigned int nr = vcpu->arch.exception.nr;
373         bool has_payload = vcpu->arch.exception.has_payload;
374         unsigned long payload = vcpu->arch.exception.payload;
375 
376         if (nr == PF_VECTOR) {
377                 if (vcpu->arch.exception.nested_apf) {
378                         *exit_qual = vcpu->arch.apf.nested_apf_token;
379                         return 1;
380                 }
381                 if (nested_vmx_is_page_fault_vmexit(vmcs12,
382                                                     vcpu->arch.exception.error_code)) {
383                         *exit_qual = has_payload ? payload : vcpu->arch.cr2;
384                         return 1;
385                 }
386         } else if (vmcs12->exception_bitmap & (1u << nr)) {
387                 if (nr == DB_VECTOR) {
388                         if (!has_payload) {
389                                 payload = vcpu->arch.dr6;
390                                 payload &= ~(DR6_FIXED_1 | DR6_BT);
391                                 payload ^= DR6_RTM;
392                         }
393                         *exit_qual = payload;
394                 } else
395                         *exit_qual = 0;
396                 return 1;
397         }
398 
399         return 0;
400 }
401 
402 
403 static void vmx_inject_page_fault_nested(struct kvm_vcpu *vcpu,
404                 struct x86_exception *fault)
405 {
406         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
407 
408         WARN_ON(!is_guest_mode(vcpu));
409 
410         if (nested_vmx_is_page_fault_vmexit(vmcs12, fault->error_code) &&
411                 !to_vmx(vcpu)->nested.nested_run_pending) {
412                 vmcs12->vm_exit_intr_error_code = fault->error_code;
413                 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI,
414                                   PF_VECTOR | INTR_TYPE_HARD_EXCEPTION |
415                                   INTR_INFO_DELIVER_CODE_MASK | INTR_INFO_VALID_MASK,
416                                   fault->address);
417         } else {
418                 kvm_inject_page_fault(vcpu, fault);
419         }
420 }
421 
422 static bool page_address_valid(struct kvm_vcpu *vcpu, gpa_t gpa)
423 {
424         return PAGE_ALIGNED(gpa) && !(gpa >> cpuid_maxphyaddr(vcpu));
425 }
426 
427 static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu,
428                                                struct vmcs12 *vmcs12)
429 {
430         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
431                 return 0;
432 
433         if (!page_address_valid(vcpu, vmcs12->io_bitmap_a) ||
434             !page_address_valid(vcpu, vmcs12->io_bitmap_b))
435                 return -EINVAL;
436 
437         return 0;
438 }
439 
440 static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu,
441                                                 struct vmcs12 *vmcs12)
442 {
443         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
444                 return 0;
445 
446         if (!page_address_valid(vcpu, vmcs12->msr_bitmap))
447                 return -EINVAL;
448 
449         return 0;
450 }
451 
452 static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu,
453                                                 struct vmcs12 *vmcs12)
454 {
455         if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
456                 return 0;
457 
458         if (!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr))
459                 return -EINVAL;
460 
461         return 0;
462 }
463 
464 /*
465  * Check if MSR is intercepted for L01 MSR bitmap.
466  */
467 static bool msr_write_intercepted_l01(struct kvm_vcpu *vcpu, u32 msr)
468 {
469         unsigned long *msr_bitmap;
470         int f = sizeof(unsigned long);
471 
472         if (!cpu_has_vmx_msr_bitmap())
473                 return true;
474 
475         msr_bitmap = to_vmx(vcpu)->vmcs01.msr_bitmap;
476 
477         if (msr <= 0x1fff) {
478                 return !!test_bit(msr, msr_bitmap + 0x800 / f);
479         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
480                 msr &= 0x1fff;
481                 return !!test_bit(msr, msr_bitmap + 0xc00 / f);
482         }
483 
484         return true;
485 }
486 
487 /*
488  * If a msr is allowed by L0, we should check whether it is allowed by L1.
489  * The corresponding bit will be cleared unless both of L0 and L1 allow it.
490  */
491 static void nested_vmx_disable_intercept_for_msr(unsigned long *msr_bitmap_l1,
492                                                unsigned long *msr_bitmap_nested,
493                                                u32 msr, int type)
494 {
495         int f = sizeof(unsigned long);
496 
497         /*
498          * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
499          * have the write-low and read-high bitmap offsets the wrong way round.
500          * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
501          */
502         if (msr <= 0x1fff) {
503                 if (type & MSR_TYPE_R &&
504                    !test_bit(msr, msr_bitmap_l1 + 0x000 / f))
505                         /* read-low */
506                         __clear_bit(msr, msr_bitmap_nested + 0x000 / f);
507 
508                 if (type & MSR_TYPE_W &&
509                    !test_bit(msr, msr_bitmap_l1 + 0x800 / f))
510                         /* write-low */
511                         __clear_bit(msr, msr_bitmap_nested + 0x800 / f);
512 
513         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
514                 msr &= 0x1fff;
515                 if (type & MSR_TYPE_R &&
516                    !test_bit(msr, msr_bitmap_l1 + 0x400 / f))
517                         /* read-high */
518                         __clear_bit(msr, msr_bitmap_nested + 0x400 / f);
519 
520                 if (type & MSR_TYPE_W &&
521                    !test_bit(msr, msr_bitmap_l1 + 0xc00 / f))
522                         /* write-high */
523                         __clear_bit(msr, msr_bitmap_nested + 0xc00 / f);
524 
525         }
526 }
527 
528 static inline void enable_x2apic_msr_intercepts(unsigned long *msr_bitmap) {
529         int msr;
530 
531         for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
532                 unsigned word = msr / BITS_PER_LONG;
533 
534                 msr_bitmap[word] = ~0;
535                 msr_bitmap[word + (0x800 / sizeof(long))] = ~0;
536         }
537 }
538 
539 /*
540  * Merge L0's and L1's MSR bitmap, return false to indicate that
541  * we do not use the hardware.
542  */
543 static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu,
544                                                  struct vmcs12 *vmcs12)
545 {
546         int msr;
547         unsigned long *msr_bitmap_l1;
548         unsigned long *msr_bitmap_l0 = to_vmx(vcpu)->nested.vmcs02.msr_bitmap;
549         struct kvm_host_map *map = &to_vmx(vcpu)->nested.msr_bitmap_map;
550 
551         /* Nothing to do if the MSR bitmap is not in use.  */
552         if (!cpu_has_vmx_msr_bitmap() ||
553             !nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
554                 return false;
555 
556         if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->msr_bitmap), map))
557                 return false;
558 
559         msr_bitmap_l1 = (unsigned long *)map->hva;
560 
561         /*
562          * To keep the control flow simple, pay eight 8-byte writes (sixteen
563          * 4-byte writes on 32-bit systems) up front to enable intercepts for
564          * the x2APIC MSR range and selectively disable them below.
565          */
566         enable_x2apic_msr_intercepts(msr_bitmap_l0);
567 
568         if (nested_cpu_has_virt_x2apic_mode(vmcs12)) {
569                 if (nested_cpu_has_apic_reg_virt(vmcs12)) {
570                         /*
571                          * L0 need not intercept reads for MSRs between 0x800
572                          * and 0x8ff, it just lets the processor take the value
573                          * from the virtual-APIC page; take those 256 bits
574                          * directly from the L1 bitmap.
575                          */
576                         for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
577                                 unsigned word = msr / BITS_PER_LONG;
578 
579                                 msr_bitmap_l0[word] = msr_bitmap_l1[word];
580                         }
581                 }
582 
583                 nested_vmx_disable_intercept_for_msr(
584                         msr_bitmap_l1, msr_bitmap_l0,
585                         X2APIC_MSR(APIC_TASKPRI),
586                         MSR_TYPE_R | MSR_TYPE_W);
587 
588                 if (nested_cpu_has_vid(vmcs12)) {
589                         nested_vmx_disable_intercept_for_msr(
590                                 msr_bitmap_l1, msr_bitmap_l0,
591                                 X2APIC_MSR(APIC_EOI),
592                                 MSR_TYPE_W);
593                         nested_vmx_disable_intercept_for_msr(
594                                 msr_bitmap_l1, msr_bitmap_l0,
595                                 X2APIC_MSR(APIC_SELF_IPI),
596                                 MSR_TYPE_W);
597                 }
598         }
599 
600         /* KVM unconditionally exposes the FS/GS base MSRs to L1. */
601         nested_vmx_disable_intercept_for_msr(msr_bitmap_l1, msr_bitmap_l0,
602                                              MSR_FS_BASE, MSR_TYPE_RW);
603 
604         nested_vmx_disable_intercept_for_msr(msr_bitmap_l1, msr_bitmap_l0,
605                                              MSR_GS_BASE, MSR_TYPE_RW);
606 
607         nested_vmx_disable_intercept_for_msr(msr_bitmap_l1, msr_bitmap_l0,
608                                              MSR_KERNEL_GS_BASE, MSR_TYPE_RW);
609 
610         /*
611          * Checking the L0->L1 bitmap is trying to verify two things:
612          *
613          * 1. L0 gave a permission to L1 to actually passthrough the MSR. This
614          *    ensures that we do not accidentally generate an L02 MSR bitmap
615          *    from the L12 MSR bitmap that is too permissive.
616          * 2. That L1 or L2s have actually used the MSR. This avoids
617          *    unnecessarily merging of the bitmap if the MSR is unused. This
618          *    works properly because we only update the L01 MSR bitmap lazily.
619          *    So even if L0 should pass L1 these MSRs, the L01 bitmap is only
620          *    updated to reflect this when L1 (or its L2s) actually write to
621          *    the MSR.
622          */
623         if (!msr_write_intercepted_l01(vcpu, MSR_IA32_SPEC_CTRL))
624                 nested_vmx_disable_intercept_for_msr(
625                                         msr_bitmap_l1, msr_bitmap_l0,
626                                         MSR_IA32_SPEC_CTRL,
627                                         MSR_TYPE_R | MSR_TYPE_W);
628 
629         if (!msr_write_intercepted_l01(vcpu, MSR_IA32_PRED_CMD))
630                 nested_vmx_disable_intercept_for_msr(
631                                         msr_bitmap_l1, msr_bitmap_l0,
632                                         MSR_IA32_PRED_CMD,
633                                         MSR_TYPE_W);
634 
635         kvm_vcpu_unmap(vcpu, &to_vmx(vcpu)->nested.msr_bitmap_map, false);
636 
637         return true;
638 }
639 
640 static void nested_cache_shadow_vmcs12(struct kvm_vcpu *vcpu,
641                                        struct vmcs12 *vmcs12)
642 {
643         struct kvm_host_map map;
644         struct vmcs12 *shadow;
645 
646         if (!nested_cpu_has_shadow_vmcs(vmcs12) ||
647             vmcs12->vmcs_link_pointer == -1ull)
648                 return;
649 
650         shadow = get_shadow_vmcs12(vcpu);
651 
652         if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->vmcs_link_pointer), &map))
653                 return;
654 
655         memcpy(shadow, map.hva, VMCS12_SIZE);
656         kvm_vcpu_unmap(vcpu, &map, false);
657 }
658 
659 static void nested_flush_cached_shadow_vmcs12(struct kvm_vcpu *vcpu,
660                                               struct vmcs12 *vmcs12)
661 {
662         struct vcpu_vmx *vmx = to_vmx(vcpu);
663 
664         if (!nested_cpu_has_shadow_vmcs(vmcs12) ||
665             vmcs12->vmcs_link_pointer == -1ull)
666                 return;
667 
668         kvm_write_guest(vmx->vcpu.kvm, vmcs12->vmcs_link_pointer,
669                         get_shadow_vmcs12(vcpu), VMCS12_SIZE);
670 }
671 
672 /*
673  * In nested virtualization, check if L1 has set
674  * VM_EXIT_ACK_INTR_ON_EXIT
675  */
676 static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu)
677 {
678         return get_vmcs12(vcpu)->vm_exit_controls &
679                 VM_EXIT_ACK_INTR_ON_EXIT;
680 }
681 
682 static bool nested_exit_on_nmi(struct kvm_vcpu *vcpu)
683 {
684         return nested_cpu_has_nmi_exiting(get_vmcs12(vcpu));
685 }
686 
687 static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu,
688                                           struct vmcs12 *vmcs12)
689 {
690         if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) &&
691             !page_address_valid(vcpu, vmcs12->apic_access_addr))
692                 return -EINVAL;
693         else
694                 return 0;
695 }
696 
697 static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu,
698                                            struct vmcs12 *vmcs12)
699 {
700         if (!nested_cpu_has_virt_x2apic_mode(vmcs12) &&
701             !nested_cpu_has_apic_reg_virt(vmcs12) &&
702             !nested_cpu_has_vid(vmcs12) &&
703             !nested_cpu_has_posted_intr(vmcs12))
704                 return 0;
705 
706         /*
707          * If virtualize x2apic mode is enabled,
708          * virtualize apic access must be disabled.
709          */
710         if (nested_cpu_has_virt_x2apic_mode(vmcs12) &&
711             nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
712                 return -EINVAL;
713 
714         /*
715          * If virtual interrupt delivery is enabled,
716          * we must exit on external interrupts.
717          */
718         if (nested_cpu_has_vid(vmcs12) &&
719            !nested_exit_on_intr(vcpu))
720                 return -EINVAL;
721 
722         /*
723          * bits 15:8 should be zero in posted_intr_nv,
724          * the descriptor address has been already checked
725          * in nested_get_vmcs12_pages.
726          *
727          * bits 5:0 of posted_intr_desc_addr should be zero.
728          */
729         if (nested_cpu_has_posted_intr(vmcs12) &&
730            (!nested_cpu_has_vid(vmcs12) ||
731             !nested_exit_intr_ack_set(vcpu) ||
732             (vmcs12->posted_intr_nv & 0xff00) ||
733             (vmcs12->posted_intr_desc_addr & 0x3f) ||
734             (vmcs12->posted_intr_desc_addr >> cpuid_maxphyaddr(vcpu))))
735                 return -EINVAL;
736 
737         /* tpr shadow is needed by all apicv features. */
738         if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
739                 return -EINVAL;
740 
741         return 0;
742 }
743 
744 static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu,
745                                        u32 count, u64 addr)
746 {
747         int maxphyaddr;
748 
749         if (count == 0)
750                 return 0;
751         maxphyaddr = cpuid_maxphyaddr(vcpu);
752         if (!IS_ALIGNED(addr, 16) || addr >> maxphyaddr ||
753             (addr + count * sizeof(struct vmx_msr_entry) - 1) >> maxphyaddr)
754                 return -EINVAL;
755 
756         return 0;
757 }
758 
759 static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu,
760                                                      struct vmcs12 *vmcs12)
761 {
762         if (nested_vmx_check_msr_switch(vcpu, vmcs12->vm_exit_msr_load_count,
763                                         vmcs12->vm_exit_msr_load_addr) ||
764             nested_vmx_check_msr_switch(vcpu, vmcs12->vm_exit_msr_store_count,
765                                         vmcs12->vm_exit_msr_store_addr))
766                 return -EINVAL;
767 
768         return 0;
769 }
770 
771 static int nested_vmx_check_entry_msr_switch_controls(struct kvm_vcpu *vcpu,
772                                                       struct vmcs12 *vmcs12)
773 {
774         if (nested_vmx_check_msr_switch(vcpu, vmcs12->vm_entry_msr_load_count,
775                                         vmcs12->vm_entry_msr_load_addr))
776                 return -EINVAL;
777 
778         return 0;
779 }
780 
781 static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu,
782                                          struct vmcs12 *vmcs12)
783 {
784         if (!nested_cpu_has_pml(vmcs12))
785                 return 0;
786 
787         if (!nested_cpu_has_ept(vmcs12) ||
788             !page_address_valid(vcpu, vmcs12->pml_address))
789                 return -EINVAL;
790 
791         return 0;
792 }
793 
794 static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu,
795                                                         struct vmcs12 *vmcs12)
796 {
797         if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST) &&
798             !nested_cpu_has_ept(vmcs12))
799                 return -EINVAL;
800         return 0;
801 }
802 
803 static int nested_vmx_check_mode_based_ept_exec_controls(struct kvm_vcpu *vcpu,
804                                                          struct vmcs12 *vmcs12)
805 {
806         if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC) &&
807             !nested_cpu_has_ept(vmcs12))
808                 return -EINVAL;
809         return 0;
810 }
811 
812 static int nested_vmx_check_shadow_vmcs_controls(struct kvm_vcpu *vcpu,
813                                                  struct vmcs12 *vmcs12)
814 {
815         if (!nested_cpu_has_shadow_vmcs(vmcs12))
816                 return 0;
817 
818         if (!page_address_valid(vcpu, vmcs12->vmread_bitmap) ||
819             !page_address_valid(vcpu, vmcs12->vmwrite_bitmap))
820                 return -EINVAL;
821 
822         return 0;
823 }
824 
825 static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu,
826                                        struct vmx_msr_entry *e)
827 {
828         /* x2APIC MSR accesses are not allowed */
829         if (vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8)
830                 return -EINVAL;
831         if (e->index == MSR_IA32_UCODE_WRITE || /* SDM Table 35-2 */
832             e->index == MSR_IA32_UCODE_REV)
833                 return -EINVAL;
834         if (e->reserved != 0)
835                 return -EINVAL;
836         return 0;
837 }
838 
839 static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu,
840                                      struct vmx_msr_entry *e)
841 {
842         if (e->index == MSR_FS_BASE ||
843             e->index == MSR_GS_BASE ||
844             e->index == MSR_IA32_SMM_MONITOR_CTL || /* SMM is not supported */
845             nested_vmx_msr_check_common(vcpu, e))
846                 return -EINVAL;
847         return 0;
848 }
849 
850 static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu,
851                                       struct vmx_msr_entry *e)
852 {
853         if (e->index == MSR_IA32_SMBASE || /* SMM is not supported */
854             nested_vmx_msr_check_common(vcpu, e))
855                 return -EINVAL;
856         return 0;
857 }
858 
859 /*
860  * Load guest's/host's msr at nested entry/exit.
861  * return 0 for success, entry index for failure.
862  */
863 static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
864 {
865         u32 i;
866         struct vmx_msr_entry e;
867         struct msr_data msr;
868 
869         msr.host_initiated = false;
870         for (i = 0; i < count; i++) {
871                 if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e),
872                                         &e, sizeof(e))) {
873                         pr_debug_ratelimited(
874                                 "%s cannot read MSR entry (%u, 0x%08llx)\n",
875                                 __func__, i, gpa + i * sizeof(e));
876                         goto fail;
877                 }
878                 if (nested_vmx_load_msr_check(vcpu, &e)) {
879                         pr_debug_ratelimited(
880                                 "%s check failed (%u, 0x%x, 0x%x)\n",
881                                 __func__, i, e.index, e.reserved);
882                         goto fail;
883                 }
884                 msr.index = e.index;
885                 msr.data = e.value;
886                 if (kvm_set_msr(vcpu, &msr)) {
887                         pr_debug_ratelimited(
888                                 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
889                                 __func__, i, e.index, e.value);
890                         goto fail;
891                 }
892         }
893         return 0;
894 fail:
895         return i + 1;
896 }
897 
898 static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
899 {
900         u32 i;
901         struct vmx_msr_entry e;
902 
903         for (i = 0; i < count; i++) {
904                 struct msr_data msr_info;
905                 if (kvm_vcpu_read_guest(vcpu,
906                                         gpa + i * sizeof(e),
907                                         &e, 2 * sizeof(u32))) {
908                         pr_debug_ratelimited(
909                                 "%s cannot read MSR entry (%u, 0x%08llx)\n",
910                                 __func__, i, gpa + i * sizeof(e));
911                         return -EINVAL;
912                 }
913                 if (nested_vmx_store_msr_check(vcpu, &e)) {
914                         pr_debug_ratelimited(
915                                 "%s check failed (%u, 0x%x, 0x%x)\n",
916                                 __func__, i, e.index, e.reserved);
917                         return -EINVAL;
918                 }
919                 msr_info.host_initiated = false;
920                 msr_info.index = e.index;
921                 if (kvm_get_msr(vcpu, &msr_info)) {
922                         pr_debug_ratelimited(
923                                 "%s cannot read MSR (%u, 0x%x)\n",
924                                 __func__, i, e.index);
925                         return -EINVAL;
926                 }
927                 if (kvm_vcpu_write_guest(vcpu,
928                                          gpa + i * sizeof(e) +
929                                              offsetof(struct vmx_msr_entry, value),
930                                          &msr_info.data, sizeof(msr_info.data))) {
931                         pr_debug_ratelimited(
932                                 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
933                                 __func__, i, e.index, msr_info.data);
934                         return -EINVAL;
935                 }
936         }
937         return 0;
938 }
939 
940 static bool nested_cr3_valid(struct kvm_vcpu *vcpu, unsigned long val)
941 {
942         unsigned long invalid_mask;
943 
944         invalid_mask = (~0ULL) << cpuid_maxphyaddr(vcpu);
945         return (val & invalid_mask) == 0;
946 }
947 
948 /*
949  * Load guest's/host's cr3 at nested entry/exit. nested_ept is true if we are
950  * emulating VM entry into a guest with EPT enabled.
951  * Returns 0 on success, 1 on failure. Invalid state exit qualification code
952  * is assigned to entry_failure_code on failure.
953  */
954 static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept,
955                                u32 *entry_failure_code)
956 {
957         if (cr3 != kvm_read_cr3(vcpu) || (!nested_ept && pdptrs_changed(vcpu))) {
958                 if (!nested_cr3_valid(vcpu, cr3)) {
959                         *entry_failure_code = ENTRY_FAIL_DEFAULT;
960                         return -EINVAL;
961                 }
962 
963                 /*
964                  * If PAE paging and EPT are both on, CR3 is not used by the CPU and
965                  * must not be dereferenced.
966                  */
967                 if (is_pae_paging(vcpu) && !nested_ept) {
968                         if (!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) {
969                                 *entry_failure_code = ENTRY_FAIL_PDPTE;
970                                 return -EINVAL;
971                         }
972                 }
973         }
974 
975         if (!nested_ept)
976                 kvm_mmu_new_cr3(vcpu, cr3, false);
977 
978         vcpu->arch.cr3 = cr3;
979         __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
980 
981         kvm_init_mmu(vcpu, false);
982 
983         return 0;
984 }
985 
986 /*
987  * Returns if KVM is able to config CPU to tag TLB entries
988  * populated by L2 differently than TLB entries populated
989  * by L1.
990  *
991  * If L1 uses EPT, then TLB entries are tagged with different EPTP.
992  *
993  * If L1 uses VPID and we allocated a vpid02, TLB entries are tagged
994  * with different VPID (L1 entries are tagged with vmx->vpid
995  * while L2 entries are tagged with vmx->nested.vpid02).
996  */
997 static bool nested_has_guest_tlb_tag(struct kvm_vcpu *vcpu)
998 {
999         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
1000 
1001         return nested_cpu_has_ept(vmcs12) ||
1002                (nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02);
1003 }
1004 
1005 static u16 nested_get_vpid02(struct kvm_vcpu *vcpu)
1006 {
1007         struct vcpu_vmx *vmx = to_vmx(vcpu);
1008 
1009         return vmx->nested.vpid02 ? vmx->nested.vpid02 : vmx->vpid;
1010 }
1011 
1012 
1013 static inline bool vmx_control_verify(u32 control, u32 low, u32 high)
1014 {
1015         return fixed_bits_valid(control, low, high);
1016 }
1017 
1018 static inline u64 vmx_control_msr(u32 low, u32 high)
1019 {
1020         return low | ((u64)high << 32);
1021 }
1022 
1023 static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask)
1024 {
1025         superset &= mask;
1026         subset &= mask;
1027 
1028         return (superset | subset) == superset;
1029 }
1030 
1031 static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data)
1032 {
1033         const u64 feature_and_reserved =
1034                 /* feature (except bit 48; see below) */
1035                 BIT_ULL(49) | BIT_ULL(54) | BIT_ULL(55) |
1036                 /* reserved */
1037                 BIT_ULL(31) | GENMASK_ULL(47, 45) | GENMASK_ULL(63, 56);
1038         u64 vmx_basic = vmx->nested.msrs.basic;
1039 
1040         if (!is_bitwise_subset(vmx_basic, data, feature_and_reserved))
1041                 return -EINVAL;
1042 
1043         /*
1044          * KVM does not emulate a version of VMX that constrains physical
1045          * addresses of VMX structures (e.g. VMCS) to 32-bits.
1046          */
1047         if (data & BIT_ULL(48))
1048                 return -EINVAL;
1049 
1050         if (vmx_basic_vmcs_revision_id(vmx_basic) !=
1051             vmx_basic_vmcs_revision_id(data))
1052                 return -EINVAL;
1053 
1054         if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data))
1055                 return -EINVAL;
1056 
1057         vmx->nested.msrs.basic = data;
1058         return 0;
1059 }
1060 
1061 static int
1062 vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
1063 {
1064         u64 supported;
1065         u32 *lowp, *highp;
1066 
1067         switch (msr_index) {
1068         case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
1069                 lowp = &vmx->nested.msrs.pinbased_ctls_low;
1070                 highp = &vmx->nested.msrs.pinbased_ctls_high;
1071                 break;
1072         case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
1073                 lowp = &vmx->nested.msrs.procbased_ctls_low;
1074                 highp = &vmx->nested.msrs.procbased_ctls_high;
1075                 break;
1076         case MSR_IA32_VMX_TRUE_EXIT_CTLS:
1077                 lowp = &vmx->nested.msrs.exit_ctls_low;
1078                 highp = &vmx->nested.msrs.exit_ctls_high;
1079                 break;
1080         case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
1081                 lowp = &vmx->nested.msrs.entry_ctls_low;
1082                 highp = &vmx->nested.msrs.entry_ctls_high;
1083                 break;
1084         case MSR_IA32_VMX_PROCBASED_CTLS2:
1085                 lowp = &vmx->nested.msrs.secondary_ctls_low;
1086                 highp = &vmx->nested.msrs.secondary_ctls_high;
1087                 break;
1088         default:
1089                 BUG();
1090         }
1091 
1092         supported = vmx_control_msr(*lowp, *highp);
1093 
1094         /* Check must-be-1 bits are still 1. */
1095         if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0)))
1096                 return -EINVAL;
1097 
1098         /* Check must-be-0 bits are still 0. */
1099         if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32)))
1100                 return -EINVAL;
1101 
1102         *lowp = data;
1103         *highp = data >> 32;
1104         return 0;
1105 }
1106 
1107 static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data)
1108 {
1109         const u64 feature_and_reserved_bits =
1110                 /* feature */
1111                 BIT_ULL(5) | GENMASK_ULL(8, 6) | BIT_ULL(14) | BIT_ULL(15) |
1112                 BIT_ULL(28) | BIT_ULL(29) | BIT_ULL(30) |
1113                 /* reserved */
1114                 GENMASK_ULL(13, 9) | BIT_ULL(31);
1115         u64 vmx_misc;
1116 
1117         vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low,
1118                                    vmx->nested.msrs.misc_high);
1119 
1120         if (!is_bitwise_subset(vmx_misc, data, feature_and_reserved_bits))
1121                 return -EINVAL;
1122 
1123         if ((vmx->nested.msrs.pinbased_ctls_high &
1124              PIN_BASED_VMX_PREEMPTION_TIMER) &&
1125             vmx_misc_preemption_timer_rate(data) !=
1126             vmx_misc_preemption_timer_rate(vmx_misc))
1127                 return -EINVAL;
1128 
1129         if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc))
1130                 return -EINVAL;
1131 
1132         if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc))
1133                 return -EINVAL;
1134 
1135         if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc))
1136                 return -EINVAL;
1137 
1138         vmx->nested.msrs.misc_low = data;
1139         vmx->nested.msrs.misc_high = data >> 32;
1140 
1141         return 0;
1142 }
1143 
1144 static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data)
1145 {
1146         u64 vmx_ept_vpid_cap;
1147 
1148         vmx_ept_vpid_cap = vmx_control_msr(vmx->nested.msrs.ept_caps,
1149                                            vmx->nested.msrs.vpid_caps);
1150 
1151         /* Every bit is either reserved or a feature bit. */
1152         if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL))
1153                 return -EINVAL;
1154 
1155         vmx->nested.msrs.ept_caps = data;
1156         vmx->nested.msrs.vpid_caps = data >> 32;
1157         return 0;
1158 }
1159 
1160 static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
1161 {
1162         u64 *msr;
1163 
1164         switch (msr_index) {
1165         case MSR_IA32_VMX_CR0_FIXED0:
1166                 msr = &vmx->nested.msrs.cr0_fixed0;
1167                 break;
1168         case MSR_IA32_VMX_CR4_FIXED0:
1169                 msr = &vmx->nested.msrs.cr4_fixed0;
1170                 break;
1171         default:
1172                 BUG();
1173         }
1174 
1175         /*
1176          * 1 bits (which indicates bits which "must-be-1" during VMX operation)
1177          * must be 1 in the restored value.
1178          */
1179         if (!is_bitwise_subset(data, *msr, -1ULL))
1180                 return -EINVAL;
1181 
1182         *msr = data;
1183         return 0;
1184 }
1185 
1186 /*
1187  * Called when userspace is restoring VMX MSRs.
1188  *
1189  * Returns 0 on success, non-0 otherwise.
1190  */
1191 int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
1192 {
1193         struct vcpu_vmx *vmx = to_vmx(vcpu);
1194 
1195         /*
1196          * Don't allow changes to the VMX capability MSRs while the vCPU
1197          * is in VMX operation.
1198          */
1199         if (vmx->nested.vmxon)
1200                 return -EBUSY;
1201 
1202         switch (msr_index) {
1203         case MSR_IA32_VMX_BASIC:
1204                 return vmx_restore_vmx_basic(vmx, data);
1205         case MSR_IA32_VMX_PINBASED_CTLS:
1206         case MSR_IA32_VMX_PROCBASED_CTLS:
1207         case MSR_IA32_VMX_EXIT_CTLS:
1208         case MSR_IA32_VMX_ENTRY_CTLS:
1209                 /*
1210                  * The "non-true" VMX capability MSRs are generated from the
1211                  * "true" MSRs, so we do not support restoring them directly.
1212                  *
1213                  * If userspace wants to emulate VMX_BASIC[55]=0, userspace
1214                  * should restore the "true" MSRs with the must-be-1 bits
1215                  * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND
1216                  * DEFAULT SETTINGS".
1217                  */
1218                 return -EINVAL;
1219         case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
1220         case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
1221         case MSR_IA32_VMX_TRUE_EXIT_CTLS:
1222         case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
1223         case MSR_IA32_VMX_PROCBASED_CTLS2:
1224                 return vmx_restore_control_msr(vmx, msr_index, data);
1225         case MSR_IA32_VMX_MISC:
1226                 return vmx_restore_vmx_misc(vmx, data);
1227         case MSR_IA32_VMX_CR0_FIXED0:
1228         case MSR_IA32_VMX_CR4_FIXED0:
1229                 return vmx_restore_fixed0_msr(vmx, msr_index, data);
1230         case MSR_IA32_VMX_CR0_FIXED1:
1231         case MSR_IA32_VMX_CR4_FIXED1:
1232                 /*
1233                  * These MSRs are generated based on the vCPU's CPUID, so we
1234                  * do not support restoring them directly.
1235                  */
1236                 return -EINVAL;
1237         case MSR_IA32_VMX_EPT_VPID_CAP:
1238                 return vmx_restore_vmx_ept_vpid_cap(vmx, data);
1239         case MSR_IA32_VMX_VMCS_ENUM:
1240                 vmx->nested.msrs.vmcs_enum = data;
1241                 return 0;
1242         case MSR_IA32_VMX_VMFUNC:
1243                 if (data & ~vmx->nested.msrs.vmfunc_controls)
1244                         return -EINVAL;
1245                 vmx->nested.msrs.vmfunc_controls = data;
1246                 return 0;
1247         default:
1248                 /*
1249                  * The rest of the VMX capability MSRs do not support restore.
1250                  */
1251                 return -EINVAL;
1252         }
1253 }
1254 
1255 /* Returns 0 on success, non-0 otherwise. */
1256 int vmx_get_vmx_msr(struct nested_vmx_msrs *msrs, u32 msr_index, u64 *pdata)
1257 {
1258         switch (msr_index) {
1259         case MSR_IA32_VMX_BASIC:
1260                 *pdata = msrs->basic;
1261                 break;
1262         case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
1263         case MSR_IA32_VMX_PINBASED_CTLS:
1264                 *pdata = vmx_control_msr(
1265                         msrs->pinbased_ctls_low,
1266                         msrs->pinbased_ctls_high);
1267                 if (msr_index == MSR_IA32_VMX_PINBASED_CTLS)
1268                         *pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
1269                 break;
1270         case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
1271         case MSR_IA32_VMX_PROCBASED_CTLS:
1272                 *pdata = vmx_control_msr(
1273                         msrs->procbased_ctls_low,
1274                         msrs->procbased_ctls_high);
1275                 if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS)
1276                         *pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
1277                 break;
1278         case MSR_IA32_VMX_TRUE_EXIT_CTLS:
1279         case MSR_IA32_VMX_EXIT_CTLS:
1280                 *pdata = vmx_control_msr(
1281                         msrs->exit_ctls_low,
1282                         msrs->exit_ctls_high);
1283                 if (msr_index == MSR_IA32_VMX_EXIT_CTLS)
1284                         *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
1285                 break;
1286         case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
1287         case MSR_IA32_VMX_ENTRY_CTLS:
1288                 *pdata = vmx_control_msr(
1289                         msrs->entry_ctls_low,
1290                         msrs->entry_ctls_high);
1291                 if (msr_index == MSR_IA32_VMX_ENTRY_CTLS)
1292                         *pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
1293                 break;
1294         case MSR_IA32_VMX_MISC:
1295                 *pdata = vmx_control_msr(
1296                         msrs->misc_low,
1297                         msrs->misc_high);
1298                 break;
1299         case MSR_IA32_VMX_CR0_FIXED0:
1300                 *pdata = msrs->cr0_fixed0;
1301                 break;
1302         case MSR_IA32_VMX_CR0_FIXED1:
1303                 *pdata = msrs->cr0_fixed1;
1304                 break;
1305         case MSR_IA32_VMX_CR4_FIXED0:
1306                 *pdata = msrs->cr4_fixed0;
1307                 break;
1308         case MSR_IA32_VMX_CR4_FIXED1:
1309                 *pdata = msrs->cr4_fixed1;
1310                 break;
1311         case MSR_IA32_VMX_VMCS_ENUM:
1312                 *pdata = msrs->vmcs_enum;
1313                 break;
1314         case MSR_IA32_VMX_PROCBASED_CTLS2:
1315                 *pdata = vmx_control_msr(
1316                         msrs->secondary_ctls_low,
1317                         msrs->secondary_ctls_high);
1318                 break;
1319         case MSR_IA32_VMX_EPT_VPID_CAP:
1320                 *pdata = msrs->ept_caps |
1321                         ((u64)msrs->vpid_caps << 32);
1322                 break;
1323         case MSR_IA32_VMX_VMFUNC:
1324                 *pdata = msrs->vmfunc_controls;
1325                 break;
1326         default:
1327                 return 1;
1328         }
1329 
1330         return 0;
1331 }
1332 
1333 /*
1334  * Copy the writable VMCS shadow fields back to the VMCS12, in case they have
1335  * been modified by the L1 guest.  Note, "writable" in this context means
1336  * "writable by the guest", i.e. tagged SHADOW_FIELD_RW; the set of
1337  * fields tagged SHADOW_FIELD_RO may or may not align with the "read-only"
1338  * VM-exit information fields (which are actually writable if the vCPU is
1339  * configured to support "VMWRITE to any supported field in the VMCS").
1340  */
1341 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx)
1342 {
1343         struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
1344         struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu);
1345         struct shadow_vmcs_field field;
1346         unsigned long val;
1347         int i;
1348 
1349         if (WARN_ON(!shadow_vmcs))
1350                 return;
1351 
1352         preempt_disable();
1353 
1354         vmcs_load(shadow_vmcs);
1355 
1356         for (i = 0; i < max_shadow_read_write_fields; i++) {
1357                 field = shadow_read_write_fields[i];
1358                 val = __vmcs_readl(field.encoding);
1359                 vmcs12_write_any(vmcs12, field.encoding, field.offset, val);
1360         }
1361 
1362         vmcs_clear(shadow_vmcs);
1363         vmcs_load(vmx->loaded_vmcs->vmcs);
1364 
1365         preempt_enable();
1366 }
1367 
1368 static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx)
1369 {
1370         const struct shadow_vmcs_field *fields[] = {
1371                 shadow_read_write_fields,
1372                 shadow_read_only_fields
1373         };
1374         const int max_fields[] = {
1375                 max_shadow_read_write_fields,
1376                 max_shadow_read_only_fields
1377         };
1378         struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
1379         struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu);
1380         struct shadow_vmcs_field field;
1381         unsigned long val;
1382         int i, q;
1383 
1384         if (WARN_ON(!shadow_vmcs))
1385                 return;
1386 
1387         vmcs_load(shadow_vmcs);
1388 
1389         for (q = 0; q < ARRAY_SIZE(fields); q++) {
1390                 for (i = 0; i < max_fields[q]; i++) {
1391                         field = fields[q][i];
1392                         val = vmcs12_read_any(vmcs12, field.encoding,
1393                                               field.offset);
1394                         __vmcs_writel(field.encoding, val);
1395                 }
1396         }
1397 
1398         vmcs_clear(shadow_vmcs);
1399         vmcs_load(vmx->loaded_vmcs->vmcs);
1400 }
1401 
1402 static int copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx)
1403 {
1404         struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12;
1405         struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs;
1406 
1407         /* HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE */
1408         vmcs12->tpr_threshold = evmcs->tpr_threshold;
1409         vmcs12->guest_rip = evmcs->guest_rip;
1410 
1411         if (unlikely(!(evmcs->hv_clean_fields &
1412                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC))) {
1413                 vmcs12->guest_rsp = evmcs->guest_rsp;
1414                 vmcs12->guest_rflags = evmcs->guest_rflags;
1415                 vmcs12->guest_interruptibility_info =
1416                         evmcs->guest_interruptibility_info;
1417         }
1418 
1419         if (unlikely(!(evmcs->hv_clean_fields &
1420                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) {
1421                 vmcs12->cpu_based_vm_exec_control =
1422                         evmcs->cpu_based_vm_exec_control;
1423         }
1424 
1425         if (unlikely(!(evmcs->hv_clean_fields &
1426                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN))) {
1427                 vmcs12->exception_bitmap = evmcs->exception_bitmap;
1428         }
1429 
1430         if (unlikely(!(evmcs->hv_clean_fields &
1431                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY))) {
1432                 vmcs12->vm_entry_controls = evmcs->vm_entry_controls;
1433         }
1434 
1435         if (unlikely(!(evmcs->hv_clean_fields &
1436                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT))) {
1437                 vmcs12->vm_entry_intr_info_field =
1438                         evmcs->vm_entry_intr_info_field;
1439                 vmcs12->vm_entry_exception_error_code =
1440                         evmcs->vm_entry_exception_error_code;
1441                 vmcs12->vm_entry_instruction_len =
1442                         evmcs->vm_entry_instruction_len;
1443         }
1444 
1445         if (unlikely(!(evmcs->hv_clean_fields &
1446                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) {
1447                 vmcs12->host_ia32_pat = evmcs->host_ia32_pat;
1448                 vmcs12->host_ia32_efer = evmcs->host_ia32_efer;
1449                 vmcs12->host_cr0 = evmcs->host_cr0;
1450                 vmcs12->host_cr3 = evmcs->host_cr3;
1451                 vmcs12->host_cr4 = evmcs->host_cr4;
1452                 vmcs12->host_ia32_sysenter_esp = evmcs->host_ia32_sysenter_esp;
1453                 vmcs12->host_ia32_sysenter_eip = evmcs->host_ia32_sysenter_eip;
1454                 vmcs12->host_rip = evmcs->host_rip;
1455                 vmcs12->host_ia32_sysenter_cs = evmcs->host_ia32_sysenter_cs;
1456                 vmcs12->host_es_selector = evmcs->host_es_selector;
1457                 vmcs12->host_cs_selector = evmcs->host_cs_selector;
1458                 vmcs12->host_ss_selector = evmcs->host_ss_selector;
1459                 vmcs12->host_ds_selector = evmcs->host_ds_selector;
1460                 vmcs12->host_fs_selector = evmcs->host_fs_selector;
1461                 vmcs12->host_gs_selector = evmcs->host_gs_selector;
1462                 vmcs12->host_tr_selector = evmcs->host_tr_selector;
1463         }
1464 
1465         if (unlikely(!(evmcs->hv_clean_fields &
1466                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1))) {
1467                 vmcs12->pin_based_vm_exec_control =
1468                         evmcs->pin_based_vm_exec_control;
1469                 vmcs12->vm_exit_controls = evmcs->vm_exit_controls;
1470                 vmcs12->secondary_vm_exec_control =
1471                         evmcs->secondary_vm_exec_control;
1472         }
1473 
1474         if (unlikely(!(evmcs->hv_clean_fields &
1475                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP))) {
1476                 vmcs12->io_bitmap_a = evmcs->io_bitmap_a;
1477                 vmcs12->io_bitmap_b = evmcs->io_bitmap_b;
1478         }
1479 
1480         if (unlikely(!(evmcs->hv_clean_fields &
1481                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP))) {
1482                 vmcs12->msr_bitmap = evmcs->msr_bitmap;
1483         }
1484 
1485         if (unlikely(!(evmcs->hv_clean_fields &
1486                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2))) {
1487                 vmcs12->guest_es_base = evmcs->guest_es_base;
1488                 vmcs12->guest_cs_base = evmcs->guest_cs_base;
1489                 vmcs12->guest_ss_base = evmcs->guest_ss_base;
1490                 vmcs12->guest_ds_base = evmcs->guest_ds_base;
1491                 vmcs12->guest_fs_base = evmcs->guest_fs_base;
1492                 vmcs12->guest_gs_base = evmcs->guest_gs_base;
1493                 vmcs12->guest_ldtr_base = evmcs->guest_ldtr_base;
1494                 vmcs12->guest_tr_base = evmcs->guest_tr_base;
1495                 vmcs12->guest_gdtr_base = evmcs->guest_gdtr_base;
1496                 vmcs12->guest_idtr_base = evmcs->guest_idtr_base;
1497                 vmcs12->guest_es_limit = evmcs->guest_es_limit;
1498                 vmcs12->guest_cs_limit = evmcs->guest_cs_limit;
1499                 vmcs12->guest_ss_limit = evmcs->guest_ss_limit;
1500                 vmcs12->guest_ds_limit = evmcs->guest_ds_limit;
1501                 vmcs12->guest_fs_limit = evmcs->guest_fs_limit;
1502                 vmcs12->guest_gs_limit = evmcs->guest_gs_limit;
1503                 vmcs12->guest_ldtr_limit = evmcs->guest_ldtr_limit;
1504                 vmcs12->guest_tr_limit = evmcs->guest_tr_limit;
1505                 vmcs12->guest_gdtr_limit = evmcs->guest_gdtr_limit;
1506                 vmcs12->guest_idtr_limit = evmcs->guest_idtr_limit;
1507                 vmcs12->guest_es_ar_bytes = evmcs->guest_es_ar_bytes;
1508                 vmcs12->guest_cs_ar_bytes = evmcs->guest_cs_ar_bytes;
1509                 vmcs12->guest_ss_ar_bytes = evmcs->guest_ss_ar_bytes;
1510                 vmcs12->guest_ds_ar_bytes = evmcs->guest_ds_ar_bytes;
1511                 vmcs12->guest_fs_ar_bytes = evmcs->guest_fs_ar_bytes;
1512                 vmcs12->guest_gs_ar_bytes = evmcs->guest_gs_ar_bytes;
1513                 vmcs12->guest_ldtr_ar_bytes = evmcs->guest_ldtr_ar_bytes;
1514                 vmcs12->guest_tr_ar_bytes = evmcs->guest_tr_ar_bytes;
1515                 vmcs12->guest_es_selector = evmcs->guest_es_selector;
1516                 vmcs12->guest_cs_selector = evmcs->guest_cs_selector;
1517                 vmcs12->guest_ss_selector = evmcs->guest_ss_selector;
1518                 vmcs12->guest_ds_selector = evmcs->guest_ds_selector;
1519                 vmcs12->guest_fs_selector = evmcs->guest_fs_selector;
1520                 vmcs12->guest_gs_selector = evmcs->guest_gs_selector;
1521                 vmcs12->guest_ldtr_selector = evmcs->guest_ldtr_selector;
1522                 vmcs12->guest_tr_selector = evmcs->guest_tr_selector;
1523         }
1524 
1525         if (unlikely(!(evmcs->hv_clean_fields &
1526                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2))) {
1527                 vmcs12->tsc_offset = evmcs->tsc_offset;
1528                 vmcs12->virtual_apic_page_addr = evmcs->virtual_apic_page_addr;
1529                 vmcs12->xss_exit_bitmap = evmcs->xss_exit_bitmap;
1530         }
1531 
1532         if (unlikely(!(evmcs->hv_clean_fields &
1533                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR))) {
1534                 vmcs12->cr0_guest_host_mask = evmcs->cr0_guest_host_mask;
1535                 vmcs12->cr4_guest_host_mask = evmcs->cr4_guest_host_mask;
1536                 vmcs12->cr0_read_shadow = evmcs->cr0_read_shadow;
1537                 vmcs12->cr4_read_shadow = evmcs->cr4_read_shadow;
1538                 vmcs12->guest_cr0 = evmcs->guest_cr0;
1539                 vmcs12->guest_cr3 = evmcs->guest_cr3;
1540                 vmcs12->guest_cr4 = evmcs->guest_cr4;
1541                 vmcs12->guest_dr7 = evmcs->guest_dr7;
1542         }
1543 
1544         if (unlikely(!(evmcs->hv_clean_fields &
1545                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER))) {
1546                 vmcs12->host_fs_base = evmcs->host_fs_base;
1547                 vmcs12->host_gs_base = evmcs->host_gs_base;
1548                 vmcs12->host_tr_base = evmcs->host_tr_base;
1549                 vmcs12->host_gdtr_base = evmcs->host_gdtr_base;
1550                 vmcs12->host_idtr_base = evmcs->host_idtr_base;
1551                 vmcs12->host_rsp = evmcs->host_rsp;
1552         }
1553 
1554         if (unlikely(!(evmcs->hv_clean_fields &
1555                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT))) {
1556                 vmcs12->ept_pointer = evmcs->ept_pointer;
1557                 vmcs12->virtual_processor_id = evmcs->virtual_processor_id;
1558         }
1559 
1560         if (unlikely(!(evmcs->hv_clean_fields &
1561                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1))) {
1562                 vmcs12->vmcs_link_pointer = evmcs->vmcs_link_pointer;
1563                 vmcs12->guest_ia32_debugctl = evmcs->guest_ia32_debugctl;
1564                 vmcs12->guest_ia32_pat = evmcs->guest_ia32_pat;
1565                 vmcs12->guest_ia32_efer = evmcs->guest_ia32_efer;
1566                 vmcs12->guest_pdptr0 = evmcs->guest_pdptr0;
1567                 vmcs12->guest_pdptr1 = evmcs->guest_pdptr1;
1568                 vmcs12->guest_pdptr2 = evmcs->guest_pdptr2;
1569                 vmcs12->guest_pdptr3 = evmcs->guest_pdptr3;
1570                 vmcs12->guest_pending_dbg_exceptions =
1571                         evmcs->guest_pending_dbg_exceptions;
1572                 vmcs12->guest_sysenter_esp = evmcs->guest_sysenter_esp;
1573                 vmcs12->guest_sysenter_eip = evmcs->guest_sysenter_eip;
1574                 vmcs12->guest_bndcfgs = evmcs->guest_bndcfgs;
1575                 vmcs12->guest_activity_state = evmcs->guest_activity_state;
1576                 vmcs12->guest_sysenter_cs = evmcs->guest_sysenter_cs;
1577         }
1578 
1579         /*
1580          * Not used?
1581          * vmcs12->vm_exit_msr_store_addr = evmcs->vm_exit_msr_store_addr;
1582          * vmcs12->vm_exit_msr_load_addr = evmcs->vm_exit_msr_load_addr;
1583          * vmcs12->vm_entry_msr_load_addr = evmcs->vm_entry_msr_load_addr;
1584          * vmcs12->cr3_target_value0 = evmcs->cr3_target_value0;
1585          * vmcs12->cr3_target_value1 = evmcs->cr3_target_value1;
1586          * vmcs12->cr3_target_value2 = evmcs->cr3_target_value2;
1587          * vmcs12->cr3_target_value3 = evmcs->cr3_target_value3;
1588          * vmcs12->page_fault_error_code_mask =
1589          *              evmcs->page_fault_error_code_mask;
1590          * vmcs12->page_fault_error_code_match =
1591          *              evmcs->page_fault_error_code_match;
1592          * vmcs12->cr3_target_count = evmcs->cr3_target_count;
1593          * vmcs12->vm_exit_msr_store_count = evmcs->vm_exit_msr_store_count;
1594          * vmcs12->vm_exit_msr_load_count = evmcs->vm_exit_msr_load_count;
1595          * vmcs12->vm_entry_msr_load_count = evmcs->vm_entry_msr_load_count;
1596          */
1597 
1598         /*
1599          * Read only fields:
1600          * vmcs12->guest_physical_address = evmcs->guest_physical_address;
1601          * vmcs12->vm_instruction_error = evmcs->vm_instruction_error;
1602          * vmcs12->vm_exit_reason = evmcs->vm_exit_reason;
1603          * vmcs12->vm_exit_intr_info = evmcs->vm_exit_intr_info;
1604          * vmcs12->vm_exit_intr_error_code = evmcs->vm_exit_intr_error_code;
1605          * vmcs12->idt_vectoring_info_field = evmcs->idt_vectoring_info_field;
1606          * vmcs12->idt_vectoring_error_code = evmcs->idt_vectoring_error_code;
1607          * vmcs12->vm_exit_instruction_len = evmcs->vm_exit_instruction_len;
1608          * vmcs12->vmx_instruction_info = evmcs->vmx_instruction_info;
1609          * vmcs12->exit_qualification = evmcs->exit_qualification;
1610          * vmcs12->guest_linear_address = evmcs->guest_linear_address;
1611          *
1612          * Not present in struct vmcs12:
1613          * vmcs12->exit_io_instruction_ecx = evmcs->exit_io_instruction_ecx;
1614          * vmcs12->exit_io_instruction_esi = evmcs->exit_io_instruction_esi;
1615          * vmcs12->exit_io_instruction_edi = evmcs->exit_io_instruction_edi;
1616          * vmcs12->exit_io_instruction_eip = evmcs->exit_io_instruction_eip;
1617          */
1618 
1619         return 0;
1620 }
1621 
1622 static int copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx)
1623 {
1624         struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12;
1625         struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs;
1626 
1627         /*
1628          * Should not be changed by KVM:
1629          *
1630          * evmcs->host_es_selector = vmcs12->host_es_selector;
1631          * evmcs->host_cs_selector = vmcs12->host_cs_selector;
1632          * evmcs->host_ss_selector = vmcs12->host_ss_selector;
1633          * evmcs->host_ds_selector = vmcs12->host_ds_selector;
1634          * evmcs->host_fs_selector = vmcs12->host_fs_selector;
1635          * evmcs->host_gs_selector = vmcs12->host_gs_selector;
1636          * evmcs->host_tr_selector = vmcs12->host_tr_selector;
1637          * evmcs->host_ia32_pat = vmcs12->host_ia32_pat;
1638          * evmcs->host_ia32_efer = vmcs12->host_ia32_efer;
1639          * evmcs->host_cr0 = vmcs12->host_cr0;
1640          * evmcs->host_cr3 = vmcs12->host_cr3;
1641          * evmcs->host_cr4 = vmcs12->host_cr4;
1642          * evmcs->host_ia32_sysenter_esp = vmcs12->host_ia32_sysenter_esp;
1643          * evmcs->host_ia32_sysenter_eip = vmcs12->host_ia32_sysenter_eip;
1644          * evmcs->host_rip = vmcs12->host_rip;
1645          * evmcs->host_ia32_sysenter_cs = vmcs12->host_ia32_sysenter_cs;
1646          * evmcs->host_fs_base = vmcs12->host_fs_base;
1647          * evmcs->host_gs_base = vmcs12->host_gs_base;
1648          * evmcs->host_tr_base = vmcs12->host_tr_base;
1649          * evmcs->host_gdtr_base = vmcs12->host_gdtr_base;
1650          * evmcs->host_idtr_base = vmcs12->host_idtr_base;
1651          * evmcs->host_rsp = vmcs12->host_rsp;
1652          * sync_vmcs02_to_vmcs12() doesn't read these:
1653          * evmcs->io_bitmap_a = vmcs12->io_bitmap_a;
1654          * evmcs->io_bitmap_b = vmcs12->io_bitmap_b;
1655          * evmcs->msr_bitmap = vmcs12->msr_bitmap;
1656          * evmcs->ept_pointer = vmcs12->ept_pointer;
1657          * evmcs->xss_exit_bitmap = vmcs12->xss_exit_bitmap;
1658          * evmcs->vm_exit_msr_store_addr = vmcs12->vm_exit_msr_store_addr;
1659          * evmcs->vm_exit_msr_load_addr = vmcs12->vm_exit_msr_load_addr;
1660          * evmcs->vm_entry_msr_load_addr = vmcs12->vm_entry_msr_load_addr;
1661          * evmcs->cr3_target_value0 = vmcs12->cr3_target_value0;
1662          * evmcs->cr3_target_value1 = vmcs12->cr3_target_value1;
1663          * evmcs->cr3_target_value2 = vmcs12->cr3_target_value2;
1664          * evmcs->cr3_target_value3 = vmcs12->cr3_target_value3;
1665          * evmcs->tpr_threshold = vmcs12->tpr_threshold;
1666          * evmcs->virtual_processor_id = vmcs12->virtual_processor_id;
1667          * evmcs->exception_bitmap = vmcs12->exception_bitmap;
1668          * evmcs->vmcs_link_pointer = vmcs12->vmcs_link_pointer;
1669          * evmcs->pin_based_vm_exec_control = vmcs12->pin_based_vm_exec_control;
1670          * evmcs->vm_exit_controls = vmcs12->vm_exit_controls;
1671          * evmcs->secondary_vm_exec_control = vmcs12->secondary_vm_exec_control;
1672          * evmcs->page_fault_error_code_mask =
1673          *              vmcs12->page_fault_error_code_mask;
1674          * evmcs->page_fault_error_code_match =
1675          *              vmcs12->page_fault_error_code_match;
1676          * evmcs->cr3_target_count = vmcs12->cr3_target_count;
1677          * evmcs->virtual_apic_page_addr = vmcs12->virtual_apic_page_addr;
1678          * evmcs->tsc_offset = vmcs12->tsc_offset;
1679          * evmcs->guest_ia32_debugctl = vmcs12->guest_ia32_debugctl;
1680          * evmcs->cr0_guest_host_mask = vmcs12->cr0_guest_host_mask;
1681          * evmcs->cr4_guest_host_mask = vmcs12->cr4_guest_host_mask;
1682          * evmcs->cr0_read_shadow = vmcs12->cr0_read_shadow;
1683          * evmcs->cr4_read_shadow = vmcs12->cr4_read_shadow;
1684          * evmcs->vm_exit_msr_store_count = vmcs12->vm_exit_msr_store_count;
1685          * evmcs->vm_exit_msr_load_count = vmcs12->vm_exit_msr_load_count;
1686          * evmcs->vm_entry_msr_load_count = vmcs12->vm_entry_msr_load_count;
1687          *
1688          * Not present in struct vmcs12:
1689          * evmcs->exit_io_instruction_ecx = vmcs12->exit_io_instruction_ecx;
1690          * evmcs->exit_io_instruction_esi = vmcs12->exit_io_instruction_esi;
1691          * evmcs->exit_io_instruction_edi = vmcs12->exit_io_instruction_edi;
1692          * evmcs->exit_io_instruction_eip = vmcs12->exit_io_instruction_eip;
1693          */
1694 
1695         evmcs->guest_es_selector = vmcs12->guest_es_selector;
1696         evmcs->guest_cs_selector = vmcs12->guest_cs_selector;
1697         evmcs->guest_ss_selector = vmcs12->guest_ss_selector;
1698         evmcs->guest_ds_selector = vmcs12->guest_ds_selector;
1699         evmcs->guest_fs_selector = vmcs12->guest_fs_selector;
1700         evmcs->guest_gs_selector = vmcs12->guest_gs_selector;
1701         evmcs->guest_ldtr_selector = vmcs12->guest_ldtr_selector;
1702         evmcs->guest_tr_selector = vmcs12->guest_tr_selector;
1703 
1704         evmcs->guest_es_limit = vmcs12->guest_es_limit;
1705         evmcs->guest_cs_limit = vmcs12->guest_cs_limit;
1706         evmcs->guest_ss_limit = vmcs12->guest_ss_limit;
1707         evmcs->guest_ds_limit = vmcs12->guest_ds_limit;
1708         evmcs->guest_fs_limit = vmcs12->guest_fs_limit;
1709         evmcs->guest_gs_limit = vmcs12->guest_gs_limit;
1710         evmcs->guest_ldtr_limit = vmcs12->guest_ldtr_limit;
1711         evmcs->guest_tr_limit = vmcs12->guest_tr_limit;
1712         evmcs->guest_gdtr_limit = vmcs12->guest_gdtr_limit;
1713         evmcs->guest_idtr_limit = vmcs12->guest_idtr_limit;
1714 
1715         evmcs->guest_es_ar_bytes = vmcs12->guest_es_ar_bytes;
1716         evmcs->guest_cs_ar_bytes = vmcs12->guest_cs_ar_bytes;
1717         evmcs->guest_ss_ar_bytes = vmcs12->guest_ss_ar_bytes;
1718         evmcs->guest_ds_ar_bytes = vmcs12->guest_ds_ar_bytes;
1719         evmcs->guest_fs_ar_bytes = vmcs12->guest_fs_ar_bytes;
1720         evmcs->guest_gs_ar_bytes = vmcs12->guest_gs_ar_bytes;
1721         evmcs->guest_ldtr_ar_bytes = vmcs12->guest_ldtr_ar_bytes;
1722         evmcs->guest_tr_ar_bytes = vmcs12->guest_tr_ar_bytes;
1723 
1724         evmcs->guest_es_base = vmcs12->guest_es_base;
1725         evmcs->guest_cs_base = vmcs12->guest_cs_base;
1726         evmcs->guest_ss_base = vmcs12->guest_ss_base;
1727         evmcs->guest_ds_base = vmcs12->guest_ds_base;
1728         evmcs->guest_fs_base = vmcs12->guest_fs_base;
1729         evmcs->guest_gs_base = vmcs12->guest_gs_base;
1730         evmcs->guest_ldtr_base = vmcs12->guest_ldtr_base;
1731         evmcs->guest_tr_base = vmcs12->guest_tr_base;
1732         evmcs->guest_gdtr_base = vmcs12->guest_gdtr_base;
1733         evmcs->guest_idtr_base = vmcs12->guest_idtr_base;
1734 
1735         evmcs->guest_ia32_pat = vmcs12->guest_ia32_pat;
1736         evmcs->guest_ia32_efer = vmcs12->guest_ia32_efer;
1737 
1738         evmcs->guest_pdptr0 = vmcs12->guest_pdptr0;
1739         evmcs->guest_pdptr1 = vmcs12->guest_pdptr1;
1740         evmcs->guest_pdptr2 = vmcs12->guest_pdptr2;
1741         evmcs->guest_pdptr3 = vmcs12->guest_pdptr3;
1742 
1743         evmcs->guest_pending_dbg_exceptions =
1744                 vmcs12->guest_pending_dbg_exceptions;
1745         evmcs->guest_sysenter_esp = vmcs12->guest_sysenter_esp;
1746         evmcs->guest_sysenter_eip = vmcs12->guest_sysenter_eip;
1747 
1748         evmcs->guest_activity_state = vmcs12->guest_activity_state;
1749         evmcs->guest_sysenter_cs = vmcs12->guest_sysenter_cs;
1750 
1751         evmcs->guest_cr0 = vmcs12->guest_cr0;
1752         evmcs->guest_cr3 = vmcs12->guest_cr3;
1753         evmcs->guest_cr4 = vmcs12->guest_cr4;
1754         evmcs->guest_dr7 = vmcs12->guest_dr7;
1755 
1756         evmcs->guest_physical_address = vmcs12->guest_physical_address;
1757 
1758         evmcs->vm_instruction_error = vmcs12->vm_instruction_error;
1759         evmcs->vm_exit_reason = vmcs12->vm_exit_reason;
1760         evmcs->vm_exit_intr_info = vmcs12->vm_exit_intr_info;
1761         evmcs->vm_exit_intr_error_code = vmcs12->vm_exit_intr_error_code;
1762         evmcs->idt_vectoring_info_field = vmcs12->idt_vectoring_info_field;
1763         evmcs->idt_vectoring_error_code = vmcs12->idt_vectoring_error_code;
1764         evmcs->vm_exit_instruction_len = vmcs12->vm_exit_instruction_len;
1765         evmcs->vmx_instruction_info = vmcs12->vmx_instruction_info;
1766 
1767         evmcs->exit_qualification = vmcs12->exit_qualification;
1768 
1769         evmcs->guest_linear_address = vmcs12->guest_linear_address;
1770         evmcs->guest_rsp = vmcs12->guest_rsp;
1771         evmcs->guest_rflags = vmcs12->guest_rflags;
1772 
1773         evmcs->guest_interruptibility_info =
1774                 vmcs12->guest_interruptibility_info;
1775         evmcs->cpu_based_vm_exec_control = vmcs12->cpu_based_vm_exec_control;
1776         evmcs->vm_entry_controls = vmcs12->vm_entry_controls;
1777         evmcs->vm_entry_intr_info_field = vmcs12->vm_entry_intr_info_field;
1778         evmcs->vm_entry_exception_error_code =
1779                 vmcs12->vm_entry_exception_error_code;
1780         evmcs->vm_entry_instruction_len = vmcs12->vm_entry_instruction_len;
1781 
1782         evmcs->guest_rip = vmcs12->guest_rip;
1783 
1784         evmcs->guest_bndcfgs = vmcs12->guest_bndcfgs;
1785 
1786         return 0;
1787 }
1788 
1789 /*
1790  * This is an equivalent of the nested hypervisor executing the vmptrld
1791  * instruction.
1792  */
1793 static int nested_vmx_handle_enlightened_vmptrld(struct kvm_vcpu *vcpu,
1794                                                  bool from_launch)
1795 {
1796         struct vcpu_vmx *vmx = to_vmx(vcpu);
1797         bool evmcs_gpa_changed = false;
1798         u64 evmcs_gpa;
1799 
1800         if (likely(!vmx->nested.enlightened_vmcs_enabled))
1801                 return 1;
1802 
1803         if (!nested_enlightened_vmentry(vcpu, &evmcs_gpa))
1804                 return 1;
1805 
1806         if (unlikely(evmcs_gpa != vmx->nested.hv_evmcs_vmptr)) {
1807                 if (!vmx->nested.hv_evmcs)
1808                         vmx->nested.current_vmptr = -1ull;
1809 
1810                 nested_release_evmcs(vcpu);
1811 
1812                 if (kvm_vcpu_map(vcpu, gpa_to_gfn(evmcs_gpa),
1813                                  &vmx->nested.hv_evmcs_map))
1814                         return 0;
1815 
1816                 vmx->nested.hv_evmcs = vmx->nested.hv_evmcs_map.hva;
1817 
1818                 /*
1819                  * Currently, KVM only supports eVMCS version 1
1820                  * (== KVM_EVMCS_VERSION) and thus we expect guest to set this
1821                  * value to first u32 field of eVMCS which should specify eVMCS
1822                  * VersionNumber.
1823                  *
1824                  * Guest should be aware of supported eVMCS versions by host by
1825                  * examining CPUID.0x4000000A.EAX[0:15]. Host userspace VMM is
1826                  * expected to set this CPUID leaf according to the value
1827                  * returned in vmcs_version from nested_enable_evmcs().
1828                  *
1829                  * However, it turns out that Microsoft Hyper-V fails to comply
1830                  * to their own invented interface: When Hyper-V use eVMCS, it
1831                  * just sets first u32 field of eVMCS to revision_id specified
1832                  * in MSR_IA32_VMX_BASIC. Instead of used eVMCS version number
1833                  * which is one of the supported versions specified in
1834                  * CPUID.0x4000000A.EAX[0:15].
1835                  *
1836                  * To overcome Hyper-V bug, we accept here either a supported
1837                  * eVMCS version or VMCS12 revision_id as valid values for first
1838                  * u32 field of eVMCS.
1839                  */
1840                 if ((vmx->nested.hv_evmcs->revision_id != KVM_EVMCS_VERSION) &&
1841                     (vmx->nested.hv_evmcs->revision_id != VMCS12_REVISION)) {
1842                         nested_release_evmcs(vcpu);
1843                         return 0;
1844                 }
1845 
1846                 vmx->nested.dirty_vmcs12 = true;
1847                 vmx->nested.hv_evmcs_vmptr = evmcs_gpa;
1848 
1849                 evmcs_gpa_changed = true;
1850                 /*
1851                  * Unlike normal vmcs12, enlightened vmcs12 is not fully
1852                  * reloaded from guest's memory (read only fields, fields not
1853                  * present in struct hv_enlightened_vmcs, ...). Make sure there
1854                  * are no leftovers.
1855                  */
1856                 if (from_launch) {
1857                         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
1858                         memset(vmcs12, 0, sizeof(*vmcs12));
1859                         vmcs12->hdr.revision_id = VMCS12_REVISION;
1860                 }
1861 
1862         }
1863 
1864         /*
1865          * Clean fields data can't de used on VMLAUNCH and when we switch
1866          * between different L2 guests as KVM keeps a single VMCS12 per L1.
1867          */
1868         if (from_launch || evmcs_gpa_changed)
1869                 vmx->nested.hv_evmcs->hv_clean_fields &=
1870                         ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL;
1871 
1872         return 1;
1873 }
1874 
1875 void nested_sync_vmcs12_to_shadow(struct kvm_vcpu *vcpu)
1876 {
1877         struct vcpu_vmx *vmx = to_vmx(vcpu);
1878 
1879         /*
1880          * hv_evmcs may end up being not mapped after migration (when
1881          * L2 was running), map it here to make sure vmcs12 changes are
1882          * properly reflected.
1883          */
1884         if (vmx->nested.enlightened_vmcs_enabled && !vmx->nested.hv_evmcs)
1885                 nested_vmx_handle_enlightened_vmptrld(vcpu, false);
1886 
1887         if (vmx->nested.hv_evmcs) {
1888                 copy_vmcs12_to_enlightened(vmx);
1889                 /* All fields are clean */
1890                 vmx->nested.hv_evmcs->hv_clean_fields |=
1891                         HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL;
1892         } else {
1893                 copy_vmcs12_to_shadow(vmx);
1894         }
1895 
1896         vmx->nested.need_vmcs12_to_shadow_sync = false;
1897 }
1898 
1899 static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer)
1900 {
1901         struct vcpu_vmx *vmx =
1902                 container_of(timer, struct vcpu_vmx, nested.preemption_timer);
1903 
1904         vmx->nested.preemption_timer_expired = true;
1905         kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu);
1906         kvm_vcpu_kick(&vmx->vcpu);
1907 
1908         return HRTIMER_NORESTART;
1909 }
1910 
1911 static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu)
1912 {
1913         u64 preemption_timeout = get_vmcs12(vcpu)->vmx_preemption_timer_value;
1914         struct vcpu_vmx *vmx = to_vmx(vcpu);
1915 
1916         /*
1917          * A timer value of zero is architecturally guaranteed to cause
1918          * a VMExit prior to executing any instructions in the guest.
1919          */
1920         if (preemption_timeout == 0) {
1921                 vmx_preemption_timer_fn(&vmx->nested.preemption_timer);
1922                 return;
1923         }
1924 
1925         if (vcpu->arch.virtual_tsc_khz == 0)
1926                 return;
1927 
1928         preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
1929         preemption_timeout *= 1000000;
1930         do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz);
1931         hrtimer_start(&vmx->nested.preemption_timer,
1932                       ns_to_ktime(preemption_timeout), HRTIMER_MODE_REL);
1933 }
1934 
1935 static u64 nested_vmx_calc_efer(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12)
1936 {
1937         if (vmx->nested.nested_run_pending &&
1938             (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER))
1939                 return vmcs12->guest_ia32_efer;
1940         else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE)
1941                 return vmx->vcpu.arch.efer | (EFER_LMA | EFER_LME);
1942         else
1943                 return vmx->vcpu.arch.efer & ~(EFER_LMA | EFER_LME);
1944 }
1945 
1946 static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx)
1947 {
1948         /*
1949          * If vmcs02 hasn't been initialized, set the constant vmcs02 state
1950          * according to L0's settings (vmcs12 is irrelevant here).  Host
1951          * fields that come from L0 and are not constant, e.g. HOST_CR3,
1952          * will be set as needed prior to VMLAUNCH/VMRESUME.
1953          */
1954         if (vmx->nested.vmcs02_initialized)
1955                 return;
1956         vmx->nested.vmcs02_initialized = true;
1957 
1958         /*
1959          * We don't care what the EPTP value is we just need to guarantee
1960          * it's valid so we don't get a false positive when doing early
1961          * consistency checks.
1962          */
1963         if (enable_ept && nested_early_check)
1964                 vmcs_write64(EPT_POINTER, construct_eptp(&vmx->vcpu, 0));
1965 
1966         /* All VMFUNCs are currently emulated through L0 vmexits.  */
1967         if (cpu_has_vmx_vmfunc())
1968                 vmcs_write64(VM_FUNCTION_CONTROL, 0);
1969 
1970         if (cpu_has_vmx_posted_intr())
1971                 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR);
1972 
1973         if (cpu_has_vmx_msr_bitmap())
1974                 vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap));
1975 
1976         /*
1977          * The PML address never changes, so it is constant in vmcs02.
1978          * Conceptually we want to copy the PML index from vmcs01 here,
1979          * and then back to vmcs01 on nested vmexit.  But since we flush
1980          * the log and reset GUEST_PML_INDEX on each vmexit, the PML
1981          * index is also effectively constant in vmcs02.
1982          */
1983         if (enable_pml) {
1984                 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg));
1985                 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
1986         }
1987 
1988         if (cpu_has_vmx_encls_vmexit())
1989                 vmcs_write64(ENCLS_EXITING_BITMAP, -1ull);
1990 
1991         /*
1992          * Set the MSR load/store lists to match L0's settings.  Only the
1993          * addresses are constant (for vmcs02), the counts can change based
1994          * on L2's behavior, e.g. switching to/from long mode.
1995          */
1996         vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
1997         vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val));
1998         vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val));
1999 
2000         vmx_set_constant_host_state(vmx);
2001 }
2002 
2003 static void prepare_vmcs02_early_rare(struct vcpu_vmx *vmx,
2004                                       struct vmcs12 *vmcs12)
2005 {
2006         prepare_vmcs02_constant_state(vmx);
2007 
2008         vmcs_write64(VMCS_LINK_POINTER, -1ull);
2009 
2010         if (enable_vpid) {
2011                 if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02)
2012                         vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02);
2013                 else
2014                         vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
2015         }
2016 }
2017 
2018 static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12)
2019 {
2020         u32 exec_control, vmcs12_exec_ctrl;
2021         u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12);
2022 
2023         if (vmx->nested.dirty_vmcs12 || vmx->nested.hv_evmcs)
2024                 prepare_vmcs02_early_rare(vmx, vmcs12);
2025 
2026         /*
2027          * PIN CONTROLS
2028          */
2029         exec_control = vmx_pin_based_exec_ctrl(vmx);
2030         exec_control |= (vmcs12->pin_based_vm_exec_control &
2031                          ~PIN_BASED_VMX_PREEMPTION_TIMER);
2032 
2033         /* Posted interrupts setting is only taken from vmcs12.  */
2034         if (nested_cpu_has_posted_intr(vmcs12)) {
2035                 vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv;
2036                 vmx->nested.pi_pending = false;
2037         } else {
2038                 exec_control &= ~PIN_BASED_POSTED_INTR;
2039         }
2040         pin_controls_set(vmx, exec_control);
2041 
2042         /*
2043          * EXEC CONTROLS
2044          */
2045         exec_control = vmx_exec_control(vmx); /* L0's desires */
2046         exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
2047         exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING;
2048         exec_control &= ~CPU_BASED_TPR_SHADOW;
2049         exec_control |= vmcs12->cpu_based_vm_exec_control;
2050 
2051         if (exec_control & CPU_BASED_TPR_SHADOW)
2052                 vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold);
2053 #ifdef CONFIG_X86_64
2054         else
2055                 exec_control |= CPU_BASED_CR8_LOAD_EXITING |
2056                                 CPU_BASED_CR8_STORE_EXITING;
2057 #endif
2058 
2059         /*
2060          * A vmexit (to either L1 hypervisor or L0 userspace) is always needed
2061          * for I/O port accesses.
2062          */
2063         exec_control |= CPU_BASED_UNCOND_IO_EXITING;
2064         exec_control &= ~CPU_BASED_USE_IO_BITMAPS;
2065 
2066         /*
2067          * This bit will be computed in nested_get_vmcs12_pages, because
2068          * we do not have access to L1's MSR bitmap yet.  For now, keep
2069          * the same bit as before, hoping to avoid multiple VMWRITEs that
2070          * only set/clear this bit.
2071          */
2072         exec_control &= ~CPU_BASED_USE_MSR_BITMAPS;
2073         exec_control |= exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS;
2074 
2075         exec_controls_set(vmx, exec_control);
2076 
2077         /*
2078          * SECONDARY EXEC CONTROLS
2079          */
2080         if (cpu_has_secondary_exec_ctrls()) {
2081                 exec_control = vmx->secondary_exec_control;
2082 
2083                 /* Take the following fields only from vmcs12 */
2084                 exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
2085                                   SECONDARY_EXEC_ENABLE_INVPCID |
2086                                   SECONDARY_EXEC_RDTSCP |
2087                                   SECONDARY_EXEC_XSAVES |
2088                                   SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
2089                                   SECONDARY_EXEC_APIC_REGISTER_VIRT |
2090                                   SECONDARY_EXEC_ENABLE_VMFUNC);
2091                 if (nested_cpu_has(vmcs12,
2092                                    CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) {
2093                         vmcs12_exec_ctrl = vmcs12->secondary_vm_exec_control &
2094                                 ~SECONDARY_EXEC_ENABLE_PML;
2095                         exec_control |= vmcs12_exec_ctrl;
2096                 }
2097 
2098                 /* VMCS shadowing for L2 is emulated for now */
2099                 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS;
2100 
2101                 /*
2102                  * Preset *DT exiting when emulating UMIP, so that vmx_set_cr4()
2103                  * will not have to rewrite the controls just for this bit.
2104                  */
2105                 if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated() &&
2106                     (vmcs12->guest_cr4 & X86_CR4_UMIP))
2107                         exec_control |= SECONDARY_EXEC_DESC;
2108 
2109                 if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)
2110                         vmcs_write16(GUEST_INTR_STATUS,
2111                                 vmcs12->guest_intr_status);
2112 
2113                 secondary_exec_controls_set(vmx, exec_control);
2114         }
2115 
2116         /*
2117          * ENTRY CONTROLS
2118          *
2119          * vmcs12's VM_{ENTRY,EXIT}_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE
2120          * are emulated by vmx_set_efer() in prepare_vmcs02(), but speculate
2121          * on the related bits (if supported by the CPU) in the hope that
2122          * we can avoid VMWrites during vmx_set_efer().
2123          */
2124         exec_control = (vmcs12->vm_entry_controls | vmx_vmentry_ctrl()) &
2125                         ~VM_ENTRY_IA32E_MODE & ~VM_ENTRY_LOAD_IA32_EFER;
2126         if (cpu_has_load_ia32_efer()) {
2127                 if (guest_efer & EFER_LMA)
2128                         exec_control |= VM_ENTRY_IA32E_MODE;
2129                 if (guest_efer != host_efer)
2130                         exec_control |= VM_ENTRY_LOAD_IA32_EFER;
2131         }
2132         vm_entry_controls_set(vmx, exec_control);
2133 
2134         /*
2135          * EXIT CONTROLS
2136          *
2137          * L2->L1 exit controls are emulated - the hardware exit is to L0 so
2138          * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER
2139          * bits may be modified by vmx_set_efer() in prepare_vmcs02().
2140          */
2141         exec_control = vmx_vmexit_ctrl();
2142         if (cpu_has_load_ia32_efer() && guest_efer != host_efer)
2143                 exec_control |= VM_EXIT_LOAD_IA32_EFER;
2144         vm_exit_controls_set(vmx, exec_control);
2145 
2146         /*
2147          * Interrupt/Exception Fields
2148          */
2149         if (vmx->nested.nested_run_pending) {
2150                 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2151                              vmcs12->vm_entry_intr_info_field);
2152                 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE,
2153                              vmcs12->vm_entry_exception_error_code);
2154                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
2155                              vmcs12->vm_entry_instruction_len);
2156                 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO,
2157                              vmcs12->guest_interruptibility_info);
2158                 vmx->loaded_vmcs->nmi_known_unmasked =
2159                         !(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_NMI);
2160         } else {
2161                 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
2162         }
2163 }
2164 
2165 static void prepare_vmcs02_rare(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12)
2166 {
2167         struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs;
2168 
2169         if (!hv_evmcs || !(hv_evmcs->hv_clean_fields &
2170                            HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) {
2171                 vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector);
2172                 vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector);
2173                 vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector);
2174                 vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector);
2175                 vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector);
2176                 vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector);
2177                 vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector);
2178                 vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector);
2179                 vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit);
2180                 vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit);
2181                 vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit);
2182                 vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit);
2183                 vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit);
2184                 vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit);
2185                 vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit);
2186                 vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit);
2187                 vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit);
2188                 vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit);
2189                 vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes);
2190                 vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes);
2191                 vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes);
2192                 vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes);
2193                 vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes);
2194                 vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes);
2195                 vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes);
2196                 vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes);
2197                 vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base);
2198                 vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base);
2199                 vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base);
2200                 vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base);
2201                 vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base);
2202                 vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base);
2203                 vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base);
2204                 vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base);
2205                 vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base);
2206                 vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base);
2207         }
2208 
2209         if (!hv_evmcs || !(hv_evmcs->hv_clean_fields &
2210                            HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1)) {
2211                 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs);
2212                 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
2213                             vmcs12->guest_pending_dbg_exceptions);
2214                 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp);
2215                 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip);
2216 
2217                 /*
2218                  * L1 may access the L2's PDPTR, so save them to construct
2219                  * vmcs12
2220                  */
2221                 if (enable_ept) {
2222                         vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0);
2223                         vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1);
2224                         vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2);
2225                         vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3);
2226                 }
2227 
2228                 if (kvm_mpx_supported() && vmx->nested.nested_run_pending &&
2229                     (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))
2230                         vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs);
2231         }
2232 
2233         if (nested_cpu_has_xsaves(vmcs12))
2234                 vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap);
2235 
2236         /*
2237          * Whether page-faults are trapped is determined by a combination of
2238          * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF.
2239          * If enable_ept, L0 doesn't care about page faults and we should
2240          * set all of these to L1's desires. However, if !enable_ept, L0 does
2241          * care about (at least some) page faults, and because it is not easy
2242          * (if at all possible?) to merge L0 and L1's desires, we simply ask
2243          * to exit on each and every L2 page fault. This is done by setting
2244          * MASK=MATCH=0 and (see below) EB.PF=1.
2245          * Note that below we don't need special code to set EB.PF beyond the
2246          * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept,
2247          * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when
2248          * !enable_ept, EB.PF is 1, so the "or" will always be 1.
2249          */
2250         vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK,
2251                 enable_ept ? vmcs12->page_fault_error_code_mask : 0);
2252         vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH,
2253                 enable_ept ? vmcs12->page_fault_error_code_match : 0);
2254 
2255         if (cpu_has_vmx_apicv()) {
2256                 vmcs_write64(EOI_EXIT_BITMAP0, vmcs12->eoi_exit_bitmap0);
2257                 vmcs_write64(EOI_EXIT_BITMAP1, vmcs12->eoi_exit_bitmap1);
2258                 vmcs_write64(EOI_EXIT_BITMAP2, vmcs12->eoi_exit_bitmap2);
2259                 vmcs_write64(EOI_EXIT_BITMAP3, vmcs12->eoi_exit_bitmap3);
2260         }
2261 
2262         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
2263         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
2264 
2265         set_cr4_guest_host_mask(vmx);
2266 }
2267 
2268 /*
2269  * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested
2270  * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it
2271  * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2
2272  * guest in a way that will both be appropriate to L1's requests, and our
2273  * needs. In addition to modifying the active vmcs (which is vmcs02), this
2274  * function also has additional necessary side-effects, like setting various
2275  * vcpu->arch fields.
2276  * Returns 0 on success, 1 on failure. Invalid state exit qualification code
2277  * is assigned to entry_failure_code on failure.
2278  */
2279 static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
2280                           u32 *entry_failure_code)
2281 {
2282         struct vcpu_vmx *vmx = to_vmx(vcpu);
2283         struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs;
2284         bool load_guest_pdptrs_vmcs12 = false;
2285 
2286         if (vmx->nested.dirty_vmcs12 || hv_evmcs) {
2287                 prepare_vmcs02_rare(vmx, vmcs12);
2288                 vmx->nested.dirty_vmcs12 = false;
2289 
2290                 load_guest_pdptrs_vmcs12 = !hv_evmcs ||
2291                         !(hv_evmcs->hv_clean_fields &
2292                           HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1);
2293         }
2294 
2295         if (vmx->nested.nested_run_pending &&
2296             (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) {
2297                 kvm_set_dr(vcpu, 7, vmcs12->guest_dr7);
2298                 vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl);
2299         } else {
2300                 kvm_set_dr(vcpu, 7, vcpu->arch.dr7);
2301                 vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.vmcs01_debugctl);
2302         }
2303         if (kvm_mpx_supported() && (!vmx->nested.nested_run_pending ||
2304             !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)))
2305                 vmcs_write64(GUEST_BNDCFGS, vmx->nested.vmcs01_guest_bndcfgs);
2306         vmx_set_rflags(vcpu, vmcs12->guest_rflags);
2307 
2308         /* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the
2309          * bitwise-or of what L1 wants to trap for L2, and what we want to
2310          * trap. Note that CR0.TS also needs updating - we do this later.
2311          */
2312         update_exception_bitmap(vcpu);
2313         vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask;
2314         vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
2315 
2316         if (vmx->nested.nested_run_pending &&
2317             (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) {
2318                 vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat);
2319                 vcpu->arch.pat = vmcs12->guest_ia32_pat;
2320         } else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
2321                 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);
2322         }
2323 
2324         vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
2325 
2326         if (kvm_has_tsc_control)
2327                 decache_tsc_multiplier(vmx);
2328 
2329         if (enable_vpid) {
2330                 /*
2331                  * There is no direct mapping between vpid02 and vpid12, the
2332                  * vpid02 is per-vCPU for L0 and reused while the value of
2333                  * vpid12 is changed w/ one invvpid during nested vmentry.
2334                  * The vpid12 is allocated by L1 for L2, so it will not
2335                  * influence global bitmap(for vpid01 and vpid02 allocation)
2336                  * even if spawn a lot of nested vCPUs.
2337                  */
2338                 if (nested_cpu_has_vpid(vmcs12) && nested_has_guest_tlb_tag(vcpu)) {
2339                         if (vmcs12->virtual_processor_id != vmx->nested.last_vpid) {
2340                                 vmx->nested.last_vpid = vmcs12->virtual_processor_id;
2341                                 __vmx_flush_tlb(vcpu, nested_get_vpid02(vcpu), false);
2342                         }
2343                 } else {
2344                         /*
2345                          * If L1 use EPT, then L0 needs to execute INVEPT on
2346                          * EPTP02 instead of EPTP01. Therefore, delay TLB
2347                          * flush until vmcs02->eptp is fully updated by
2348                          * KVM_REQ_LOAD_CR3. Note that this assumes
2349                          * KVM_REQ_TLB_FLUSH is evaluated after
2350                          * KVM_REQ_LOAD_CR3 in vcpu_enter_guest().
2351                          */
2352                         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2353                 }
2354         }
2355 
2356         if (nested_cpu_has_ept(vmcs12))
2357                 nested_ept_init_mmu_context(vcpu);
2358         else if (nested_cpu_has2(vmcs12,
2359                                  SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
2360                 vmx_flush_tlb(vcpu, true);
2361 
2362         /*
2363          * This sets GUEST_CR0 to vmcs12->guest_cr0, possibly modifying those
2364          * bits which we consider mandatory enabled.
2365          * The CR0_READ_SHADOW is what L2 should have expected to read given
2366          * the specifications by L1; It's not enough to take
2367          * vmcs12->cr0_read_shadow because on our cr0_guest_host_mask we we
2368          * have more bits than L1 expected.
2369          */
2370         vmx_set_cr0(vcpu, vmcs12->guest_cr0);
2371         vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12));
2372 
2373         vmx_set_cr4(vcpu, vmcs12->guest_cr4);
2374         vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12));
2375 
2376         vcpu->arch.efer = nested_vmx_calc_efer(vmx, vmcs12);
2377         /* Note: may modify VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */
2378         vmx_set_efer(vcpu, vcpu->arch.efer);
2379 
2380         /*
2381          * Guest state is invalid and unrestricted guest is disabled,
2382          * which means L1 attempted VMEntry to L2 with invalid state.
2383          * Fail the VMEntry.
2384          */
2385         if (vmx->emulation_required) {
2386                 *entry_failure_code = ENTRY_FAIL_DEFAULT;
2387                 return -EINVAL;
2388         }
2389 
2390         /* Shadow page tables on either EPT or shadow page tables. */
2391         if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_cpu_has_ept(vmcs12),
2392                                 entry_failure_code))
2393                 return -EINVAL;
2394 
2395         /*
2396          * Immediately write vmcs02.GUEST_CR3.  It will be propagated to vmcs12
2397          * on nested VM-Exit, which can occur without actually running L2 and
2398          * thus without hitting vmx_set_cr3(), e.g. if L1 is entering L2 with
2399          * vmcs12.GUEST_ACTIVITYSTATE=HLT, in which case KVM will intercept the
2400          * transition to HLT instead of running L2.
2401          */
2402         if (enable_ept)
2403                 vmcs_writel(GUEST_CR3, vmcs12->guest_cr3);
2404 
2405         /* Late preparation of GUEST_PDPTRs now that EFER and CRs are set. */
2406         if (load_guest_pdptrs_vmcs12 && nested_cpu_has_ept(vmcs12) &&
2407             is_pae_paging(vcpu)) {
2408                 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0);
2409                 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1);
2410                 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2);
2411                 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3);
2412         }
2413 
2414         if (!enable_ept)
2415                 vcpu->arch.walk_mmu->inject_page_fault = vmx_inject_page_fault_nested;
2416 
2417         kvm_rsp_write(vcpu, vmcs12->guest_rsp);
2418         kvm_rip_write(vcpu, vmcs12->guest_rip);
2419         return 0;
2420 }
2421 
2422 static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12)
2423 {
2424         if (!nested_cpu_has_nmi_exiting(vmcs12) &&
2425             nested_cpu_has_virtual_nmis(vmcs12))
2426                 return -EINVAL;
2427 
2428         if (!nested_cpu_has_virtual_nmis(vmcs12) &&
2429             nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING))
2430                 return -EINVAL;
2431 
2432         return 0;
2433 }
2434 
2435 static bool valid_ept_address(struct kvm_vcpu *vcpu, u64 address)
2436 {
2437         struct vcpu_vmx *vmx = to_vmx(vcpu);
2438         int maxphyaddr = cpuid_maxphyaddr(vcpu);
2439 
2440         /* Check for memory type validity */
2441         switch (address & VMX_EPTP_MT_MASK) {
2442         case VMX_EPTP_MT_UC:
2443                 if (!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT))
2444                         return false;
2445                 break;
2446         case VMX_EPTP_MT_WB:
2447                 if (!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT))
2448                         return false;
2449                 break;
2450         default:
2451                 return false;
2452         }
2453 
2454         /* only 4 levels page-walk length are valid */
2455         if ((address & VMX_EPTP_PWL_MASK) != VMX_EPTP_PWL_4)
2456                 return false;
2457 
2458         /* Reserved bits should not be set */
2459         if (address >> maxphyaddr || ((address >> 7) & 0x1f))
2460                 return false;
2461 
2462         /* AD, if set, should be supported */
2463         if (address & VMX_EPTP_AD_ENABLE_BIT) {
2464                 if (!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT))
2465                         return false;
2466         }
2467 
2468         return true;
2469 }
2470 
2471 /*
2472  * Checks related to VM-Execution Control Fields
2473  */
2474 static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu,
2475                                               struct vmcs12 *vmcs12)
2476 {
2477         struct vcpu_vmx *vmx = to_vmx(vcpu);
2478 
2479         if (!vmx_control_verify(vmcs12->pin_based_vm_exec_control,
2480                                 vmx->nested.msrs.pinbased_ctls_low,
2481                                 vmx->nested.msrs.pinbased_ctls_high) ||
2482             !vmx_control_verify(vmcs12->cpu_based_vm_exec_control,
2483                                 vmx->nested.msrs.procbased_ctls_low,
2484                                 vmx->nested.msrs.procbased_ctls_high))
2485                 return -EINVAL;
2486 
2487         if (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) &&
2488             !vmx_control_verify(vmcs12->secondary_vm_exec_control,
2489                                  vmx->nested.msrs.secondary_ctls_low,
2490                                  vmx->nested.msrs.secondary_ctls_high))
2491                 return -EINVAL;
2492 
2493         if (vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu) ||
2494             nested_vmx_check_io_bitmap_controls(vcpu, vmcs12) ||
2495             nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12) ||
2496             nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12) ||
2497             nested_vmx_check_apic_access_controls(vcpu, vmcs12) ||
2498             nested_vmx_check_apicv_controls(vcpu, vmcs12) ||
2499             nested_vmx_check_nmi_controls(vmcs12) ||
2500             nested_vmx_check_pml_controls(vcpu, vmcs12) ||
2501             nested_vmx_check_unrestricted_guest_controls(vcpu, vmcs12) ||
2502             nested_vmx_check_mode_based_ept_exec_controls(vcpu, vmcs12) ||
2503             nested_vmx_check_shadow_vmcs_controls(vcpu, vmcs12) ||
2504             (nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id))
2505                 return -EINVAL;
2506 
2507         if (!nested_cpu_has_preemption_timer(vmcs12) &&
2508             nested_cpu_has_save_preemption_timer(vmcs12))
2509                 return -EINVAL;
2510 
2511         if (nested_cpu_has_ept(vmcs12) &&
2512             !valid_ept_address(vcpu, vmcs12->ept_pointer))
2513                 return -EINVAL;
2514 
2515         if (nested_cpu_has_vmfunc(vmcs12)) {
2516                 if (vmcs12->vm_function_control &
2517                     ~vmx->nested.msrs.vmfunc_controls)
2518                         return -EINVAL;
2519 
2520                 if (nested_cpu_has_eptp_switching(vmcs12)) {
2521                         if (!nested_cpu_has_ept(vmcs12) ||
2522                             !page_address_valid(vcpu, vmcs12->eptp_list_address))
2523                                 return -EINVAL;
2524                 }
2525         }
2526 
2527         return 0;
2528 }
2529 
2530 /*
2531  * Checks related to VM-Exit Control Fields
2532  */
2533 static int nested_check_vm_exit_controls(struct kvm_vcpu *vcpu,
2534                                          struct vmcs12 *vmcs12)
2535 {
2536         struct vcpu_vmx *vmx = to_vmx(vcpu);
2537 
2538         if (!vmx_control_verify(vmcs12->vm_exit_controls,
2539                                 vmx->nested.msrs.exit_ctls_low,
2540                                 vmx->nested.msrs.exit_ctls_high) ||
2541             nested_vmx_check_exit_msr_switch_controls(vcpu, vmcs12))
2542                 return -EINVAL;
2543 
2544         return 0;
2545 }
2546 
2547 /*
2548  * Checks related to VM-Entry Control Fields
2549  */
2550 static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu,
2551                                           struct vmcs12 *vmcs12)
2552 {
2553         struct vcpu_vmx *vmx = to_vmx(vcpu);
2554 
2555         if (!vmx_control_verify(vmcs12->vm_entry_controls,
2556                                 vmx->nested.msrs.entry_ctls_low,
2557                                 vmx->nested.msrs.entry_ctls_high))
2558                 return -EINVAL;
2559 
2560         /*
2561          * From the Intel SDM, volume 3:
2562          * Fields relevant to VM-entry event injection must be set properly.
2563          * These fields are the VM-entry interruption-information field, the
2564          * VM-entry exception error code, and the VM-entry instruction length.
2565          */
2566         if (vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) {
2567                 u32 intr_info = vmcs12->vm_entry_intr_info_field;
2568                 u8 vector = intr_info & INTR_INFO_VECTOR_MASK;
2569                 u32 intr_type = intr_info & INTR_INFO_INTR_TYPE_MASK;
2570                 bool has_error_code = intr_info & INTR_INFO_DELIVER_CODE_MASK;
2571                 bool should_have_error_code;
2572                 bool urg = nested_cpu_has2(vmcs12,
2573                                            SECONDARY_EXEC_UNRESTRICTED_GUEST);
2574                 bool prot_mode = !urg || vmcs12->guest_cr0 & X86_CR0_PE;
2575 
2576                 /* VM-entry interruption-info field: interruption type */
2577                 if (intr_type == INTR_TYPE_RESERVED ||
2578                     (intr_type == INTR_TYPE_OTHER_EVENT &&
2579                      !nested_cpu_supports_monitor_trap_flag(vcpu)))
2580                         return -EINVAL;
2581 
2582                 /* VM-entry interruption-info field: vector */
2583                 if ((intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) ||
2584                     (intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) ||
2585                     (intr_type == INTR_TYPE_OTHER_EVENT && vector != 0))
2586                         return -EINVAL;
2587 
2588                 /* VM-entry interruption-info field: deliver error code */
2589                 should_have_error_code =
2590                         intr_type == INTR_TYPE_HARD_EXCEPTION && prot_mode &&
2591                         x86_exception_has_error_code(vector);
2592                 if (has_error_code != should_have_error_code)
2593                         return -EINVAL;
2594 
2595                 /* VM-entry exception error code */
2596                 if (has_error_code &&
2597                        vmcs12->vm_entry_exception_error_code & GENMASK(31, 16))
2598                         return -EINVAL;
2599 
2600                 /* VM-entry interruption-info field: reserved bits */
2601                 if (intr_info & INTR_INFO_RESVD_BITS_MASK)
2602                         return -EINVAL;
2603 
2604                 /* VM-entry instruction length */
2605                 switch (intr_type) {
2606                 case INTR_TYPE_SOFT_EXCEPTION:
2607                 case INTR_TYPE_SOFT_INTR:
2608                 case INTR_TYPE_PRIV_SW_EXCEPTION:
2609                         if ((vmcs12->vm_entry_instruction_len > 15) ||
2610                             (vmcs12->vm_entry_instruction_len == 0 &&
2611                              !nested_cpu_has_zero_length_injection(vcpu)))
2612                                 return -EINVAL;
2613                 }
2614         }
2615 
2616         if (nested_vmx_check_entry_msr_switch_controls(vcpu, vmcs12))
2617                 return -EINVAL;
2618 
2619         return 0;
2620 }
2621 
2622 static int nested_vmx_check_controls(struct kvm_vcpu *vcpu,
2623                                      struct vmcs12 *vmcs12)
2624 {
2625         if (nested_check_vm_execution_controls(vcpu, vmcs12) ||
2626             nested_check_vm_exit_controls(vcpu, vmcs12) ||
2627             nested_check_vm_entry_controls(vcpu, vmcs12))
2628                 return -EINVAL;
2629 
2630         return 0;
2631 }
2632 
2633 static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu,
2634                                        struct vmcs12 *vmcs12)
2635 {
2636         bool ia32e;
2637 
2638         if (!nested_host_cr0_valid(vcpu, vmcs12->host_cr0) ||
2639             !nested_host_cr4_valid(vcpu, vmcs12->host_cr4) ||
2640             !nested_cr3_valid(vcpu, vmcs12->host_cr3))
2641                 return -EINVAL;
2642 
2643         if (is_noncanonical_address(vmcs12->host_ia32_sysenter_esp, vcpu) ||
2644             is_noncanonical_address(vmcs12->host_ia32_sysenter_eip, vcpu))
2645                 return -EINVAL;
2646 
2647         if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) &&
2648             !kvm_pat_valid(vmcs12->host_ia32_pat))
2649                 return -EINVAL;
2650 
2651         ia32e = (vmcs12->vm_exit_controls &
2652                  VM_EXIT_HOST_ADDR_SPACE_SIZE) != 0;
2653 
2654         if (vmcs12->host_cs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK) ||
2655             vmcs12->host_ss_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK) ||
2656             vmcs12->host_ds_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK) ||
2657             vmcs12->host_es_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK) ||
2658             vmcs12->host_fs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK) ||
2659             vmcs12->host_gs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK) ||
2660             vmcs12->host_tr_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK) ||
2661             vmcs12->host_cs_selector == 0 ||
2662             vmcs12->host_tr_selector == 0 ||
2663             (vmcs12->host_ss_selector == 0 && !ia32e))
2664                 return -EINVAL;
2665 
2666 #ifdef CONFIG_X86_64
2667         if (is_noncanonical_address(vmcs12->host_fs_base, vcpu) ||
2668             is_noncanonical_address(vmcs12->host_gs_base, vcpu) ||
2669             is_noncanonical_address(vmcs12->host_gdtr_base, vcpu) ||
2670             is_noncanonical_address(vmcs12->host_idtr_base, vcpu) ||
2671             is_noncanonical_address(vmcs12->host_tr_base, vcpu))
2672                 return -EINVAL;
2673 #endif
2674 
2675         /*
2676          * If the load IA32_EFER VM-exit control is 1, bits reserved in the
2677          * IA32_EFER MSR must be 0 in the field for that register. In addition,
2678          * the values of the LMA and LME bits in the field must each be that of
2679          * the host address-space size VM-exit control.
2680          */
2681         if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) {
2682                 if (!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer) ||
2683                     ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA) ||
2684                     ia32e != !!(vmcs12->host_ia32_efer & EFER_LME))
2685                         return -EINVAL;
2686         }
2687 
2688         return 0;
2689 }
2690 
2691 static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu,
2692                                           struct vmcs12 *vmcs12)
2693 {
2694         int r = 0;
2695         struct vmcs12 *shadow;
2696         struct kvm_host_map map;
2697 
2698         if (vmcs12->vmcs_link_pointer == -1ull)
2699                 return 0;
2700 
2701         if (!page_address_valid(vcpu, vmcs12->vmcs_link_pointer))
2702                 return -EINVAL;
2703 
2704         if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->vmcs_link_pointer), &map))
2705                 return -EINVAL;
2706 
2707         shadow = map.hva;
2708 
2709         if (shadow->hdr.revision_id != VMCS12_REVISION ||
2710             shadow->hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12))
2711                 r = -EINVAL;
2712 
2713         kvm_vcpu_unmap(vcpu, &map, false);
2714         return r;
2715 }
2716 
2717 /*
2718  * Checks related to Guest Non-register State
2719  */
2720 static int nested_check_guest_non_reg_state(struct vmcs12 *vmcs12)
2721 {
2722         if (vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE &&
2723             vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT)
2724                 return -EINVAL;
2725 
2726         return 0;
2727 }
2728 
2729 static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu,
2730                                         struct vmcs12 *vmcs12,
2731                                         u32 *exit_qual)
2732 {
2733         bool ia32e;
2734 
2735         *exit_qual = ENTRY_FAIL_DEFAULT;
2736 
2737         if (!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0) ||
2738             !nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4))
2739                 return -EINVAL;
2740 
2741         if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) &&
2742             !kvm_pat_valid(vmcs12->guest_ia32_pat))
2743                 return -EINVAL;
2744 
2745         if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) {
2746                 *exit_qual = ENTRY_FAIL_VMCS_LINK_PTR;
2747                 return -EINVAL;
2748         }
2749 
2750         /*
2751          * If the load IA32_EFER VM-entry control is 1, the following checks
2752          * are performed on the field for the IA32_EFER MSR:
2753          * - Bits reserved in the IA32_EFER MSR must be 0.
2754          * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of
2755          *   the IA-32e mode guest VM-exit control. It must also be identical
2756          *   to bit 8 (LME) if bit 31 in the CR0 field (corresponding to
2757          *   CR0.PG) is 1.
2758          */
2759         if (to_vmx(vcpu)->nested.nested_run_pending &&
2760             (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) {
2761                 ia32e = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) != 0;
2762                 if (!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer) ||
2763                     ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA) ||
2764                     ((vmcs12->guest_cr0 & X86_CR0_PG) &&
2765                      ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME)))
2766                         return -EINVAL;
2767         }
2768 
2769         if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS) &&
2770             (is_noncanonical_address(vmcs12->guest_bndcfgs & PAGE_MASK, vcpu) ||
2771              (vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD)))
2772                 return -EINVAL;
2773 
2774         if (nested_check_guest_non_reg_state(vmcs12))
2775                 return -EINVAL;
2776 
2777         return 0;
2778 }
2779 
2780 static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu)
2781 {
2782         struct vcpu_vmx *vmx = to_vmx(vcpu);
2783         unsigned long cr3, cr4;
2784         bool vm_fail;
2785 
2786         if (!nested_early_check)
2787                 return 0;
2788 
2789         if (vmx->msr_autoload.host.nr)
2790                 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
2791         if (vmx->msr_autoload.guest.nr)
2792                 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
2793 
2794         preempt_disable();
2795 
2796         vmx_prepare_switch_to_guest(vcpu);
2797 
2798         /*
2799          * Induce a consistency check VMExit by clearing bit 1 in GUEST_RFLAGS,
2800          * which is reserved to '1' by hardware.  GUEST_RFLAGS is guaranteed to
2801          * be written (by preparve_vmcs02()) before the "real" VMEnter, i.e.
2802          * there is no need to preserve other bits or save/restore the field.
2803          */
2804         vmcs_writel(GUEST_RFLAGS, 0);
2805 
2806         cr3 = __get_current_cr3_fast();
2807         if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) {
2808                 vmcs_writel(HOST_CR3, cr3);
2809                 vmx->loaded_vmcs->host_state.cr3 = cr3;
2810         }
2811 
2812         cr4 = cr4_read_shadow();
2813         if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) {
2814                 vmcs_writel(HOST_CR4, cr4);
2815                 vmx->loaded_vmcs->host_state.cr4 = cr4;
2816         }
2817 
2818         asm(
2819                 "sub $%c[wordsize], %%" _ASM_SP "\n\t" /* temporarily adjust RSP for CALL */
2820                 "cmp %%" _ASM_SP ", %c[host_state_rsp](%[loaded_vmcs]) \n\t"
2821                 "je 1f \n\t"
2822                 __ex("vmwrite %%" _ASM_SP ", %[HOST_RSP]") "\n\t"
2823                 "mov %%" _ASM_SP ", %c[host_state_rsp](%[loaded_vmcs]) \n\t"
2824                 "1: \n\t"
2825                 "add $%c[wordsize], %%" _ASM_SP "\n\t" /* un-adjust RSP */
2826 
2827                 /* Check if vmlaunch or vmresume is needed */
2828                 "cmpb $0, %c[launched](%[loaded_vmcs])\n\t"
2829 
2830                 /*
2831                  * VMLAUNCH and VMRESUME clear RFLAGS.{CF,ZF} on VM-Exit, set
2832                  * RFLAGS.CF on VM-Fail Invalid and set RFLAGS.ZF on VM-Fail
2833                  * Valid.  vmx_vmenter() directly "returns" RFLAGS, and so the
2834                  * results of VM-Enter is captured via CC_{SET,OUT} to vm_fail.
2835                  */
2836                 "call vmx_vmenter\n\t"
2837 
2838                 CC_SET(be)
2839               : ASM_CALL_CONSTRAINT, CC_OUT(be) (vm_fail)
2840               : [HOST_RSP]"r"((unsigned long)HOST_RSP),
2841                 [loaded_vmcs]"r"(vmx->loaded_vmcs),
2842                 [launched]"i"(offsetof(struct loaded_vmcs, launched)),
2843                 [host_state_rsp]"i"(offsetof(struct loaded_vmcs, host_state.rsp)),
2844                 [wordsize]"i"(sizeof(ulong))
2845               : "memory"
2846         );
2847 
2848         if (vmx->msr_autoload.host.nr)
2849                 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
2850         if (vmx->msr_autoload.guest.nr)
2851                 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
2852 
2853         if (vm_fail) {
2854                 preempt_enable();
2855                 WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) !=
2856                              VMXERR_ENTRY_INVALID_CONTROL_FIELD);
2857                 return 1;
2858         }
2859 
2860         /*
2861          * VMExit clears RFLAGS.IF and DR7, even on a consistency check.
2862          */
2863         local_irq_enable();
2864         if (hw_breakpoint_active())
2865                 set_debugreg(__this_cpu_read(cpu_dr7), 7);
2866         preempt_enable();
2867 
2868         /*
2869          * A non-failing VMEntry means we somehow entered guest mode with
2870          * an illegal RIP, and that's just the tip of the iceberg.  There
2871          * is no telling what memory has been modified or what state has
2872          * been exposed to unknown code.  Hitting this all but guarantees
2873          * a (very critical) hardware issue.
2874          */
2875         WARN_ON(!(vmcs_read32(VM_EXIT_REASON) &
2876                 VMX_EXIT_REASONS_FAILED_VMENTRY));
2877 
2878         return 0;
2879 }
2880 
2881 static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu,
2882                                                  struct vmcs12 *vmcs12);
2883 
2884 static void nested_get_vmcs12_pages(struct kvm_vcpu *vcpu)
2885 {
2886         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
2887         struct vcpu_vmx *vmx = to_vmx(vcpu);
2888         struct kvm_host_map *map;
2889         struct page *page;
2890         u64 hpa;
2891 
2892         if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
2893                 /*
2894                  * Translate L1 physical address to host physical
2895                  * address for vmcs02. Keep the page pinned, so this
2896                  * physical address remains valid. We keep a reference
2897                  * to it so we can release it later.
2898                  */
2899                 if (vmx->nested.apic_access_page) { /* shouldn't happen */
2900                         kvm_release_page_dirty(vmx->nested.apic_access_page);
2901                         vmx->nested.apic_access_page = NULL;
2902                 }
2903                 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->apic_access_addr);
2904                 /*
2905                  * If translation failed, no matter: This feature asks
2906                  * to exit when accessing the given address, and if it
2907                  * can never be accessed, this feature won't do
2908                  * anything anyway.
2909                  */
2910                 if (!is_error_page(page)) {
2911                         vmx->nested.apic_access_page = page;
2912                         hpa = page_to_phys(vmx->nested.apic_access_page);
2913                         vmcs_write64(APIC_ACCESS_ADDR, hpa);
2914                 } else {
2915                         secondary_exec_controls_clearbit(vmx,
2916                                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES);
2917                 }
2918         }
2919 
2920         if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
2921                 map = &vmx->nested.virtual_apic_map;
2922 
2923                 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->virtual_apic_page_addr), map)) {
2924                         vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, pfn_to_hpa(map->pfn));
2925                 } else if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING) &&
2926                            nested_cpu_has(vmcs12, CPU_BASED_CR8_STORE_EXITING) &&
2927                            !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
2928                         /*
2929                          * The processor will never use the TPR shadow, simply
2930                          * clear the bit from the execution control.  Such a
2931                          * configuration is useless, but it happens in tests.
2932                          * For any other configuration, failing the vm entry is
2933                          * _not_ what the processor does but it's basically the
2934                          * only possibility we have.
2935                          */
2936                         exec_controls_clearbit(vmx, CPU_BASED_TPR_SHADOW);
2937                 } else {
2938                         /*
2939                          * Write an illegal value to VIRTUAL_APIC_PAGE_ADDR to
2940                          * force VM-Entry to fail.
2941                          */
2942                         vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, -1ull);
2943                 }
2944         }
2945 
2946         if (nested_cpu_has_posted_intr(vmcs12)) {
2947                 map = &vmx->nested.pi_desc_map;
2948 
2949                 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->posted_intr_desc_addr), map)) {
2950                         vmx->nested.pi_desc =
2951                                 (struct pi_desc *)(((void *)map->hva) +
2952                                 offset_in_page(vmcs12->posted_intr_desc_addr));
2953                         vmcs_write64(POSTED_INTR_DESC_ADDR,
2954                                      pfn_to_hpa(map->pfn) + offset_in_page(vmcs12->posted_intr_desc_addr));
2955                 }
2956         }
2957         if (nested_vmx_prepare_msr_bitmap(vcpu, vmcs12))
2958                 exec_controls_setbit(vmx, CPU_BASED_USE_MSR_BITMAPS);
2959         else
2960                 exec_controls_clearbit(vmx, CPU_BASED_USE_MSR_BITMAPS);
2961 }
2962 
2963 /*
2964  * Intel's VMX Instruction Reference specifies a common set of prerequisites
2965  * for running VMX instructions (except VMXON, whose prerequisites are
2966  * slightly different). It also specifies what exception to inject otherwise.
2967  * Note that many of these exceptions have priority over VM exits, so they
2968  * don't have to be checked again here.
2969  */
2970 static int nested_vmx_check_permission(struct kvm_vcpu *vcpu)
2971 {
2972         if (!to_vmx(vcpu)->nested.vmxon) {
2973                 kvm_queue_exception(vcpu, UD_VECTOR);
2974                 return 0;
2975         }
2976 
2977         if (vmx_get_cpl(vcpu)) {
2978                 kvm_inject_gp(vcpu, 0);
2979                 return 0;
2980         }
2981 
2982         return 1;
2983 }
2984 
2985 static u8 vmx_has_apicv_interrupt(struct kvm_vcpu *vcpu)
2986 {
2987         u8 rvi = vmx_get_rvi();
2988         u8 vppr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_PROCPRI);
2989 
2990         return ((rvi & 0xf0) > (vppr & 0xf0));
2991 }
2992 
2993 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
2994                                    struct vmcs12 *vmcs12);
2995 
2996 /*
2997  * If from_vmentry is false, this is being called from state restore (either RSM
2998  * or KVM_SET_NESTED_STATE).  Otherwise it's called from vmlaunch/vmresume.
2999 + *
3000 + * Returns:
3001 + *   0 - success, i.e. proceed with actual VMEnter
3002 + *   1 - consistency check VMExit
3003 + *  -1 - consistency check VMFail
3004  */
3005 int nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, bool from_vmentry)
3006 {
3007         struct vcpu_vmx *vmx = to_vmx(vcpu);
3008         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3009         bool evaluate_pending_interrupts;
3010         u32 exit_reason = EXIT_REASON_INVALID_STATE;
3011         u32 exit_qual;
3012 
3013         evaluate_pending_interrupts = exec_controls_get(vmx) &
3014                 (CPU_BASED_VIRTUAL_INTR_PENDING | CPU_BASED_VIRTUAL_NMI_PENDING);
3015         if (likely(!evaluate_pending_interrupts) && kvm_vcpu_apicv_active(vcpu))
3016                 evaluate_pending_interrupts |= vmx_has_apicv_interrupt(vcpu);
3017 
3018         if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS))
3019                 vmx->nested.vmcs01_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL);
3020         if (kvm_mpx_supported() &&
3021                 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))
3022                 vmx->nested.vmcs01_guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS);
3023 
3024         /*
3025          * Overwrite vmcs01.GUEST_CR3 with L1's CR3 if EPT is disabled *and*
3026          * nested early checks are disabled.  In the event of a "late" VM-Fail,
3027          * i.e. a VM-Fail detected by hardware but not KVM, KVM must unwind its
3028          * software model to the pre-VMEntry host state.  When EPT is disabled,
3029          * GUEST_CR3 holds KVM's shadow CR3, not L1's "real" CR3, which causes
3030          * nested_vmx_restore_host_state() to corrupt vcpu->arch.cr3.  Stuffing
3031          * vmcs01.GUEST_CR3 results in the unwind naturally setting arch.cr3 to
3032          * the correct value.  Smashing vmcs01.GUEST_CR3 is safe because nested
3033          * VM-Exits, and the unwind, reset KVM's MMU, i.e. vmcs01.GUEST_CR3 is
3034          * guaranteed to be overwritten with a shadow CR3 prior to re-entering
3035          * L1.  Don't stuff vmcs01.GUEST_CR3 when using nested early checks as
3036          * KVM modifies vcpu->arch.cr3 if and only if the early hardware checks
3037          * pass, and early VM-Fails do not reset KVM's MMU, i.e. the VM-Fail
3038          * path would need to manually save/restore vmcs01.GUEST_CR3.
3039          */
3040         if (!enable_ept && !nested_early_check)
3041                 vmcs_writel(GUEST_CR3, vcpu->arch.cr3);
3042 
3043         vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02);
3044 
3045         prepare_vmcs02_early(vmx, vmcs12);
3046 
3047         if (from_vmentry) {
3048                 nested_get_vmcs12_pages(vcpu);
3049 
3050                 if (nested_vmx_check_vmentry_hw(vcpu)) {
3051                         vmx_switch_vmcs(vcpu, &vmx->vmcs01);
3052                         return -1;
3053                 }
3054 
3055                 if (nested_vmx_check_guest_state(vcpu, vmcs12, &exit_qual))
3056                         goto vmentry_fail_vmexit;
3057         }
3058 
3059         enter_guest_mode(vcpu);
3060         if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING)
3061                 vcpu->arch.tsc_offset += vmcs12->tsc_offset;
3062 
3063         if (prepare_vmcs02(vcpu, vmcs12, &exit_qual))
3064                 goto vmentry_fail_vmexit_guest_mode;
3065 
3066         if (from_vmentry) {
3067                 exit_reason = EXIT_REASON_MSR_LOAD_FAIL;
3068                 exit_qual = nested_vmx_load_msr(vcpu,
3069                                                 vmcs12->vm_entry_msr_load_addr,
3070                                                 vmcs12->vm_entry_msr_load_count);
3071                 if (exit_qual)
3072                         goto vmentry_fail_vmexit_guest_mode;
3073         } else {
3074                 /*
3075                  * The MMU is not initialized to point at the right entities yet and
3076                  * "get pages" would need to read data from the guest (i.e. we will
3077                  * need to perform gpa to hpa translation). Request a call
3078                  * to nested_get_vmcs12_pages before the next VM-entry.  The MSRs
3079                  * have already been set at vmentry time and should not be reset.
3080                  */
3081                 kvm_make_request(KVM_REQ_GET_VMCS12_PAGES, vcpu);
3082         }
3083 
3084         /*
3085          * If L1 had a pending IRQ/NMI until it executed
3086          * VMLAUNCH/VMRESUME which wasn't delivered because it was
3087          * disallowed (e.g. interrupts disabled), L0 needs to
3088          * evaluate if this pending event should cause an exit from L2
3089          * to L1 or delivered directly to L2 (e.g. In case L1 don't
3090          * intercept EXTERNAL_INTERRUPT).
3091          *
3092          * Usually this would be handled by the processor noticing an
3093          * IRQ/NMI window request, or checking RVI during evaluation of
3094          * pending virtual interrupts.  However, this setting was done
3095          * on VMCS01 and now VMCS02 is active instead. Thus, we force L0
3096          * to perform pending event evaluation by requesting a KVM_REQ_EVENT.
3097          */
3098         if (unlikely(evaluate_pending_interrupts))
3099                 kvm_make_request(KVM_REQ_EVENT, vcpu);
3100 
3101         /*
3102          * Do not start the preemption timer hrtimer until after we know
3103          * we are successful, so that only nested_vmx_vmexit needs to cancel
3104          * the timer.
3105          */
3106         vmx->nested.preemption_timer_expired = false;
3107         if (nested_cpu_has_preemption_timer(vmcs12))
3108                 vmx_start_preemption_timer(vcpu);
3109 
3110         /*
3111          * Note no nested_vmx_succeed or nested_vmx_fail here. At this point
3112          * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet
3113          * returned as far as L1 is concerned. It will only return (and set
3114          * the success flag) when L2 exits (see nested_vmx_vmexit()).
3115          */
3116         return 0;
3117 
3118         /*
3119          * A failed consistency check that leads to a VMExit during L1's
3120          * VMEnter to L2 is a variation of a normal VMexit, as explained in
3121          * 26.7 "VM-entry failures during or after loading guest state".
3122          */
3123 vmentry_fail_vmexit_guest_mode:
3124         if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING)
3125                 vcpu->arch.tsc_offset -= vmcs12->tsc_offset;
3126         leave_guest_mode(vcpu);
3127 
3128 vmentry_fail_vmexit:
3129         vmx_switch_vmcs(vcpu, &vmx->vmcs01);
3130 
3131         if (!from_vmentry)
3132                 return 1;
3133 
3134         load_vmcs12_host_state(vcpu, vmcs12);
3135         vmcs12->vm_exit_reason = exit_reason | VMX_EXIT_REASONS_FAILED_VMENTRY;
3136         vmcs12->exit_qualification = exit_qual;
3137         if (enable_shadow_vmcs || vmx->nested.hv_evmcs)
3138                 vmx->nested.need_vmcs12_to_shadow_sync = true;
3139         return 1;
3140 }
3141 
3142 /*
3143  * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1
3144  * for running an L2 nested guest.
3145  */
3146 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
3147 {
3148         struct vmcs12 *vmcs12;
3149         struct vcpu_vmx *vmx = to_vmx(vcpu);
3150         u32 interrupt_shadow = vmx_get_interrupt_shadow(vcpu);
3151         int ret;
3152 
3153         if (!nested_vmx_check_permission(vcpu))
3154                 return 1;
3155 
3156         if (!nested_vmx_handle_enlightened_vmptrld(vcpu, launch))
3157                 return 1;
3158 
3159         if (!vmx->nested.hv_evmcs && vmx->nested.current_vmptr == -1ull)
3160                 return nested_vmx_failInvalid(vcpu);
3161 
3162         vmcs12 = get_vmcs12(vcpu);
3163 
3164         /*
3165          * Can't VMLAUNCH or VMRESUME a shadow VMCS. Despite the fact
3166          * that there *is* a valid VMCS pointer, RFLAGS.CF is set
3167          * rather than RFLAGS.ZF, and no error number is stored to the
3168          * VM-instruction error field.
3169          */
3170         if (vmcs12->hdr.shadow_vmcs)
3171                 return nested_vmx_failInvalid(vcpu);
3172 
3173         if (vmx->nested.hv_evmcs) {
3174                 copy_enlightened_to_vmcs12(vmx);
3175                 /* Enlightened VMCS doesn't have launch state */
3176                 vmcs12->launch_state = !launch;
3177         } else if (enable_shadow_vmcs) {
3178                 copy_shadow_to_vmcs12(vmx);
3179         }
3180 
3181         /*
3182          * The nested entry process starts with enforcing various prerequisites
3183          * on vmcs12 as required by the Intel SDM, and act appropriately when
3184          * they fail: As the SDM explains, some conditions should cause the
3185          * instruction to fail, while others will cause the instruction to seem
3186          * to succeed, but return an EXIT_REASON_INVALID_STATE.
3187          * To speed up the normal (success) code path, we should avoid checking
3188          * for misconfigurations which will anyway be caught by the processor
3189          * when using the merged vmcs02.
3190          */
3191         if (interrupt_shadow & KVM_X86_SHADOW_INT_MOV_SS)
3192                 return nested_vmx_failValid(vcpu,
3193                         VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS);
3194 
3195         if (vmcs12->launch_state == launch)
3196                 return nested_vmx_failValid(vcpu,
3197                         launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS
3198                                : VMXERR_VMRESUME_NONLAUNCHED_VMCS);
3199 
3200         if (nested_vmx_check_controls(vcpu, vmcs12))
3201                 return nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
3202 
3203         if (nested_vmx_check_host_state(vcpu, vmcs12))
3204                 return nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD);
3205 
3206         /*
3207          * We're finally done with prerequisite checking, and can start with
3208          * the nested entry.
3209          */
3210         vmx->nested.nested_run_pending = 1;
3211         ret = nested_vmx_enter_non_root_mode(vcpu, true);
3212         vmx->nested.nested_run_pending = !ret;
3213         if (ret > 0)
3214                 return 1;
3215         else if (ret)
3216                 return nested_vmx_failValid(vcpu,
3217                         VMXERR_ENTRY_INVALID_CONTROL_FIELD);
3218 
3219         /* Hide L1D cache contents from the nested guest.  */
3220         vmx->vcpu.arch.l1tf_flush_l1d = true;
3221 
3222         /*
3223          * Must happen outside of nested_vmx_enter_non_root_mode() as it will
3224          * also be used as part of restoring nVMX state for
3225          * snapshot restore (migration).
3226          *
3227          * In this flow, it is assumed that vmcs12 cache was
3228          * trasferred as part of captured nVMX state and should
3229          * therefore not be read from guest memory (which may not
3230          * exist on destination host yet).
3231          */
3232         nested_cache_shadow_vmcs12(vcpu, vmcs12);
3233 
3234         /*
3235          * If we're entering a halted L2 vcpu and the L2 vcpu won't be
3236          * awakened by event injection or by an NMI-window VM-exit or
3237          * by an interrupt-window VM-exit, halt the vcpu.
3238          */
3239         if ((vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT) &&
3240             !(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) &&
3241             !(vmcs12->cpu_based_vm_exec_control & CPU_BASED_VIRTUAL_NMI_PENDING) &&
3242             !((vmcs12->cpu_based_vm_exec_control & CPU_BASED_VIRTUAL_INTR_PENDING) &&
3243               (vmcs12->guest_rflags & X86_EFLAGS_IF))) {
3244                 vmx->nested.nested_run_pending = 0;
3245                 return kvm_vcpu_halt(vcpu);
3246         }
3247         return 1;
3248 }
3249 
3250 /*
3251  * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date
3252  * because L2 may have changed some cr0 bits directly (CRO_GUEST_HOST_MASK).
3253  * This function returns the new value we should put in vmcs12.guest_cr0.
3254  * It's not enough to just return the vmcs02 GUEST_CR0. Rather,
3255  *  1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now
3256  *     available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0
3257  *     didn't trap the bit, because if L1 did, so would L0).
3258  *  2. Bits that L1 asked to trap (and therefore L0 also did) could not have
3259  *     been modified by L2, and L1 knows it. So just leave the old value of
3260  *     the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0
3261  *     isn't relevant, because if L0 traps this bit it can set it to anything.
3262  *  3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have
3263  *     changed these bits, and therefore they need to be updated, but L0
3264  *     didn't necessarily allow them to be changed in GUEST_CR0 - and rather
3265  *     put them in vmcs02 CR0_READ_SHADOW. So take these bits from there.
3266  */
3267 static inline unsigned long
3268 vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
3269 {
3270         return
3271         /*1*/   (vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) |
3272         /*2*/   (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) |
3273         /*3*/   (vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask |
3274                         vcpu->arch.cr0_guest_owned_bits));
3275 }
3276 
3277 static inline unsigned long
3278 vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
3279 {
3280         return
3281         /*1*/   (vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) |
3282         /*2*/   (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) |
3283         /*3*/   (vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask |
3284                         vcpu->arch.cr4_guest_owned_bits));
3285 }
3286 
3287 static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu,
3288                                       struct vmcs12 *vmcs12)
3289 {
3290         u32 idt_vectoring;
3291         unsigned int nr;
3292 
3293         if (vcpu->arch.exception.injected) {
3294                 nr = vcpu->arch.exception.nr;
3295                 idt_vectoring = nr | VECTORING_INFO_VALID_MASK;
3296 
3297                 if (kvm_exception_is_soft(nr)) {
3298                         vmcs12->vm_exit_instruction_len =
3299                                 vcpu->arch.event_exit_inst_len;
3300                         idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION;
3301                 } else
3302                         idt_vectoring |= INTR_TYPE_HARD_EXCEPTION;
3303 
3304                 if (vcpu->arch.exception.has_error_code) {
3305                         idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK;
3306                         vmcs12->idt_vectoring_error_code =
3307                                 vcpu->arch.exception.error_code;
3308                 }
3309 
3310                 vmcs12->idt_vectoring_info_field = idt_vectoring;
3311         } else if (vcpu->arch.nmi_injected) {
3312                 vmcs12->idt_vectoring_info_field =
3313                         INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR;
3314         } else if (vcpu->arch.interrupt.injected) {
3315                 nr = vcpu->arch.interrupt.nr;
3316                 idt_vectoring = nr | VECTORING_INFO_VALID_MASK;
3317 
3318                 if (vcpu->arch.interrupt.soft) {
3319                         idt_vectoring |= INTR_TYPE_SOFT_INTR;
3320                         vmcs12->vm_entry_instruction_len =
3321                                 vcpu->arch.event_exit_inst_len;
3322                 } else
3323                         idt_vectoring |= INTR_TYPE_EXT_INTR;
3324 
3325                 vmcs12->idt_vectoring_info_field = idt_vectoring;
3326         }
3327 }
3328 
3329 
3330 static void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu)
3331 {
3332         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3333         gfn_t gfn;
3334 
3335         /*
3336          * Don't need to mark the APIC access page dirty; it is never
3337          * written to by the CPU during APIC virtualization.
3338          */
3339 
3340         if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
3341                 gfn = vmcs12->virtual_apic_page_addr >> PAGE_SHIFT;
3342                 kvm_vcpu_mark_page_dirty(vcpu, gfn);
3343         }
3344 
3345         if (nested_cpu_has_posted_intr(vmcs12)) {
3346                 gfn = vmcs12->posted_intr_desc_addr >> PAGE_SHIFT;
3347                 kvm_vcpu_mark_page_dirty(vcpu, gfn);
3348         }
3349 }
3350 
3351 static void vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu)
3352 {
3353         struct vcpu_vmx *vmx = to_vmx(vcpu);
3354         int max_irr;
3355         void *vapic_page;
3356         u16 status;
3357 
3358         if (!vmx->nested.pi_desc || !vmx->nested.pi_pending)
3359                 return;
3360 
3361         vmx->nested.pi_pending = false;
3362         if (!pi_test_and_clear_on(vmx->nested.pi_desc))
3363                 return;
3364 
3365         max_irr = find_last_bit((unsigned long *)vmx->nested.pi_desc->pir, 256);
3366         if (max_irr != 256) {
3367                 vapic_page = vmx->nested.virtual_apic_map.hva;
3368                 if (!vapic_page)
3369                         return;
3370 
3371                 __kvm_apic_update_irr(vmx->nested.pi_desc->pir,
3372                         vapic_page, &max_irr);
3373                 status = vmcs_read16(GUEST_INTR_STATUS);
3374                 if ((u8)max_irr > ((u8)status & 0xff)) {
3375                         status &= ~0xff;
3376                         status |= (u8)max_irr;
3377                         vmcs_write16(GUEST_INTR_STATUS, status);
3378                 }
3379         }
3380 
3381         nested_mark_vmcs12_pages_dirty(vcpu);
3382 }
3383 
3384 static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu,
3385                                                unsigned long exit_qual)
3386 {
3387         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3388         unsigned int nr = vcpu->arch.exception.nr;
3389         u32 intr_info = nr | INTR_INFO_VALID_MASK;
3390 
3391         if (vcpu->arch.exception.has_error_code) {
3392                 vmcs12->vm_exit_intr_error_code = vcpu->arch.exception.error_code;
3393                 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
3394         }
3395 
3396         if (kvm_exception_is_soft(nr))
3397                 intr_info |= INTR_TYPE_SOFT_EXCEPTION;
3398         else
3399                 intr_info |= INTR_TYPE_HARD_EXCEPTION;
3400 
3401         if (!(vmcs12->idt_vectoring_info_field & VECTORING_INFO_VALID_MASK) &&
3402             vmx_get_nmi_mask(vcpu))
3403                 intr_info |= INTR_INFO_UNBLOCK_NMI;
3404 
3405         nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, intr_info, exit_qual);
3406 }
3407 
3408 static int vmx_check_nested_events(struct kvm_vcpu *vcpu, bool external_intr)
3409 {
3410         struct vcpu_vmx *vmx = to_vmx(vcpu);
3411         unsigned long exit_qual;
3412         bool block_nested_events =
3413             vmx->nested.nested_run_pending || kvm_event_needs_reinjection(vcpu);
3414 
3415         if (vcpu->arch.exception.pending &&
3416                 nested_vmx_check_exception(vcpu, &exit_qual)) {
3417                 if (block_nested_events)
3418                         return -EBUSY;
3419                 nested_vmx_inject_exception_vmexit(vcpu, exit_qual);
3420                 return 0;
3421         }
3422 
3423         if (nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) &&
3424             vmx->nested.preemption_timer_expired) {
3425                 if (block_nested_events)
3426                         return -EBUSY;
3427                 nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0);
3428                 return 0;
3429         }
3430 
3431         if (vcpu->arch.nmi_pending && nested_exit_on_nmi(vcpu)) {
3432                 if (block_nested_events)
3433                         return -EBUSY;
3434                 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI,
3435                                   NMI_VECTOR | INTR_TYPE_NMI_INTR |
3436                                   INTR_INFO_VALID_MASK, 0);
3437                 /*
3438                  * The NMI-triggered VM exit counts as injection:
3439                  * clear this one and block further NMIs.
3440                  */
3441                 vcpu->arch.nmi_pending = 0;
3442                 vmx_set_nmi_mask(vcpu, true);
3443                 return 0;
3444         }
3445 
3446         if ((kvm_cpu_has_interrupt(vcpu) || external_intr) &&
3447             nested_exit_on_intr(vcpu)) {
3448                 if (block_nested_events)
3449                         return -EBUSY;
3450                 nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0);
3451                 return 0;
3452         }
3453 
3454         vmx_complete_nested_posted_interrupt(vcpu);
3455         return 0;
3456 }
3457 
3458 static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu)
3459 {
3460         ktime_t remaining =
3461                 hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer);
3462         u64 value;
3463 
3464         if (ktime_to_ns(remaining) <= 0)
3465                 return 0;
3466 
3467         value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz;
3468         do_div(value, 1000000);
3469         return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
3470 }
3471 
3472 static bool is_vmcs12_ext_field(unsigned long field)
3473 {
3474         switch (field) {
3475         case GUEST_ES_SELECTOR:
3476         case GUEST_CS_SELECTOR:
3477         case GUEST_SS_SELECTOR:
3478         case GUEST_DS_SELECTOR:
3479         case GUEST_FS_SELECTOR:
3480         case GUEST_GS_SELECTOR:
3481         case GUEST_LDTR_SELECTOR:
3482         case GUEST_TR_SELECTOR:
3483         case GUEST_ES_LIMIT:
3484         case GUEST_CS_LIMIT:
3485         case GUEST_SS_LIMIT:
3486         case GUEST_DS_LIMIT:
3487         case GUEST_FS_LIMIT:
3488         case GUEST_GS_LIMIT:
3489         case GUEST_LDTR_LIMIT:
3490         case GUEST_TR_LIMIT:
3491         case GUEST_GDTR_LIMIT:
3492         case GUEST_IDTR_LIMIT:
3493         case GUEST_ES_AR_BYTES:
3494         case GUEST_DS_AR_BYTES:
3495         case GUEST_FS_AR_BYTES:
3496         case GUEST_GS_AR_BYTES:
3497         case GUEST_LDTR_AR_BYTES:
3498         case GUEST_TR_AR_BYTES:
3499         case GUEST_ES_BASE:
3500         case GUEST_CS_BASE:
3501         case GUEST_SS_BASE:
3502         case GUEST_DS_BASE:
3503         case GUEST_FS_BASE:
3504         case GUEST_GS_BASE:
3505         case GUEST_LDTR_BASE:
3506         case GUEST_TR_BASE:
3507         case GUEST_GDTR_BASE:
3508         case GUEST_IDTR_BASE:
3509         case GUEST_PENDING_DBG_EXCEPTIONS:
3510         case GUEST_BNDCFGS:
3511                 return true;
3512         default:
3513                 break;
3514         }
3515 
3516         return false;
3517 }
3518 
3519 static void sync_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu,
3520                                        struct vmcs12 *vmcs12)
3521 {
3522         struct vcpu_vmx *vmx = to_vmx(vcpu);
3523 
3524         vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR);
3525         vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR);
3526         vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR);
3527         vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR);
3528         vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR);
3529         vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR);
3530         vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR);
3531         vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR);
3532         vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT);
3533         vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT);
3534         vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT);
3535         vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT);
3536         vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT);
3537         vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT);
3538         vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT);
3539         vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT);
3540         vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT);
3541         vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT);
3542         vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES);
3543         vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES);
3544         vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES);
3545         vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES);
3546         vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES);
3547         vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES);
3548         vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE);
3549         vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE);
3550         vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE);
3551         vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE);
3552         vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE);
3553         vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE);
3554         vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE);
3555         vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE);
3556         vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE);
3557         vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE);
3558         vmcs12->guest_pending_dbg_exceptions =
3559                 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS);
3560         if (kvm_mpx_supported())
3561                 vmcs12->guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS);
3562 
3563         vmx->nested.need_sync_vmcs02_to_vmcs12_rare = false;
3564 }
3565 
3566 static void copy_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu,
3567                                        struct vmcs12 *vmcs12)
3568 {
3569         struct vcpu_vmx *vmx = to_vmx(vcpu);
3570         int cpu;
3571 
3572         if (!vmx->nested.need_sync_vmcs02_to_vmcs12_rare)
3573                 return;
3574 
3575 
3576         WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01);
3577 
3578         cpu = get_cpu();
3579         vmx->loaded_vmcs = &vmx->nested.vmcs02;
3580         vmx_vcpu_load(&vmx->vcpu, cpu);
3581 
3582         sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
3583 
3584         vmx->loaded_vmcs = &vmx->vmcs01;
3585         vmx_vcpu_load(&vmx->vcpu, cpu);
3586         put_cpu();
3587 }
3588 
3589 /*
3590  * Update the guest state fields of vmcs12 to reflect changes that
3591  * occurred while L2 was running. (The "IA-32e mode guest" bit of the
3592  * VM-entry controls is also updated, since this is really a guest
3593  * state bit.)
3594  */
3595 static void sync_vmcs02_to_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
3596 {
3597         struct vcpu_vmx *vmx = to_vmx(vcpu);
3598 
3599         if (vmx->nested.hv_evmcs)
3600                 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
3601 
3602         vmx->nested.need_sync_vmcs02_to_vmcs12_rare = !vmx->nested.hv_evmcs;
3603 
3604         vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12);
3605         vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12);
3606 
3607         vmcs12->guest_rsp = kvm_rsp_read(vcpu);
3608         vmcs12->guest_rip = kvm_rip_read(vcpu);
3609         vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS);
3610 
3611         vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES);
3612         vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES);
3613 
3614         vmcs12->guest_sysenter_cs = vmcs_read32(GUEST_SYSENTER_CS);
3615         vmcs12->guest_sysenter_esp = vmcs_readl(GUEST_SYSENTER_ESP);
3616         vmcs12->guest_sysenter_eip = vmcs_readl(GUEST_SYSENTER_EIP);
3617 
3618         vmcs12->guest_interruptibility_info =
3619                 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
3620 
3621         if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)
3622                 vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT;
3623         else
3624                 vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE;
3625 
3626         if (nested_cpu_has_preemption_timer(vmcs12) &&
3627             vmcs12->vm_exit_controls & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)
3628                         vmcs12->vmx_preemption_timer_value =
3629                                 vmx_get_preemption_timer_value(vcpu);
3630 
3631         /*
3632          * In some cases (usually, nested EPT), L2 is allowed to change its
3633          * own CR3 without exiting. If it has changed it, we must keep it.
3634          * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined
3635          * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12.
3636          *
3637          * Additionally, restore L2's PDPTR to vmcs12.
3638          */
3639         if (enable_ept) {
3640                 vmcs12->guest_cr3 = vmcs_readl(GUEST_CR3);
3641                 if (nested_cpu_has_ept(vmcs12) && is_pae_paging(vcpu)) {
3642                         vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0);
3643                         vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1);
3644                         vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2);
3645                         vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3);
3646                 }
3647         }
3648 
3649         vmcs12->guest_linear_address = vmcs_readl(GUEST_LINEAR_ADDRESS);
3650 
3651         if (nested_cpu_has_vid(vmcs12))
3652                 vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS);
3653 
3654         vmcs12->vm_entry_controls =
3655                 (vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) |
3656                 (vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE);
3657 
3658         if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS)
3659                 kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7);
3660 
3661         if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER)
3662                 vmcs12->guest_ia32_efer = vcpu->arch.efer;
3663 }
3664 
3665 /*
3666  * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits
3667  * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12),
3668  * and this function updates it to reflect the changes to the guest state while
3669  * L2 was running (and perhaps made some exits which were handled directly by L0
3670  * without going back to L1), and to reflect the exit reason.
3671  * Note that we do not have to copy here all VMCS fields, just those that
3672  * could have changed by the L2 guest or the exit - i.e., the guest-state and
3673  * exit-information fields only. Other fields are modified by L1 with VMWRITE,
3674  * which already writes to vmcs12 directly.
3675  */
3676 static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
3677                            u32 exit_reason, u32 exit_intr_info,
3678                            unsigned long exit_qualification)
3679 {
3680         /* update exit information fields: */
3681         vmcs12->vm_exit_reason = exit_reason;
3682         vmcs12->exit_qualification = exit_qualification;
3683         vmcs12->vm_exit_intr_info = exit_intr_info;
3684 
3685         vmcs12->idt_vectoring_info_field = 0;
3686         vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
3687         vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
3688 
3689         if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) {
3690                 vmcs12->launch_state = 1;
3691 
3692                 /* vm_entry_intr_info_field is cleared on exit. Emulate this
3693                  * instead of reading the real value. */
3694                 vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK;
3695 
3696                 /*
3697                  * Transfer the event that L0 or L1 may wanted to inject into
3698                  * L2 to IDT_VECTORING_INFO_FIELD.
3699                  */
3700                 vmcs12_save_pending_event(vcpu, vmcs12);
3701 
3702                 /*
3703                  * According to spec, there's no need to store the guest's
3704                  * MSRs if the exit is due to a VM-entry failure that occurs
3705                  * during or after loading the guest state. Since this exit
3706                  * does not fall in that category, we need to save the MSRs.
3707                  */
3708                 if (nested_vmx_store_msr(vcpu,
3709                                          vmcs12->vm_exit_msr_store_addr,
3710                                          vmcs12->vm_exit_msr_store_count))
3711                         nested_vmx_abort(vcpu,
3712                                          VMX_ABORT_SAVE_GUEST_MSR_FAIL);
3713         }
3714 
3715         /*
3716          * Drop what we picked up for L2 via vmx_complete_interrupts. It is
3717          * preserved above and would only end up incorrectly in L1.
3718          */
3719         vcpu->arch.nmi_injected = false;
3720         kvm_clear_exception_queue(vcpu);
3721         kvm_clear_interrupt_queue(vcpu);
3722 }
3723 
3724 /*
3725  * A part of what we need to when the nested L2 guest exits and we want to
3726  * run its L1 parent, is to reset L1's guest state to the host state specified
3727  * in vmcs12.
3728  * This function is to be called not only on normal nested exit, but also on
3729  * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry
3730  * Failures During or After Loading Guest State").
3731  * This function should be called when the active VMCS is L1's (vmcs01).
3732  */
3733 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
3734                                    struct vmcs12 *vmcs12)
3735 {
3736         struct kvm_segment seg;
3737         u32 entry_failure_code;
3738 
3739         if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER)
3740                 vcpu->arch.efer = vmcs12->host_ia32_efer;
3741         else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
3742                 vcpu->arch.efer |= (EFER_LMA | EFER_LME);
3743         else
3744                 vcpu->arch.efer &= ~(EFER_LMA | EFER_LME);
3745         vmx_set_efer(vcpu, vcpu->arch.efer);
3746 
3747         kvm_rsp_write(vcpu, vmcs12->host_rsp);
3748         kvm_rip_write(vcpu, vmcs12->host_rip);
3749         vmx_set_rflags(vcpu, X86_EFLAGS_FIXED);
3750         vmx_set_interrupt_shadow(vcpu, 0);
3751 
3752         /*
3753          * Note that calling vmx_set_cr0 is important, even if cr0 hasn't
3754          * actually changed, because vmx_set_cr0 refers to efer set above.
3755          *
3756          * CR0_GUEST_HOST_MASK is already set in the original vmcs01
3757          * (KVM doesn't change it);
3758          */
3759         vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS;
3760         vmx_set_cr0(vcpu, vmcs12->host_cr0);
3761 
3762         /* Same as above - no reason to call set_cr4_guest_host_mask().  */
3763         vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK);
3764         vmx_set_cr4(vcpu, vmcs12->host_cr4);
3765 
3766         nested_ept_uninit_mmu_context(vcpu);
3767 
3768         /*
3769          * Only PDPTE load can fail as the value of cr3 was checked on entry and
3770          * couldn't have changed.
3771          */
3772         if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, &entry_failure_code))
3773                 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL);
3774 
3775         if (!enable_ept)
3776                 vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault;
3777 
3778         /*
3779          * If vmcs01 doesn't use VPID, CPU flushes TLB on every
3780          * VMEntry/VMExit. Thus, no need to flush TLB.
3781          *
3782          * If vmcs12 doesn't use VPID, L1 expects TLB to be
3783          * flushed on every VMEntry/VMExit.
3784          *
3785          * Otherwise, we can preserve TLB entries as long as we are
3786          * able to tag L1 TLB entries differently than L2 TLB entries.
3787          *
3788          * If vmcs12 uses EPT, we need to execute this flush on EPTP01
3789          * and therefore we request the TLB flush to happen only after VMCS EPTP
3790          * has been set by KVM_REQ_LOAD_CR3.
3791          */
3792         if (enable_vpid &&
3793             (!nested_cpu_has_vpid(vmcs12) || !nested_has_guest_tlb_tag(vcpu))) {
3794                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3795         }
3796 
3797         vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs);
3798         vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp);
3799         vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip);
3800         vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base);
3801         vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base);
3802         vmcs_write32(GUEST_IDTR_LIMIT, 0xFFFF);
3803         vmcs_write32(GUEST_GDTR_LIMIT, 0xFFFF);
3804 
3805         /* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1.  */
3806         if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS)
3807                 vmcs_write64(GUEST_BNDCFGS, 0);
3808 
3809         if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) {
3810                 vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat);
3811                 vcpu->arch.pat = vmcs12->host_ia32_pat;
3812         }
3813         if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
3814                 vmcs_write64(GUEST_IA32_PERF_GLOBAL_CTRL,
3815                         vmcs12->host_ia32_perf_global_ctrl);
3816 
3817         /* Set L1 segment info according to Intel SDM
3818             27.5.2 Loading Host Segment and Descriptor-Table Registers */
3819         seg = (struct kvm_segment) {
3820                 .base = 0,
3821                 .limit = 0xFFFFFFFF,
3822                 .selector = vmcs12->host_cs_selector,
3823                 .type = 11,
3824                 .present = 1,
3825                 .s = 1,
3826                 .g = 1
3827         };
3828         if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
3829                 seg.l = 1;
3830         else
3831                 seg.db = 1;
3832         vmx_set_segment(vcpu, &seg, VCPU_SREG_CS);
3833         seg = (struct kvm_segment) {
3834                 .base = 0,
3835                 .limit = 0xFFFFFFFF,
3836                 .type = 3,
3837                 .present = 1,
3838                 .s = 1,
3839                 .db = 1,
3840                 .g = 1
3841         };
3842         seg.selector = vmcs12->host_ds_selector;
3843         vmx_set_segment(vcpu, &seg, VCPU_SREG_DS);
3844         seg.selector = vmcs12->host_es_selector;
3845         vmx_set_segment(vcpu, &seg, VCPU_SREG_ES);
3846         seg.selector = vmcs12->host_ss_selector;
3847         vmx_set_segment(vcpu, &seg, VCPU_SREG_SS);
3848         seg.selector = vmcs12->host_fs_selector;
3849         seg.base = vmcs12->host_fs_base;
3850         vmx_set_segment(vcpu, &seg, VCPU_SREG_FS);
3851         seg.selector = vmcs12->host_gs_selector;
3852         seg.base = vmcs12->host_gs_base;
3853         vmx_set_segment(vcpu, &seg, VCPU_SREG_GS);
3854         seg = (struct kvm_segment) {
3855                 .base = vmcs12->host_tr_base,
3856                 .limit = 0x67,
3857                 .selector = vmcs12->host_tr_selector,
3858                 .type = 11,
3859                 .present = 1
3860         };
3861         vmx_set_segment(vcpu, &seg, VCPU_SREG_TR);
3862 
3863         kvm_set_dr(vcpu, 7, 0x400);
3864         vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
3865 
3866         if (cpu_has_vmx_msr_bitmap())
3867                 vmx_update_msr_bitmap(vcpu);
3868 
3869         if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr,
3870                                 vmcs12->vm_exit_msr_load_count))
3871                 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL);
3872 }
3873 
3874 static inline u64 nested_vmx_get_vmcs01_guest_efer(struct vcpu_vmx *vmx)
3875 {
3876         struct shared_msr_entry *efer_msr;
3877         unsigned int i;
3878 
3879         if (vm_entry_controls_get(vmx) & VM_ENTRY_LOAD_IA32_EFER)
3880                 return vmcs_read64(GUEST_IA32_EFER);
3881 
3882         if (cpu_has_load_ia32_efer())
3883                 return host_efer;
3884 
3885         for (i = 0; i < vmx->msr_autoload.guest.nr; ++i) {
3886                 if (vmx->msr_autoload.guest.val[i].index == MSR_EFER)
3887                         return vmx->msr_autoload.guest.val[i].value;
3888         }
3889 
3890         efer_msr = find_msr_entry(vmx, MSR_EFER);
3891         if (efer_msr)
3892                 return efer_msr->data;
3893 
3894         return host_efer;
3895 }
3896 
3897 static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu)
3898 {
3899         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3900         struct vcpu_vmx *vmx = to_vmx(vcpu);
3901         struct vmx_msr_entry g, h;
3902         struct msr_data msr;
3903         gpa_t gpa;
3904         u32 i, j;
3905 
3906         vcpu->arch.pat = vmcs_read64(GUEST_IA32_PAT);
3907 
3908         if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) {
3909                 /*
3910                  * L1's host DR7 is lost if KVM_GUESTDBG_USE_HW_BP is set
3911                  * as vmcs01.GUEST_DR7 contains a userspace defined value
3912                  * and vcpu->arch.dr7 is not squirreled away before the
3913                  * nested VMENTER (not worth adding a variable in nested_vmx).
3914                  */
3915                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
3916                         kvm_set_dr(vcpu, 7, DR7_FIXED_1);
3917                 else
3918                         WARN_ON(kvm_set_dr(vcpu, 7, vmcs_readl(GUEST_DR7)));
3919         }
3920 
3921         /*
3922          * Note that calling vmx_set_{efer,cr0,cr4} is important as they
3923          * handle a variety of side effects to KVM's software model.
3924          */
3925         vmx_set_efer(vcpu, nested_vmx_get_vmcs01_guest_efer(vmx));
3926 
3927         vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS;
3928         vmx_set_cr0(vcpu, vmcs_readl(CR0_READ_SHADOW));
3929 
3930         vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK);
3931         vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW));
3932 
3933         nested_ept_uninit_mmu_context(vcpu);
3934         vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
3935         __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
3936 
3937         /*
3938          * Use ept_save_pdptrs(vcpu) to load the MMU's cached PDPTRs
3939          * from vmcs01 (if necessary).  The PDPTRs are not loaded on
3940          * VMFail, like everything else we just need to ensure our
3941          * software model is up-to-date.
3942          */
3943         if (enable_ept)
3944                 ept_save_pdptrs(vcpu);
3945 
3946         kvm_mmu_reset_context(vcpu);
3947 
3948         if (cpu_has_vmx_msr_bitmap())
3949                 vmx_update_msr_bitmap(vcpu);
3950 
3951         /*
3952          * This nasty bit of open coding is a compromise between blindly
3953          * loading L1's MSRs using the exit load lists (incorrect emulation
3954          * of VMFail), leaving the nested VM's MSRs in the software model
3955          * (incorrect behavior) and snapshotting the modified MSRs (too
3956          * expensive since the lists are unbound by hardware).  For each
3957          * MSR that was (prematurely) loaded from the nested VMEntry load
3958          * list, reload it from the exit load list if it exists and differs
3959          * from the guest value.  The intent is to stuff host state as
3960          * silently as possible, not to fully process the exit load list.
3961          */
3962         msr.host_initiated = false;
3963         for (i = 0; i < vmcs12->vm_entry_msr_load_count; i++) {
3964                 gpa = vmcs12->vm_entry_msr_load_addr + (i * sizeof(g));
3965                 if (kvm_vcpu_read_guest(vcpu, gpa, &g, sizeof(g))) {
3966                         pr_debug_ratelimited(
3967                                 "%s read MSR index failed (%u, 0x%08llx)\n",
3968                                 __func__, i, gpa);
3969                         goto vmabort;
3970                 }
3971 
3972                 for (j = 0; j < vmcs12->vm_exit_msr_load_count; j++) {
3973                         gpa = vmcs12->vm_exit_msr_load_addr + (j * sizeof(h));
3974                         if (kvm_vcpu_read_guest(vcpu, gpa, &h, sizeof(h))) {
3975                                 pr_debug_ratelimited(
3976                                         "%s read MSR failed (%u, 0x%08llx)\n",
3977                                         __func__, j, gpa);
3978                                 goto vmabort;
3979                         }
3980                         if (h.index != g.index)
3981                                 continue;
3982                         if (h.value == g.value)
3983                                 break;
3984 
3985                         if (nested_vmx_load_msr_check(vcpu, &h)) {
3986                                 pr_debug_ratelimited(
3987                                         "%s check failed (%u, 0x%x, 0x%x)\n",
3988                                         __func__, j, h.index, h.reserved);
3989                                 goto vmabort;
3990                         }
3991 
3992                         msr.index = h.index;
3993                         msr.data = h.value;
3994                         if (kvm_set_msr(vcpu, &msr)) {
3995                                 pr_debug_ratelimited(
3996                                         "%s WRMSR failed (%u, 0x%x, 0x%llx)\n",
3997                                         __func__, j, h.index, h.value);
3998                                 goto vmabort;
3999                         }
4000                 }
4001         }
4002 
4003         return;
4004 
4005 vmabort:
4006         nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL);
4007 }
4008 
4009 /*
4010  * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1
4011  * and modify vmcs12 to make it see what it would expect to see there if
4012  * L2 was its real guest. Must only be called when in L2 (is_guest_mode())
4013  */
4014 void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
4015                        u32 exit_intr_info, unsigned long exit_qualification)
4016 {
4017         struct vcpu_vmx *vmx = to_vmx(vcpu);
4018         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
4019 
4020         /* trying to cancel vmlaunch/vmresume is a bug */
4021         WARN_ON_ONCE(vmx->nested.nested_run_pending);
4022 
4023         leave_guest_mode(vcpu);
4024 
4025         if (nested_cpu_has_preemption_timer(vmcs12))
4026                 hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer);
4027 
4028         if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING)
4029                 vcpu->arch.tsc_offset -= vmcs12->tsc_offset;
4030 
4031         if (likely(!vmx->fail)) {
4032                 sync_vmcs02_to_vmcs12(vcpu, vmcs12);
4033 
4034                 if (exit_reason != -1)
4035                         prepare_vmcs12(vcpu, vmcs12, exit_reason, exit_intr_info,
4036                                        exit_qualification);
4037 
4038                 /*
4039                  * Must happen outside of sync_vmcs02_to_vmcs12() as it will
4040                  * also be used to capture vmcs12 cache as part of
4041                  * capturing nVMX state for snapshot (migration).
4042                  *
4043                  * Otherwise, this flush will dirty guest memory at a
4044                  * point it is already assumed by user-space to be
4045                  * immutable.
4046                  */
4047                 nested_flush_cached_shadow_vmcs12(vcpu, vmcs12);
4048         } else {
4049                 /*
4050                  * The only expected VM-instruction error is "VM entry with
4051                  * invalid control field(s)." Anything else indicates a
4052                  * problem with L0.  And we should never get here with a
4053                  * VMFail of any type if early consistency checks are enabled.
4054                  */
4055                 WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) !=
4056                              VMXERR_ENTRY_INVALID_CONTROL_FIELD);
4057                 WARN_ON_ONCE(nested_early_check);
4058         }
4059 
4060         vmx_switch_vmcs(vcpu, &vmx->vmcs01);
4061 
4062         /* Update any VMCS fields that might have changed while L2 ran */
4063         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
4064         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
4065         vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
4066 
4067         if (kvm_has_tsc_control)
4068                 decache_tsc_multiplier(vmx);
4069 
4070         if (vmx->nested.change_vmcs01_virtual_apic_mode) {
4071                 vmx->nested.change_vmcs01_virtual_apic_mode = false;
4072                 vmx_set_virtual_apic_mode(vcpu);
4073         } else if (!nested_cpu_has_ept(vmcs12) &&
4074                    nested_cpu_has2(vmcs12,
4075                                    SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
4076                 vmx_flush_tlb(vcpu, true);
4077         }
4078 
4079         /* Unpin physical memory we referred to in vmcs02 */
4080         if (vmx->nested.apic_access_page) {
4081                 kvm_release_page_dirty(vmx->nested.apic_access_page);
4082                 vmx->nested.apic_access_page = NULL;
4083         }
4084         kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true);
4085         kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true);
4086         vmx->nested.pi_desc = NULL;
4087 
4088         /*
4089          * We are now running in L2, mmu_notifier will force to reload the
4090          * page's hpa for L2 vmcs. Need to reload it for L1 before entering L1.
4091          */
4092         kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
4093 
4094         if ((exit_reason != -1) && (enable_shadow_vmcs || vmx->nested.hv_evmcs))
4095                 vmx->nested.need_vmcs12_to_shadow_sync = true;
4096 
4097         /* in case we halted in L2 */
4098         vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
4099 
4100         if (likely(!vmx->fail)) {
4101                 /*
4102                  * TODO: SDM says that with acknowledge interrupt on
4103                  * exit, bit 31 of the VM-exit interrupt information
4104                  * (valid interrupt) is always set to 1 on
4105                  * EXIT_REASON_EXTERNAL_INTERRUPT, so we shouldn't
4106                  * need kvm_cpu_has_interrupt().  See the commit
4107                  * message for details.
4108                  */
4109                 if (nested_exit_intr_ack_set(vcpu) &&
4110                     exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT &&
4111                     kvm_cpu_has_interrupt(vcpu)) {
4112                         int irq = kvm_cpu_get_interrupt(vcpu);
4113                         WARN_ON(irq < 0);
4114                         vmcs12->vm_exit_intr_info = irq |
4115                                 INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR;
4116                 }
4117 
4118                 if (exit_reason != -1)
4119                         trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason,
4120                                                        vmcs12->exit_qualification,
4121                                                        vmcs12->idt_vectoring_info_field,
4122                                                        vmcs12->vm_exit_intr_info,
4123                                                        vmcs12->vm_exit_intr_error_code,
4124                                                        KVM_ISA_VMX);
4125 
4126                 load_vmcs12_host_state(vcpu, vmcs12);
4127 
4128                 return;
4129         }
4130 
4131         /*
4132          * After an early L2 VM-entry failure, we're now back
4133          * in L1 which thinks it just finished a VMLAUNCH or
4134          * VMRESUME instruction, so we need to set the failure
4135          * flag and the VM-instruction error field of the VMCS
4136          * accordingly, and skip the emulated instruction.
4137          */
4138         (void)nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
4139 
4140         /*
4141          * Restore L1's host state to KVM's software model.  We're here
4142          * because a consistency check was caught by hardware, which
4143          * means some amount of guest state has been propagated to KVM's
4144          * model and needs to be unwound to the host's state.
4145          */
4146         nested_vmx_restore_host_state(vcpu);
4147 
4148         vmx->fail = 0;
4149 }
4150 
4151 /*
4152  * Decode the memory-address operand of a vmx instruction, as recorded on an
4153  * exit caused by such an instruction (run by a guest hypervisor).
4154  * On success, returns 0. When the operand is invalid, returns 1 and throws
4155  * #UD or #GP.
4156  */
4157 int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification,
4158                         u32 vmx_instruction_info, bool wr, int len, gva_t *ret)
4159 {
4160         gva_t off;
4161         bool exn;
4162         struct kvm_segment s;
4163 
4164         /*
4165          * According to Vol. 3B, "Information for VM Exits Due to Instruction
4166          * Execution", on an exit, vmx_instruction_info holds most of the
4167          * addressing components of the operand. Only the displacement part
4168          * is put in exit_qualification (see 3B, "Basic VM-Exit Information").
4169          * For how an actual address is calculated from all these components,
4170          * refer to Vol. 1, "Operand Addressing".
4171          */
4172         int  scaling = vmx_instruction_info & 3;
4173         int  addr_size = (vmx_instruction_info >> 7) & 7;
4174         bool is_reg = vmx_instruction_info & (1u << 10);
4175         int  seg_reg = (vmx_instruction_info >> 15) & 7;
4176         int  index_reg = (vmx_instruction_info >> 18) & 0xf;
4177         bool index_is_valid = !(vmx_instruction_info & (1u << 22));
4178         int  base_reg       = (vmx_instruction_info >> 23) & 0xf;
4179         bool base_is_valid  = !(vmx_instruction_info & (1u << 27));
4180 
4181         if (is_reg) {
4182                 kvm_queue_exception(vcpu, UD_VECTOR);
4183                 return 1;
4184         }
4185 
4186         /* Addr = segment_base + offset */
4187         /* offset = base + [index * scale] + displacement */
4188         off = exit_qualification; /* holds the displacement */
4189         if (addr_size == 1)
4190                 off = (gva_t)sign_extend64(off, 31);
4191         else if (addr_size == 0)
4192                 off = (gva_t)sign_extend64(off, 15);
4193         if (base_is_valid)
4194                 off += kvm_register_read(vcpu, base_reg);
4195         if (index_is_valid)
4196                 off += kvm_register_read(vcpu, index_reg)<<scaling;
4197         vmx_get_segment(vcpu, &s, seg_reg);
4198 
4199         /*
4200          * The effective address, i.e. @off, of a memory operand is truncated
4201          * based on the address size of the instruction.  Note that this is
4202          * the *effective address*, i.e. the address prior to accounting for
4203          * the segment's base.
4204          */
4205         if (addr_size == 1) /* 32 bit */
4206                 off &= 0xffffffff;
4207         else if (addr_size == 0) /* 16 bit */
4208                 off &= 0xffff;
4209 
4210         /* Checks for #GP/#SS exceptions. */
4211         exn = false;
4212         if (is_long_mode(vcpu)) {
4213                 /*
4214                  * The virtual/linear address is never truncated in 64-bit
4215                  * mode, e.g. a 32-bit address size can yield a 64-bit virtual
4216                  * address when using FS/GS with a non-zero base.
4217                  */
4218                 if (seg_reg == VCPU_SREG_FS || seg_reg == VCPU_SREG_GS)
4219                         *ret = s.base + off;
4220                 else
4221                         *ret = off;
4222 
4223                 /* Long mode: #GP(0)/#SS(0) if the memory address is in a
4224                  * non-canonical form. This is the only check on the memory
4225                  * destination for long mode!
4226                  */
4227                 exn = is_noncanonical_address(*ret, vcpu);
4228         } else {
4229                 /*
4230                  * When not in long mode, the virtual/linear address is
4231                  * unconditionally truncated to 32 bits regardless of the
4232                  * address size.
4233                  */
4234                 *ret = (s.base + off) & 0xffffffff;
4235 
4236                 /* Protected mode: apply checks for segment validity in the
4237                  * following order:
4238                  * - segment type check (#GP(0) may be thrown)
4239                  * - usability check (#GP(0)/#SS(0))
4240                  * - limit check (#GP(0)/#SS(0))
4241                  */
4242                 if (wr)
4243                         /* #GP(0) if the destination operand is located in a
4244                          * read-only data segment or any code segment.
4245                          */
4246                         exn = ((s.type & 0xa) == 0 || (s.type & 8));
4247                 else
4248                         /* #GP(0) if the source operand is located in an
4249                          * execute-only code segment
4250                          */
4251                         exn = ((s.type & 0xa) == 8);
4252                 if (exn) {
4253                         kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
4254                         return 1;
4255                 }
4256                 /* Protected mode: #GP(0)/#SS(0) if the segment is unusable.
4257                  */
4258                 exn = (s.unusable != 0);
4259 
4260                 /*
4261                  * Protected mode: #GP(0)/#SS(0) if the memory operand is
4262                  * outside the segment limit.  All CPUs that support VMX ignore
4263                  * limit checks for flat segments, i.e. segments with base==0,
4264                  * limit==0xffffffff and of type expand-up data or code.
4265                  */
4266                 if (!(s.base == 0 && s.limit == 0xffffffff &&
4267                      ((s.type & 8) || !(s.type & 4))))
4268                         exn = exn || ((u64)off + len - 1 > s.limit);
4269         }
4270         if (exn) {
4271                 kvm_queue_exception_e(vcpu,
4272                                       seg_reg == VCPU_SREG_SS ?
4273                                                 SS_VECTOR : GP_VECTOR,
4274                                       0);
4275                 return 1;
4276         }
4277 
4278         return 0;
4279 }
4280 
4281 static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer)
4282 {
4283         gva_t gva;
4284         struct x86_exception e;
4285 
4286         if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
4287                                 vmcs_read32(VMX_INSTRUCTION_INFO), false,
4288                                 sizeof(*vmpointer), &gva))
4289                 return 1;
4290 
4291         if (kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e)) {
4292                 kvm_inject_page_fault(vcpu, &e);
4293                 return 1;
4294         }
4295 
4296         return 0;
4297 }
4298 
4299 /*
4300  * Allocate a shadow VMCS and associate it with the currently loaded
4301  * VMCS, unless such a shadow VMCS already exists. The newly allocated
4302  * VMCS is also VMCLEARed, so that it is ready for use.
4303  */
4304 static struct vmcs *alloc_shadow_vmcs(struct kvm_vcpu *vcpu)
4305 {
4306         struct vcpu_vmx *vmx = to_vmx(vcpu);
4307         struct loaded_vmcs *loaded_vmcs = vmx->loaded_vmcs;
4308 
4309         /*
4310          * We should allocate a shadow vmcs for vmcs01 only when L1
4311          * executes VMXON and free it when L1 executes VMXOFF.
4312          * As it is invalid to execute VMXON twice, we shouldn't reach
4313          * here when vmcs01 already have an allocated shadow vmcs.
4314          */
4315         WARN_ON(loaded_vmcs == &vmx->vmcs01 && loaded_vmcs->shadow_vmcs);
4316 
4317         if (!loaded_vmcs->shadow_vmcs) {
4318                 loaded_vmcs->shadow_vmcs = alloc_vmcs(true);
4319                 if (loaded_vmcs->shadow_vmcs)
4320                         vmcs_clear(loaded_vmcs->shadow_vmcs);
4321         }
4322         return loaded_vmcs->shadow_vmcs;
4323 }
4324 
4325 static int enter_vmx_operation(struct kvm_vcpu *vcpu)
4326 {
4327         struct vcpu_vmx *vmx = to_vmx(vcpu);
4328         int r;
4329 
4330         r = alloc_loaded_vmcs(&vmx->nested.vmcs02);
4331         if (r < 0)
4332                 goto out_vmcs02;
4333 
4334         vmx->nested.cached_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT);
4335         if (!vmx->nested.cached_vmcs12)
4336                 goto out_cached_vmcs12;
4337 
4338         vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT);
4339         if (!vmx->nested.cached_shadow_vmcs12)
4340                 goto out_cached_shadow_vmcs12;
4341 
4342         if (enable_shadow_vmcs && !alloc_shadow_vmcs(vcpu))
4343                 goto out_shadow_vmcs;
4344 
4345         hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC,
4346                      HRTIMER_MODE_REL_PINNED);
4347         vmx->nested.preemption_timer.function = vmx_preemption_timer_fn;
4348 
4349         vmx->nested.vpid02 = allocate_vpid();
4350 
4351         vmx->nested.vmcs02_initialized = false;
4352         vmx->nested.vmxon = true;
4353 
4354         if (pt_mode == PT_MODE_HOST_GUEST) {
4355                 vmx->pt_desc.guest.ctl = 0;
4356                 pt_update_intercept_for_msr(vmx);
4357         }
4358 
4359         return 0;
4360 
4361 out_shadow_vmcs:
4362         kfree(vmx->nested.cached_shadow_vmcs12);
4363 
4364 out_cached_shadow_vmcs12:
4365         kfree(vmx->nested.cached_vmcs12);
4366 
4367 out_cached_vmcs12:
4368         free_loaded_vmcs(&vmx->nested.vmcs02);
4369 
4370 out_vmcs02:
4371         return -ENOMEM;
4372 }
4373 
4374 /*
4375  * Emulate the VMXON instruction.
4376  * Currently, we just remember that VMX is active, and do not save or even
4377  * inspect the argument to VMXON (the so-called "VMXON pointer") because we
4378  * do not currently need to store anything in that guest-allocated memory
4379  * region. Consequently, VMCLEAR and VMPTRLD also do not verify that the their
4380  * argument is different from the VMXON pointer (which the spec says they do).
4381  */
4382 static int handle_vmon(struct kvm_vcpu *vcpu)
4383 {
4384         int ret;
4385         gpa_t vmptr;
4386         uint32_t revision;
4387         struct vcpu_vmx *vmx = to_vmx(vcpu);
4388         const u64 VMXON_NEEDED_FEATURES = FEATURE_CONTROL_LOCKED
4389                 | FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
4390 
4391         /*
4392          * The Intel VMX Instruction Reference lists a bunch of bits that are
4393          * prerequisite to running VMXON, most notably cr4.VMXE must be set to
4394          * 1 (see vmx_set_cr4() for when we allow the guest to set this).
4395          * Otherwise, we should fail with #UD.  But most faulting conditions
4396          * have already been checked by hardware, prior to the VM-exit for
4397          * VMXON.  We do test guest cr4.VMXE because processor CR4 always has
4398          * that bit set to 1 in non-root mode.
4399          */
4400         if (!kvm_read_cr4_bits(vcpu, X86_CR4_VMXE)) {
4401                 kvm_queue_exception(vcpu, UD_VECTOR);
4402                 return 1;
4403         }
4404 
4405         /* CPL=0 must be checked manually. */
4406         if (vmx_get_cpl(vcpu)) {
4407                 kvm_inject_gp(vcpu, 0);
4408                 return 1;
4409         }
4410 
4411         if (vmx->nested.vmxon)
4412                 return nested_vmx_failValid(vcpu,
4413                         VMXERR_VMXON_IN_VMX_ROOT_OPERATION);
4414 
4415         if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES)
4416                         != VMXON_NEEDED_FEATURES) {
4417                 kvm_inject_gp(vcpu, 0);
4418                 return 1;
4419         }
4420 
4421         if (nested_vmx_get_vmptr(vcpu, &vmptr))
4422                 return 1;
4423 
4424         /*
4425          * SDM 3: 24.11.5
4426          * The first 4 bytes of VMXON region contain the supported
4427          * VMCS revision identifier
4428          *
4429          * Note - IA32_VMX_BASIC[48] will never be 1 for the nested case;
4430          * which replaces physical address width with 32
4431          */
4432         if (!page_address_valid(vcpu, vmptr))
4433                 return nested_vmx_failInvalid(vcpu);
4434 
4435         if (kvm_read_guest(vcpu->kvm, vmptr, &revision, sizeof(revision)) ||
4436             revision != VMCS12_REVISION)
4437                 return nested_vmx_failInvalid(vcpu);
4438 
4439         vmx->nested.vmxon_ptr = vmptr;
4440         ret = enter_vmx_operation(vcpu);
4441         if (ret)
4442                 return ret;
4443 
4444         return nested_vmx_succeed(vcpu);
4445 }
4446 
4447 static inline void nested_release_vmcs12(struct kvm_vcpu *vcpu)
4448 {
4449         struct vcpu_vmx *vmx = to_vmx(vcpu);
4450 
4451         if (vmx->nested.current_vmptr == -1ull)
4452                 return;
4453 
4454         copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu));
4455 
4456         if (enable_shadow_vmcs) {
4457                 /* copy to memory all shadowed fields in case
4458                    they were modified */
4459                 copy_shadow_to_vmcs12(vmx);
4460                 vmx_disable_shadow_vmcs(vmx);
4461         }
4462         vmx->nested.posted_intr_nv = -1;
4463 
4464         /* Flush VMCS12 to guest memory */
4465         kvm_vcpu_write_guest_page(vcpu,
4466                                   vmx->nested.current_vmptr >> PAGE_SHIFT,
4467                                   vmx->nested.cached_vmcs12, 0, VMCS12_SIZE);
4468 
4469         kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
4470 
4471         vmx->nested.current_vmptr = -1ull;
4472 }
4473 
4474 /* Emulate the VMXOFF instruction */
4475 static int handle_vmoff(struct kvm_vcpu *vcpu)
4476 {
4477         if (!nested_vmx_check_permission(vcpu))
4478                 return 1;
4479         free_nested(vcpu);
4480         return nested_vmx_succeed(vcpu);
4481 }
4482 
4483 /* Emulate the VMCLEAR instruction */
4484 static int handle_vmclear(struct kvm_vcpu *vcpu)
4485 {
4486         struct vcpu_vmx *vmx = to_vmx(vcpu);
4487         u32 zero = 0;
4488         gpa_t vmptr;
4489         u64 evmcs_gpa;
4490 
4491         if (!nested_vmx_check_permission(vcpu))
4492                 return 1;
4493 
4494         if (nested_vmx_get_vmptr(vcpu, &vmptr))
4495                 return 1;
4496 
4497         if (!page_address_valid(vcpu, vmptr))
4498                 return nested_vmx_failValid(vcpu,
4499                         VMXERR_VMCLEAR_INVALID_ADDRESS);
4500 
4501         if (vmptr == vmx->nested.vmxon_ptr)
4502                 return nested_vmx_failValid(vcpu,
4503                         VMXERR_VMCLEAR_VMXON_POINTER);
4504 
4505         /*
4506          * When Enlightened VMEntry is enabled on the calling CPU we treat
4507          * memory area pointer by vmptr as Enlightened VMCS (as there's no good
4508          * way to distinguish it from VMCS12) and we must not corrupt it by
4509          * writing to the non-existent 'launch_state' field. The area doesn't
4510          * have to be the currently active EVMCS on the calling CPU and there's
4511          * nothing KVM has to do to transition it from 'active' to 'non-active'
4512          * state. It is possible that the area will stay mapped as
4513          * vmx->nested.hv_evmcs but this shouldn't be a problem.
4514          */
4515         if (likely(!vmx->nested.enlightened_vmcs_enabled ||
4516                    !nested_enlightened_vmentry(vcpu, &evmcs_gpa))) {
4517                 if (vmptr == vmx->nested.current_vmptr)
4518                         nested_release_vmcs12(vcpu);
4519 
4520                 kvm_vcpu_write_guest(vcpu,
4521                                      vmptr + offsetof(struct vmcs12,
4522                                                       launch_state),
4523                                      &zero, sizeof(zero));
4524         }
4525 
4526         return nested_vmx_succeed(vcpu);
4527 }
4528 
4529 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch);
4530 
4531 /* Emulate the VMLAUNCH instruction */
4532 static int handle_vmlaunch(struct kvm_vcpu *vcpu)
4533 {
4534         return nested_vmx_run(vcpu, true);
4535 }
4536 
4537 /* Emulate the VMRESUME instruction */
4538 static int handle_vmresume(struct kvm_vcpu *vcpu)
4539 {
4540 
4541         return nested_vmx_run(vcpu, false);
4542 }
4543 
4544 static int handle_vmread(struct kvm_vcpu *vcpu)
4545 {
4546         unsigned long field;
4547         u64 field_value;
4548         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
4549         u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
4550         int len;
4551         gva_t gva = 0;
4552         struct vmcs12 *vmcs12;
4553         struct x86_exception e;
4554         short offset;
4555 
4556         if (!nested_vmx_check_permission(vcpu))
4557                 return 1;
4558 
4559         if (to_vmx(vcpu)->nested.current_vmptr == -1ull)
4560                 return nested_vmx_failInvalid(vcpu);
4561 
4562         if (!is_guest_mode(vcpu))
4563                 vmcs12 = get_vmcs12(vcpu);
4564         else {
4565                 /*
4566                  * When vmcs->vmcs_link_pointer is -1ull, any VMREAD
4567                  * to shadowed-field sets the ALU flags for VMfailInvalid.
4568                  */
4569                 if (get_vmcs12(vcpu)->vmcs_link_pointer == -1ull)
4570                         return nested_vmx_failInvalid(vcpu);
4571                 vmcs12 = get_shadow_vmcs12(vcpu);
4572         }
4573 
4574         /* Decode instruction info and find the field to read */
4575         field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
4576 
4577         offset = vmcs_field_to_offset(field);
4578         if (offset < 0)
4579                 return nested_vmx_failValid(vcpu,
4580                         VMXERR_UNSUPPORTED_VMCS_COMPONENT);
4581 
4582         if (!is_guest_mode(vcpu) && is_vmcs12_ext_field(field))
4583                 copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
4584 
4585         /* Read the field, zero-extended to a u64 field_value */
4586         field_value = vmcs12_read_any(vmcs12, field, offset);
4587 
4588         /*
4589          * Now copy part of this value to register or memory, as requested.
4590          * Note that the number of bits actually copied is 32 or 64 depending
4591          * on the guest's mode (32 or 64 bit), not on the given field's length.
4592          */
4593         if (vmx_instruction_info & (1u << 10)) {
4594                 kvm_register_writel(vcpu, (((vmx_instruction_info) >> 3) & 0xf),
4595                         field_value);
4596         } else {
4597                 len = is_64_bit_mode(vcpu) ? 8 : 4;
4598                 if (get_vmx_mem_address(vcpu, exit_qualification,
4599                                 vmx_instruction_info, true, len, &gva))
4600                         return 1;
4601                 /* _system ok, nested_vmx_check_permission has verified cpl=0 */
4602                 if (kvm_write_guest_virt_system(vcpu, gva, &field_value, len, &e))
4603                         kvm_inject_page_fault(vcpu, &e);
4604         }
4605 
4606         return nested_vmx_succeed(vcpu);
4607 }
4608 
4609 static bool is_shadow_field_rw(unsigned long field)
4610 {
4611         switch (field) {
4612 #define SHADOW_FIELD_RW(x, y) case x:
4613 #include "vmcs_shadow_fields.h"
4614                 return true;
4615         default:
4616                 break;
4617         }
4618         return false;
4619 }
4620 
4621 static bool is_shadow_field_ro(unsigned long field)
4622 {
4623         switch (field) {
4624 #define SHADOW_FIELD_RO(x, y) case x:
4625 #include "vmcs_shadow_fields.h"
4626                 return true;
4627         default:
4628                 break;
4629         }
4630         return false;
4631 }
4632 
4633 static int handle_vmwrite(struct kvm_vcpu *vcpu)
4634 {
4635         unsigned long field;
4636         int len;
4637         gva_t gva;
4638         struct vcpu_vmx *vmx = to_vmx(vcpu);
4639         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
4640         u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
4641 
4642         /* The value to write might be 32 or 64 bits, depending on L1's long
4643          * mode, and eventually we need to write that into a field of several
4644          * possible lengths. The code below first zero-extends the value to 64
4645          * bit (field_value), and then copies only the appropriate number of
4646          * bits into the vmcs12 field.
4647          */
4648         u64 field_value = 0;
4649         struct x86_exception e;
4650         struct vmcs12 *vmcs12;
4651         short offset;
4652 
4653         if (!nested_vmx_check_permission(vcpu))
4654                 return 1;
4655 
4656         if (vmx->nested.current_vmptr == -1ull)
4657                 return nested_vmx_failInvalid(vcpu);
4658 
4659         if (vmx_instruction_info & (1u << 10))
4660                 field_value = kvm_register_readl(vcpu,
4661                         (((vmx_instruction_info) >> 3) & 0xf));
4662         else {
4663                 len = is_64_bit_mode(vcpu) ? 8 : 4;
4664                 if (get_vmx_mem_address(vcpu, exit_qualification,
4665                                 vmx_instruction_info, false, len, &gva))
4666                         return 1;
4667                 if (kvm_read_guest_virt(vcpu, gva, &field_value, len, &e)) {
4668                         kvm_inject_page_fault(vcpu, &e);
4669                         return 1;
4670                 }
4671         }
4672 
4673 
4674         field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
4675         /*
4676          * If the vCPU supports "VMWRITE to any supported field in the
4677          * VMCS," then the "read-only" fields are actually read/write.
4678          */
4679         if (vmcs_field_readonly(field) &&
4680             !nested_cpu_has_vmwrite_any_field(vcpu))
4681                 return nested_vmx_failValid(vcpu,
4682                         VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT);
4683 
4684         if (!is_guest_mode(vcpu)) {
4685                 vmcs12 = get_vmcs12(vcpu);
4686 
4687                 /*
4688                  * Ensure vmcs12 is up-to-date before any VMWRITE that dirties
4689                  * vmcs12, else we may crush a field or consume a stale value.
4690                  */
4691                 if (!is_shadow_field_rw(field))
4692                         copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
4693         } else {
4694                 /*
4695                  * When vmcs->vmcs_link_pointer is -1ull, any VMWRITE
4696                  * to shadowed-field sets the ALU flags for VMfailInvalid.
4697                  */
4698                 if (get_vmcs12(vcpu)->vmcs_link_pointer == -1ull)
4699                         return nested_vmx_failInvalid(vcpu);
4700                 vmcs12 = get_shadow_vmcs12(vcpu);
4701         }
4702 
4703         offset = vmcs_field_to_offset(field);
4704         if (offset < 0)
4705                 return nested_vmx_failValid(vcpu,
4706                         VMXERR_UNSUPPORTED_VMCS_COMPONENT);
4707 
4708         /*
4709          * Some Intel CPUs intentionally drop the reserved bits of the AR byte
4710          * fields on VMWRITE.  Emulate this behavior to ensure consistent KVM
4711          * behavior regardless of the underlying hardware, e.g. if an AR_BYTE
4712          * field is intercepted for VMWRITE but not VMREAD (in L1), then VMREAD
4713          * from L1 will return a different value than VMREAD from L2 (L1 sees
4714          * the stripped down value, L2 sees the full value as stored by KVM).
4715          */
4716         if (field >= GUEST_ES_AR_BYTES && field <= GUEST_TR_AR_BYTES)
4717                 field_value &= 0x1f0ff;
4718 
4719         vmcs12_write_any(vmcs12, field, offset, field_value);
4720 
4721         /*
4722          * Do not track vmcs12 dirty-state if in guest-mode as we actually
4723          * dirty shadow vmcs12 instead of vmcs12.  Fields that can be updated
4724          * by L1 without a vmexit are always updated in the vmcs02, i.e. don't
4725          * "dirty" vmcs12, all others go down the prepare_vmcs02() slow path.
4726          */
4727         if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field)) {
4728                 /*
4729                  * L1 can read these fields without exiting, ensure the
4730                  * shadow VMCS is up-to-date.
4731                  */
4732                 if (enable_shadow_vmcs && is_shadow_field_ro(field)) {
4733                         preempt_disable();
4734                         vmcs_load(vmx->vmcs01.shadow_vmcs);
4735 
4736                         __vmcs_writel(field, field_value);
4737 
4738                         vmcs_clear(vmx->vmcs01.shadow_vmcs);
4739                         vmcs_load(vmx->loaded_vmcs->vmcs);
4740                         preempt_enable();
4741                 }
4742                 vmx->nested.dirty_vmcs12 = true;
4743         }
4744 
4745         return nested_vmx_succeed(vcpu);
4746 }
4747 
4748 static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr)
4749 {
4750         vmx->nested.current_vmptr = vmptr;
4751         if (enable_shadow_vmcs) {
4752                 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_SHADOW_VMCS);
4753                 vmcs_write64(VMCS_LINK_POINTER,
4754                              __pa(vmx->vmcs01.shadow_vmcs));
4755                 vmx->nested.need_vmcs12_to_shadow_sync = true;
4756         }
4757         vmx->nested.dirty_vmcs12 = true;
4758 }
4759 
4760 /* Emulate the VMPTRLD instruction */
4761 static int handle_vmptrld(struct kvm_vcpu *vcpu)
4762 {
4763         struct vcpu_vmx *vmx = to_vmx(vcpu);
4764         gpa_t vmptr;
4765 
4766         if (!nested_vmx_check_permission(vcpu))
4767                 return 1;
4768 
4769         if (nested_vmx_get_vmptr(vcpu, &vmptr))
4770                 return 1;
4771 
4772         if (!page_address_valid(vcpu, vmptr))
4773                 return nested_vmx_failValid(vcpu,
4774                         VMXERR_VMPTRLD_INVALID_ADDRESS);
4775 
4776         if (vmptr == vmx->nested.vmxon_ptr)
4777                 return nested_vmx_failValid(vcpu,
4778                         VMXERR_VMPTRLD_VMXON_POINTER);
4779 
4780         /* Forbid normal VMPTRLD if Enlightened version was used */
4781         if (vmx->nested.hv_evmcs)
4782                 return 1;
4783 
4784         if (vmx->nested.current_vmptr != vmptr) {
4785                 struct kvm_host_map map;
4786                 struct vmcs12 *new_vmcs12;
4787 
4788                 if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmptr), &map)) {
4789                         /*
4790                          * Reads from an unbacked page return all 1s,
4791                          * which means that the 32 bits located at the
4792                          * given physical address won't match the required
4793                          * VMCS12_REVISION identifier.
4794                          */
4795                         return nested_vmx_failValid(vcpu,
4796                                 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
4797                 }
4798 
4799                 new_vmcs12 = map.hva;
4800 
4801                 if (new_vmcs12->hdr.revision_id != VMCS12_REVISION ||
4802                     (new_vmcs12->hdr.shadow_vmcs &&
4803                      !nested_cpu_has_vmx_shadow_vmcs(vcpu))) {
4804                         kvm_vcpu_unmap(vcpu, &map, false);
4805                         return nested_vmx_failValid(vcpu,
4806                                 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
4807                 }
4808 
4809                 nested_release_vmcs12(vcpu);
4810 
4811                 /*
4812                  * Load VMCS12 from guest memory since it is not already
4813                  * cached.
4814                  */
4815                 memcpy(vmx->nested.cached_vmcs12, new_vmcs12, VMCS12_SIZE);
4816                 kvm_vcpu_unmap(vcpu, &map, false);
4817 
4818                 set_current_vmptr(vmx, vmptr);
4819         }
4820 
4821         return nested_vmx_succeed(vcpu);
4822 }
4823 
4824 /* Emulate the VMPTRST instruction */
4825 static int handle_vmptrst(struct kvm_vcpu *vcpu)
4826 {
4827         unsigned long exit_qual = vmcs_readl(EXIT_QUALIFICATION);
4828         u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO);
4829         gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr;
4830         struct x86_exception e;
4831         gva_t gva;
4832 
4833         if (!nested_vmx_check_permission(vcpu))
4834                 return 1;
4835 
4836         if (unlikely(to_vmx(vcpu)->nested.hv_evmcs))
4837                 return 1;
4838 
4839         if (get_vmx_mem_address(vcpu, exit_qual, instr_info,
4840                                 true, sizeof(gpa_t), &gva))
4841                 return 1;
4842         /* *_system ok, nested_vmx_check_permission has verified cpl=0 */
4843         if (kvm_write_guest_virt_system(vcpu, gva, (void *)&current_vmptr,
4844                                         sizeof(gpa_t), &e)) {
4845                 kvm_inject_page_fault(vcpu, &e);
4846                 return 1;
4847         }
4848         return nested_vmx_succeed(vcpu);
4849 }
4850 
4851 /* Emulate the INVEPT instruction */
4852 static int handle_invept(struct kvm_vcpu *vcpu)
4853 {
4854         struct vcpu_vmx *vmx = to_vmx(vcpu);
4855         u32 vmx_instruction_info, types;
4856         unsigned long type;
4857         gva_t gva;
4858         struct x86_exception e;
4859         struct {
4860                 u64 eptp, gpa;
4861         } operand;
4862 
4863         if (!(vmx->nested.msrs.secondary_ctls_high &
4864               SECONDARY_EXEC_ENABLE_EPT) ||
4865             !(vmx->nested.msrs.ept_caps & VMX_EPT_INVEPT_BIT)) {
4866                 kvm_queue_exception(vcpu, UD_VECTOR);
4867                 return 1;
4868         }
4869 
4870         if (!nested_vmx_check_permission(vcpu))
4871                 return 1;
4872 
4873         vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
4874         type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
4875 
4876         types = (vmx->nested.msrs.ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6;
4877 
4878         if (type >= 32 || !(types & (1 << type)))
4879                 return nested_vmx_failValid(vcpu,
4880                                 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
4881 
4882         /* According to the Intel VMX instruction reference, the memory
4883          * operand is read even if it isn't needed (e.g., for type==global)
4884          */
4885         if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
4886                         vmx_instruction_info, false, sizeof(operand), &gva))
4887                 return 1;
4888         if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) {
4889                 kvm_inject_page_fault(vcpu, &e);
4890                 return 1;
4891         }
4892 
4893         switch (type) {
4894         case VMX_EPT_EXTENT_GLOBAL:
4895         case VMX_EPT_EXTENT_CONTEXT:
4896         /*
4897          * TODO: Sync the necessary shadow EPT roots here, rather than
4898          * at the next emulated VM-entry.
4899          */
4900                 break;
4901         default:
4902                 BUG_ON(1);
4903                 break;
4904         }
4905 
4906         return nested_vmx_succeed(vcpu);
4907 }
4908 
4909 static int handle_invvpid(struct kvm_vcpu *vcpu)
4910 {
4911         struct vcpu_vmx *vmx = to_vmx(vcpu);
4912         u32 vmx_instruction_info;
4913         unsigned long type, types;
4914         gva_t gva;
4915         struct x86_exception e;
4916         struct {
4917                 u64 vpid;
4918                 u64 gla;
4919         } operand;
4920         u16 vpid02;
4921 
4922         if (!(vmx->nested.msrs.secondary_ctls_high &
4923               SECONDARY_EXEC_ENABLE_VPID) ||
4924                         !(vmx->nested.msrs.vpid_caps & VMX_VPID_INVVPID_BIT)) {
4925                 kvm_queue_exception(vcpu, UD_VECTOR);
4926                 return 1;
4927         }
4928 
4929         if (!nested_vmx_check_permission(vcpu))
4930                 return 1;
4931 
4932         vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
4933         type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
4934 
4935         types = (vmx->nested.msrs.vpid_caps &
4936                         VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8;
4937 
4938         if (type >= 32 || !(types & (1 << type)))
4939                 return nested_vmx_failValid(vcpu,
4940                         VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
4941 
4942         /* according to the intel vmx instruction reference, the memory
4943          * operand is read even if it isn't needed (e.g., for type==global)
4944          */
4945         if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
4946                         vmx_instruction_info, false, sizeof(operand), &gva))
4947                 return 1;
4948         if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) {
4949                 kvm_inject_page_fault(vcpu, &e);
4950                 return 1;
4951         }
4952         if (operand.vpid >> 16)
4953                 return nested_vmx_failValid(vcpu,
4954                         VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
4955 
4956         vpid02 = nested_get_vpid02(vcpu);
4957         switch (type) {
4958         case VMX_VPID_EXTENT_INDIVIDUAL_ADDR:
4959                 if (!operand.vpid ||
4960                     is_noncanonical_address(operand.gla, vcpu))
4961                         return nested_vmx_failValid(vcpu,
4962                                 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
4963                 if (cpu_has_vmx_invvpid_individual_addr()) {
4964                         __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR,
4965                                 vpid02, operand.gla);
4966                 } else
4967                         __vmx_flush_tlb(vcpu, vpid02, false);
4968                 break;
4969         case VMX_VPID_EXTENT_SINGLE_CONTEXT:
4970         case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL:
4971                 if (!operand.vpid)
4972                         return nested_vmx_failValid(vcpu,
4973                                 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
4974                 __vmx_flush_tlb(vcpu, vpid02, false);
4975                 break;
4976         case VMX_VPID_EXTENT_ALL_CONTEXT:
4977                 __vmx_flush_tlb(vcpu, vpid02, false);
4978                 break;
4979         default:
4980                 WARN_ON_ONCE(1);
4981                 return kvm_skip_emulated_instruction(vcpu);
4982         }
4983 
4984         return nested_vmx_succeed(vcpu);
4985 }
4986 
4987 static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu,
4988                                      struct vmcs12 *vmcs12)
4989 {
4990         u32 index = kvm_rcx_read(vcpu);
4991         u64 address;
4992         bool accessed_dirty;
4993         struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
4994 
4995         if (!nested_cpu_has_eptp_switching(vmcs12) ||
4996             !nested_cpu_has_ept(vmcs12))
4997                 return 1;
4998 
4999         if (index >= VMFUNC_EPTP_ENTRIES)
5000                 return 1;
5001 
5002 
5003         if (kvm_vcpu_read_guest_page(vcpu, vmcs12->eptp_list_address >> PAGE_SHIFT,
5004                                      &address, index * 8, 8))
5005                 return 1;
5006 
5007         accessed_dirty = !!(address & VMX_EPTP_AD_ENABLE_BIT);
5008 
5009         /*
5010          * If the (L2) guest does a vmfunc to the currently
5011          * active ept pointer, we don't have to do anything else
5012          */
5013         if (vmcs12->ept_pointer != address) {
5014                 if (!valid_ept_address(vcpu, address))
5015                         return 1;
5016 
5017                 kvm_mmu_unload(vcpu);
5018                 mmu->ept_ad = accessed_dirty;
5019                 mmu->mmu_role.base.ad_disabled = !accessed_dirty;
5020                 vmcs12->ept_pointer = address;
5021                 /*
5022                  * TODO: Check what's the correct approach in case
5023                  * mmu reload fails. Currently, we just let the next
5024                  * reload potentially fail
5025                  */
5026                 kvm_mmu_reload(vcpu);
5027         }
5028 
5029         return 0;
5030 }
5031 
5032 static int handle_vmfunc(struct kvm_vcpu *vcpu)
5033 {
5034         struct vcpu_vmx *vmx = to_vmx(vcpu);
5035         struct vmcs12 *vmcs12;
5036         u32 function = kvm_rax_read(vcpu);
5037 
5038         /*
5039          * VMFUNC is only supported for nested guests, but we always enable the
5040          * secondary control for simplicity; for non-nested mode, fake that we
5041          * didn't by injecting #UD.
5042          */
5043         if (!is_guest_mode(vcpu)) {
5044                 kvm_queue_exception(vcpu, UD_VECTOR);
5045                 return 1;
5046         }
5047 
5048         vmcs12 = get_vmcs12(vcpu);
5049         if ((vmcs12->vm_function_control & (1 << function)) == 0)
5050                 goto fail;
5051 
5052         switch (function) {
5053         case 0:
5054                 if (nested_vmx_eptp_switching(vcpu, vmcs12))
5055                         goto fail;
5056                 break;
5057         default:
5058                 goto fail;
5059         }
5060         return kvm_skip_emulated_instruction(vcpu);
5061 
5062 fail:
5063         nested_vmx_vmexit(vcpu, vmx->exit_reason,
5064                           vmcs_read32(VM_EXIT_INTR_INFO),
5065                           vmcs_readl(EXIT_QUALIFICATION));
5066         return 1;
5067 }
5068 
5069 
5070 static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu,
5071                                        struct vmcs12 *vmcs12)
5072 {
5073         unsigned long exit_qualification;
5074         gpa_t bitmap, last_bitmap;
5075         unsigned int port;
5076         int size;
5077         u8 b;
5078 
5079         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
5080                 return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING);
5081 
5082         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5083 
5084         port = exit_qualification >> 16;
5085         size = (exit_qualification & 7) + 1;
5086 
5087         last_bitmap = (gpa_t)-1;
5088         b = -1;
5089 
5090         while (size > 0) {
5091                 if (port < 0x8000)
5092                         bitmap = vmcs12->io_bitmap_a;
5093                 else if (port < 0x10000)
5094                         bitmap = vmcs12->io_bitmap_b;
5095                 else
5096                         return true;
5097                 bitmap += (port & 0x7fff) / 8;
5098 
5099                 if (last_bitmap != bitmap)
5100                         if (kvm_vcpu_read_guest(vcpu, bitmap, &b, 1))
5101                                 return true;
5102                 if (b & (1 << (port & 7)))
5103                         return true;
5104 
5105                 port++;
5106                 size--;
5107                 last_bitmap = bitmap;
5108         }
5109 
5110         return false;
5111 }
5112 
5113 /*
5114  * Return 1 if we should exit from L2 to L1 to handle an MSR access access,
5115  * rather than handle it ourselves in L0. I.e., check whether L1 expressed
5116  * disinterest in the current event (read or write a specific MSR) by using an
5117  * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps.
5118  */
5119 static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
5120         struct vmcs12 *vmcs12, u32 exit_reason)
5121 {
5122         u32 msr_index = kvm_rcx_read(vcpu);
5123         gpa_t bitmap;
5124 
5125         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
5126                 return true;
5127 
5128         /*
5129          * The MSR_BITMAP page is divided into four 1024-byte bitmaps,
5130          * for the four combinations of read/write and low/high MSR numbers.
5131          * First we need to figure out which of the four to use:
5132          */
5133         bitmap = vmcs12->msr_bitmap;
5134         if (exit_reason == EXIT_REASON_MSR_WRITE)
5135                 bitmap += 2048;
5136         if (msr_index >= 0xc0000000) {
5137                 msr_index -= 0xc0000000;
5138                 bitmap += 1024;
5139         }
5140 
5141         /* Then read the msr_index'th bit from this bitmap: */
5142         if (msr_index < 1024*8) {
5143                 unsigned char b;
5144                 if (kvm_vcpu_read_guest(vcpu, bitmap + msr_index/8, &b, 1))
5145                         return true;
5146                 return 1 & (b >> (msr_index & 7));
5147         } else
5148                 return true; /* let L1 handle the wrong parameter */
5149 }
5150 
5151 /*
5152  * Return 1 if we should exit from L2 to L1 to handle a CR access exit,
5153  * rather than handle it ourselves in L0. I.e., check if L1 wanted to
5154  * intercept (via guest_host_mask etc.) the current event.
5155  */
5156 static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu,
5157         struct vmcs12 *vmcs12)
5158 {
5159         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5160         int cr = exit_qualification & 15;
5161         int reg;
5162         unsigned long val;
5163 
5164         switch ((exit_qualification >> 4) & 3) {
5165         case 0: /* mov to cr */
5166                 reg = (exit_qualification >> 8) & 15;
5167                 val = kvm_register_readl(vcpu, reg);
5168                 switch (cr) {
5169                 case 0:
5170                         if (vmcs12->cr0_guest_host_mask &
5171                             (val ^ vmcs12->cr0_read_shadow))
5172                                 return true;
5173                         break;
5174                 case 3:
5175                         if ((vmcs12->cr3_target_count >= 1 &&
5176                                         vmcs12->cr3_target_value0 == val) ||
5177                                 (vmcs12->cr3_target_count >= 2 &&
5178                                         vmcs12->cr3_target_value1 == val) ||
5179                                 (vmcs12->cr3_target_count >= 3 &&
5180                                         vmcs12->cr3_target_value2 == val) ||
5181                                 (vmcs12->cr3_target_count >= 4 &&
5182                                         vmcs12->cr3_target_value3 == val))
5183                                 return false;
5184                         if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING))
5185                                 return true;
5186                         break;
5187                 case 4:
5188                         if (vmcs12->cr4_guest_host_mask &
5189                             (vmcs12->cr4_read_shadow ^ val))
5190                                 return true;
5191                         break;
5192                 case 8:
5193                         if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING))
5194                                 return true;
5195                         break;
5196                 }
5197                 break;
5198         case 2: /* clts */
5199                 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) &&
5200                     (vmcs12->cr0_read_shadow & X86_CR0_TS))
5201                         return true;
5202                 break;
5203         case 1: /* mov from cr */
5204                 switch (cr) {
5205                 case 3:
5206                         if (vmcs12->cpu_based_vm_exec_control &
5207                             CPU_BASED_CR3_STORE_EXITING)
5208                                 return true;
5209                         break;
5210                 case 8:
5211                         if (vmcs12->cpu_based_vm_exec_control &
5212                             CPU_BASED_CR8_STORE_EXITING)
5213                                 return true;
5214                         break;
5215                 }
5216                 break;
5217         case 3: /* lmsw */
5218                 /*
5219                  * lmsw can change bits 1..3 of cr0, and only set bit 0 of
5220                  * cr0. Other attempted changes are ignored, with no exit.
5221                  */
5222                 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
5223                 if (vmcs12->cr0_guest_host_mask & 0xe &
5224                     (val ^ vmcs12->cr0_read_shadow))
5225                         return true;
5226                 if ((vmcs12->cr0_guest_host_mask & 0x1) &&
5227                     !(vmcs12->cr0_read_shadow & 0x1) &&
5228                     (val & 0x1))
5229                         return true;
5230                 break;
5231         }
5232         return false;
5233 }
5234 
5235 static bool nested_vmx_exit_handled_vmcs_access(struct kvm_vcpu *vcpu,
5236         struct vmcs12 *vmcs12, gpa_t bitmap)
5237 {
5238         u32 vmx_instruction_info;
5239         unsigned long field;
5240         u8 b;
5241 
5242         if (!nested_cpu_has_shadow_vmcs(vmcs12))
5243                 return true;
5244 
5245         /* Decode instruction info and find the field to access */
5246         vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
5247         field = kvm_register_read(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
5248 
5249         /* Out-of-range fields always cause a VM exit from L2 to L1 */
5250         if (field >> 15)
5251                 return true;
5252 
5253         if (kvm_vcpu_read_guest(vcpu, bitmap + field/8, &b, 1))
5254                 return true;
5255 
5256         return 1 & (b >> (field & 7));
5257 }
5258 
5259 /*
5260  * Return 1 if we should exit from L2 to L1 to handle an exit, or 0 if we
5261  * should handle it ourselves in L0 (and then continue L2). Only call this
5262  * when in is_guest_mode (L2).
5263  */
5264 bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason)
5265 {
5266         u32 intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
5267         struct vcpu_vmx *vmx = to_vmx(vcpu);
5268         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
5269 
5270         if (vmx->nested.nested_run_pending)
5271                 return false;
5272 
5273         if (unlikely(vmx->fail)) {
5274                 pr_info_ratelimited("%s failed vm entry %x\n", __func__,
5275                                     vmcs_read32(VM_INSTRUCTION_ERROR));
5276                 return true;
5277         }
5278 
5279         /*
5280          * The host physical addresses of some pages of guest memory
5281          * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC
5282          * Page). The CPU may write to these pages via their host
5283          * physical address while L2 is running, bypassing any
5284          * address-translation-based dirty tracking (e.g. EPT write
5285          * protection).
5286          *
5287          * Mark them dirty on every exit from L2 to prevent them from
5288          * getting out of sync with dirty tracking.
5289          */
5290         nested_mark_vmcs12_pages_dirty(vcpu);
5291 
5292         trace_kvm_nested_vmexit(kvm_rip_read(vcpu), exit_reason,
5293                                 vmcs_readl(EXIT_QUALIFICATION),
5294                                 vmx->idt_vectoring_info,
5295                                 intr_info,
5296                                 vmcs_read32(VM_EXIT_INTR_ERROR_CODE),
5297                                 KVM_ISA_VMX);
5298 
5299         switch (exit_reason) {
5300         case EXIT_REASON_EXCEPTION_NMI:
5301                 if (is_nmi(intr_info))
5302                         return false;
5303                 else if (is_page_fault(intr_info))
5304                         return !vmx->vcpu.arch.apf.host_apf_reason && enable_ept;
5305                 else if (is_debug(intr_info) &&
5306                          vcpu->guest_debug &
5307                          (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
5308                         return false;
5309                 else if (is_breakpoint(intr_info) &&
5310                          vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
5311                         return false;
5312                 return vmcs12->exception_bitmap &
5313                                 (1u << (intr_info & INTR_INFO_VECTOR_MASK));
5314         case EXIT_REASON_EXTERNAL_INTERRUPT:
5315                 return false;
5316         case EXIT_REASON_TRIPLE_FAULT:
5317                 return true;
5318         case EXIT_REASON_PENDING_INTERRUPT:
5319                 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_INTR_PENDING);
5320         case EXIT_REASON_NMI_WINDOW:
5321                 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING);
5322         case EXIT_REASON_TASK_SWITCH:
5323                 return true;
5324         case EXIT_REASON_CPUID:
5325                 return true;
5326         case EXIT_REASON_HLT:
5327                 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING);
5328         case EXIT_REASON_INVD:
5329                 return true;
5330         case EXIT_REASON_INVLPG:
5331                 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
5332         case EXIT_REASON_RDPMC:
5333                 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING);
5334         case EXIT_REASON_RDRAND:
5335                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDRAND_EXITING);
5336         case EXIT_REASON_RDSEED:
5337                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDSEED_EXITING);
5338         case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP:
5339                 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING);
5340         case EXIT_REASON_VMREAD:
5341                 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12,
5342                         vmcs12->vmread_bitmap);
5343         case EXIT_REASON_VMWRITE:
5344                 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12,
5345                         vmcs12->vmwrite_bitmap);
5346         case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR:
5347         case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD:
5348         case EXIT_REASON_VMPTRST: case EXIT_REASON_VMRESUME:
5349         case EXIT_REASON_VMOFF: case EXIT_REASON_VMON:
5350         case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID:
5351                 /*
5352                  * VMX instructions trap unconditionally. This allows L1 to
5353                  * emulate them for its L2 guest, i.e., allows 3-level nesting!
5354                  */
5355                 return true;
5356         case EXIT_REASON_CR_ACCESS:
5357                 return nested_vmx_exit_handled_cr(vcpu, vmcs12);
5358         case EXIT_REASON_DR_ACCESS:
5359                 return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING);
5360         case EXIT_REASON_IO_INSTRUCTION:
5361                 return nested_vmx_exit_handled_io(vcpu, vmcs12);
5362         case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR:
5363                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC);
5364         case EXIT_REASON_MSR_READ:
5365         case EXIT_REASON_MSR_WRITE:
5366                 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason);
5367         case EXIT_REASON_INVALID_STATE:
5368                 return true;
5369         case EXIT_REASON_MWAIT_INSTRUCTION:
5370                 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING);
5371         case EXIT_REASON_MONITOR_TRAP_FLAG:
5372                 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_TRAP_FLAG);
5373         case EXIT_REASON_MONITOR_INSTRUCTION:
5374                 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING);
5375         case EXIT_REASON_PAUSE_INSTRUCTION:
5376                 return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) ||
5377                         nested_cpu_has2(vmcs12,
5378                                 SECONDARY_EXEC_PAUSE_LOOP_EXITING);
5379         case EXIT_REASON_MCE_DURING_VMENTRY:
5380                 return false;
5381         case EXIT_REASON_TPR_BELOW_THRESHOLD:
5382                 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW);
5383         case EXIT_REASON_APIC_ACCESS:
5384         case EXIT_REASON_APIC_WRITE:
5385         case EXIT_REASON_EOI_INDUCED:
5386                 /*
5387                  * The controls for "virtualize APIC accesses," "APIC-
5388                  * register virtualization," and "virtual-interrupt
5389                  * delivery" only come from vmcs12.
5390                  */
5391                 return true;
5392         case EXIT_REASON_EPT_VIOLATION:
5393                 /*
5394                  * L0 always deals with the EPT violation. If nested EPT is
5395                  * used, and the nested mmu code discovers that the address is
5396                  * missing in the guest EPT table (EPT12), the EPT violation
5397                  * will be injected with nested_ept_inject_page_fault()
5398                  */
5399                 return false;
5400         case EXIT_REASON_EPT_MISCONFIG:
5401                 /*
5402                  * L2 never uses directly L1's EPT, but rather L0's own EPT
5403                  * table (shadow on EPT) or a merged EPT table that L0 built
5404                  * (EPT on EPT). So any problems with the structure of the
5405                  * table is L0's fault.
5406                  */
5407                 return false;
5408         case EXIT_REASON_INVPCID:
5409                 return
5410                         nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_INVPCID) &&
5411                         nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
5412         case EXIT_REASON_WBINVD:
5413                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING);
5414         case EXIT_REASON_XSETBV:
5415                 return true;
5416         case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS:
5417                 /*
5418                  * This should never happen, since it is not possible to
5419                  * set XSS to a non-zero value---neither in L1 nor in L2.
5420                  * If if it were, XSS would have to be checked against
5421                  * the XSS exit bitmap in vmcs12.
5422                  */
5423                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES);
5424         case EXIT_REASON_PREEMPTION_TIMER:
5425                 return false;
5426         case EXIT_REASON_PML_FULL:
5427                 /* We emulate PML support to L1. */
5428                 return false;
5429         case EXIT_REASON_VMFUNC:
5430                 /* VM functions are emulated through L2->L0 vmexits. */
5431                 return false;
5432         case EXIT_REASON_ENCLS:
5433                 /* SGX is never exposed to L1 */
5434                 return false;
5435         default:
5436                 return true;
5437         }
5438 }
5439 
5440 
5441 static int vmx_get_nested_state(struct kvm_vcpu *vcpu,
5442                                 struct kvm_nested_state __user *user_kvm_nested_state,
5443                                 u32 user_data_size)
5444 {
5445         struct vcpu_vmx *vmx;
5446         struct vmcs12 *vmcs12;
5447         struct kvm_nested_state kvm_state = {
5448                 .flags = 0,
5449                 .format = KVM_STATE_NESTED_FORMAT_VMX,
5450                 .size = sizeof(kvm_state),
5451                 .hdr.vmx.vmxon_pa = -1ull,
5452                 .hdr.vmx.vmcs12_pa = -1ull,
5453         };
5454         struct kvm_vmx_nested_state_data __user *user_vmx_nested_state =
5455                 &user_kvm_nested_state->data.vmx[0];
5456 
5457         if (!vcpu)
5458                 return kvm_state.size + sizeof(*user_vmx_nested_state);
5459 
5460         vmx = to_vmx(vcpu);
5461         vmcs12 = get_vmcs12(vcpu);
5462 
5463         if (nested_vmx_allowed(vcpu) &&
5464             (vmx->nested.vmxon || vmx->nested.smm.vmxon)) {
5465                 kvm_state.hdr.vmx.vmxon_pa = vmx->nested.vmxon_ptr;
5466                 kvm_state.hdr.vmx.vmcs12_pa = vmx->nested.current_vmptr;
5467 
5468                 if (vmx_has_valid_vmcs12(vcpu)) {
5469                         kvm_state.size += sizeof(user_vmx_nested_state->vmcs12);
5470 
5471                         if (vmx->nested.hv_evmcs)
5472                                 kvm_state.flags |= KVM_STATE_NESTED_EVMCS;
5473 
5474                         if (is_guest_mode(vcpu) &&
5475                             nested_cpu_has_shadow_vmcs(vmcs12) &&
5476                             vmcs12->vmcs_link_pointer != -1ull)
5477                                 kvm_state.size += sizeof(user_vmx_nested_state->shadow_vmcs12);
5478                 }
5479 
5480                 if (vmx->nested.smm.vmxon)
5481                         kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_VMXON;
5482 
5483                 if (vmx->nested.smm.guest_mode)
5484                         kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_GUEST_MODE;
5485 
5486                 if (is_guest_mode(vcpu)) {
5487                         kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE;
5488 
5489                         if (vmx->nested.nested_run_pending)
5490                                 kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING;
5491                 }
5492         }
5493 
5494         if (user_data_size < kvm_state.size)
5495                 goto out;
5496 
5497         if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state)))
5498                 return -EFAULT;
5499 
5500         if (!vmx_has_valid_vmcs12(vcpu))
5501                 goto out;
5502 
5503         /*
5504          * When running L2, the authoritative vmcs12 state is in the
5505          * vmcs02. When running L1, the authoritative vmcs12 state is
5506          * in the shadow or enlightened vmcs linked to vmcs01, unless
5507          * need_vmcs12_to_shadow_sync is set, in which case, the authoritative
5508          * vmcs12 state is in the vmcs12 already.
5509          */
5510         if (is_guest_mode(vcpu)) {
5511                 sync_vmcs02_to_vmcs12(vcpu, vmcs12);
5512                 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
5513         } else if (!vmx->nested.need_vmcs12_to_shadow_sync) {
5514                 if (vmx->nested.hv_evmcs)
5515                         copy_enlightened_to_vmcs12(vmx);
5516                 else if (enable_shadow_vmcs)
5517                         copy_shadow_to_vmcs12(vmx);
5518         }
5519 
5520         BUILD_BUG_ON(sizeof(user_vmx_nested_state->vmcs12) < VMCS12_SIZE);
5521         BUILD_BUG_ON(sizeof(user_vmx_nested_state->shadow_vmcs12) < VMCS12_SIZE);
5522 
5523         /*
5524          * Copy over the full allocated size of vmcs12 rather than just the size
5525          * of the struct.
5526          */
5527         if (copy_to_user(user_vmx_nested_state->vmcs12, vmcs12, VMCS12_SIZE))
5528                 return -EFAULT;
5529 
5530         if (nested_cpu_has_shadow_vmcs(vmcs12) &&
5531             vmcs12->vmcs_link_pointer != -1ull) {
5532                 if (copy_to_user(user_vmx_nested_state->shadow_vmcs12,
5533                                  get_shadow_vmcs12(vcpu), VMCS12_SIZE))
5534                         return -EFAULT;
5535         }
5536 
5537 out:
5538         return kvm_state.size;
5539 }
5540 
5541 /*
5542  * Forcibly leave nested mode in order to be able to reset the VCPU later on.
5543  */
5544 void vmx_leave_nested(struct kvm_vcpu *vcpu)
5545 {
5546         if (is_guest_mode(vcpu)) {
5547                 to_vmx(vcpu)->nested.nested_run_pending = 0;
5548                 nested_vmx_vmexit(vcpu, -1, 0, 0);
5549         }
5550         free_nested(vcpu);
5551 }
5552 
5553 static int vmx_set_nested_state(struct kvm_vcpu *vcpu,
5554                                 struct kvm_nested_state __user *user_kvm_nested_state,
5555                                 struct kvm_nested_state *kvm_state)
5556 {
5557         struct vcpu_vmx *vmx = to_vmx(vcpu);
5558         struct vmcs12 *vmcs12;
5559         u32 exit_qual;
5560         struct kvm_vmx_nested_state_data __user *user_vmx_nested_state =
5561                 &user_kvm_nested_state->data.vmx[0];
5562         int ret;
5563 
5564         if (kvm_state->format != KVM_STATE_NESTED_FORMAT_VMX)
5565                 return -EINVAL;
5566 
5567         if (kvm_state->hdr.vmx.vmxon_pa == -1ull) {
5568                 if (kvm_state->hdr.vmx.smm.flags)
5569                         return -EINVAL;
5570 
5571                 if (kvm_state->hdr.vmx.vmcs12_pa != -1ull)
5572                         return -EINVAL;
5573 
5574                 /*
5575                  * KVM_STATE_NESTED_EVMCS used to signal that KVM should
5576                  * enable eVMCS capability on vCPU. However, since then
5577                  * code was changed such that flag signals vmcs12 should
5578                  * be copied into eVMCS in guest memory.
5579                  *
5580                  * To preserve backwards compatability, allow user
5581                  * to set this flag even when there is no VMXON region.
5582                  */
5583                 if (kvm_state->flags & ~KVM_STATE_NESTED_EVMCS)
5584                         return -EINVAL;
5585         } else {
5586                 if (!nested_vmx_allowed(vcpu))
5587                         return -EINVAL;
5588 
5589                 if (!page_address_valid(vcpu, kvm_state->hdr.vmx.vmxon_pa))
5590                         return -EINVAL;
5591         }
5592 
5593         if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) &&
5594             (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE))
5595                 return -EINVAL;
5596 
5597         if (kvm_state->hdr.vmx.smm.flags &
5598             ~(KVM_STATE_NESTED_SMM_GUEST_MODE | KVM_STATE_NESTED_SMM_VMXON))
5599                 return -EINVAL;
5600 
5601         /*
5602          * SMM temporarily disables VMX, so we cannot be in guest mode,
5603          * nor can VMLAUNCH/VMRESUME be pending.  Outside SMM, SMM flags
5604          * must be zero.
5605          */
5606         if (is_smm(vcpu) ?
5607                 (kvm_state->flags &
5608                  (KVM_STATE_NESTED_GUEST_MODE | KVM_STATE_NESTED_RUN_PENDING))
5609                 : kvm_state->hdr.vmx.smm.flags)
5610                 return -EINVAL;
5611 
5612         if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) &&
5613             !(kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON))
5614                 return -EINVAL;
5615 
5616         if ((kvm_state->flags & KVM_STATE_NESTED_EVMCS) &&
5617                 (!nested_vmx_allowed(vcpu) || !vmx->nested.enlightened_vmcs_enabled))
5618                         return -EINVAL;
5619 
5620         vmx_leave_nested(vcpu);
5621 
5622         if (kvm_state->hdr.vmx.vmxon_pa == -1ull)
5623                 return 0;
5624 
5625         vmx->nested.vmxon_ptr = kvm_state->hdr.vmx.vmxon_pa;
5626         ret = enter_vmx_operation(vcpu);
5627         if (ret)
5628                 return ret;
5629 
5630         /* Empty 'VMXON' state is permitted */
5631         if (kvm_state->size < sizeof(*kvm_state) + sizeof(*vmcs12))
5632                 return 0;
5633 
5634         if (kvm_state->hdr.vmx.vmcs12_pa != -1ull) {
5635                 if (kvm_state->hdr.vmx.vmcs12_pa == kvm_state->hdr.vmx.vmxon_pa ||
5636                     !page_address_valid(vcpu, kvm_state->hdr.vmx.vmcs12_pa))
5637                         return -EINVAL;
5638 
5639                 set_current_vmptr(vmx, kvm_state->hdr.vmx.vmcs12_pa);
5640         } else if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) {
5641                 /*
5642                  * Sync eVMCS upon entry as we may not have
5643                  * HV_X64_MSR_VP_ASSIST_PAGE set up yet.
5644                  */
5645                 vmx->nested.need_vmcs12_to_shadow_sync = true;
5646         } else {
5647                 return -EINVAL;
5648         }
5649 
5650         if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON) {
5651                 vmx->nested.smm.vmxon = true;
5652                 vmx->nested.vmxon = false;
5653 
5654                 if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE)
5655                         vmx->nested.smm.guest_mode = true;
5656         }
5657 
5658         vmcs12 = get_vmcs12(vcpu);
5659         if (copy_from_user(vmcs12, user_vmx_nested_state->vmcs12, sizeof(*vmcs12)))
5660                 return -EFAULT;
5661 
5662         if (vmcs12->hdr.revision_id != VMCS12_REVISION)
5663                 return -EINVAL;
5664 
5665         if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE))
5666                 return 0;
5667 
5668         vmx->nested.nested_run_pending =
5669                 !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING);
5670 
5671         ret = -EINVAL;
5672         if (nested_cpu_has_shadow_vmcs(vmcs12) &&
5673             vmcs12->vmcs_link_pointer != -1ull) {
5674                 struct vmcs12 *shadow_vmcs12 = get_shadow_vmcs12(vcpu);
5675 
5676                 if (kvm_state->size <
5677                     sizeof(*kvm_state) +
5678                     sizeof(user_vmx_nested_state->vmcs12) + sizeof(*shadow_vmcs12))
5679                         goto error_guest_mode;
5680 
5681                 if (copy_from_user(shadow_vmcs12,
5682                                    user_vmx_nested_state->shadow_vmcs12,
5683                                    sizeof(*shadow_vmcs12))) {
5684                         ret = -EFAULT;
5685                         goto error_guest_mode;
5686                 }
5687 
5688                 if (shadow_vmcs12->hdr.revision_id != VMCS12_REVISION ||
5689                     !shadow_vmcs12->hdr.shadow_vmcs)
5690                         goto error_guest_mode;
5691         }
5692 
5693         if (nested_vmx_check_controls(vcpu, vmcs12) ||
5694             nested_vmx_check_host_state(vcpu, vmcs12) ||
5695             nested_vmx_check_guest_state(vcpu, vmcs12, &exit_qual))
5696                 goto error_guest_mode;
5697 
5698         vmx->nested.dirty_vmcs12 = true;
5699         ret = nested_vmx_enter_non_root_mode(vcpu, false);
5700         if (ret)
5701                 goto error_guest_mode;
5702 
5703         return 0;
5704 
5705 error_guest_mode:
5706         vmx->nested.nested_run_pending = 0;
5707         return ret;
5708 }
5709 
5710 void nested_vmx_vcpu_setup(void)
5711 {
5712         if (enable_shadow_vmcs) {
5713                 vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap));
5714                 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap));
5715         }
5716 }
5717 
5718 /*
5719  * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be
5720  * returned for the various VMX controls MSRs when nested VMX is enabled.
5721  * The same values should also be used to verify that vmcs12 control fields are
5722  * valid during nested entry from L1 to L2.
5723  * Each of these control msrs has a low and high 32-bit half: A low bit is on
5724  * if the corresponding bit in the (32-bit) control field *must* be on, and a
5725  * bit in the high half is on if the corresponding bit in the control field
5726  * may be on. See also vmx_control_verify().
5727  */
5728 void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps,
5729                                 bool apicv)
5730 {
5731         /*
5732          * Note that as a general rule, the high half of the MSRs (bits in
5733          * the control fields which may be 1) should be initialized by the
5734          * intersection of the underlying hardware's MSR (i.e., features which
5735          * can be supported) and the list of features we want to expose -
5736          * because they are known to be properly supported in our code.
5737          * Also, usually, the low half of the MSRs (bits which must be 1) can
5738          * be set to 0, meaning that L1 may turn off any of these bits. The
5739          * reason is that if one of these bits is necessary, it will appear
5740          * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control
5741          * fields of vmcs01 and vmcs02, will turn these bits off - and
5742          * nested_vmx_exit_reflected() will not pass related exits to L1.
5743          * These rules have exceptions below.
5744          */
5745 
5746         /* pin-based controls */
5747         rdmsr(MSR_IA32_VMX_PINBASED_CTLS,
5748                 msrs->pinbased_ctls_low,
5749                 msrs->pinbased_ctls_high);
5750         msrs->pinbased_ctls_low |=
5751                 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
5752         msrs->pinbased_ctls_high &=
5753                 PIN_BASED_EXT_INTR_MASK |
5754                 PIN_BASED_NMI_EXITING |
5755                 PIN_BASED_VIRTUAL_NMIS |
5756                 (apicv ? PIN_BASED_POSTED_INTR : 0);
5757         msrs->pinbased_ctls_high |=
5758                 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
5759                 PIN_BASED_VMX_PREEMPTION_TIMER;
5760 
5761         /* exit controls */
5762         rdmsr(MSR_IA32_VMX_EXIT_CTLS,
5763                 msrs->exit_ctls_low,
5764                 msrs->exit_ctls_high);
5765         msrs->exit_ctls_low =
5766                 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
5767 
5768         msrs->exit_ctls_high &=
5769 #ifdef CONFIG_X86_64
5770                 VM_EXIT_HOST_ADDR_SPACE_SIZE |
5771 #endif
5772                 VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT;
5773         msrs->exit_ctls_high |=
5774                 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |
5775                 VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER |
5776                 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT;
5777 
5778         /* We support free control of debug control saving. */
5779         msrs->exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS;
5780 
5781         /* entry controls */
5782         rdmsr(MSR_IA32_VMX_ENTRY_CTLS,
5783                 msrs->entry_ctls_low,
5784                 msrs->entry_ctls_high);
5785         msrs->entry_ctls_low =
5786                 VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
5787         msrs->entry_ctls_high &=
5788 #ifdef CONFIG_X86_64
5789                 VM_ENTRY_IA32E_MODE |
5790 #endif
5791                 VM_ENTRY_LOAD_IA32_PAT;
5792         msrs->entry_ctls_high |=
5793                 (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER);
5794 
5795         /* We support free control of debug control loading. */
5796         msrs->entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS;
5797 
5798         /* cpu-based controls */
5799         rdmsr(MSR_IA32_VMX_PROCBASED_CTLS,
5800                 msrs->procbased_ctls_low,
5801                 msrs->procbased_ctls_high);
5802         msrs->procbased_ctls_low =
5803                 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
5804         msrs->procbased_ctls_high &=
5805                 CPU_BASED_VIRTUAL_INTR_PENDING |
5806                 CPU_BASED_VIRTUAL_NMI_PENDING | CPU_BASED_USE_TSC_OFFSETING |
5807                 CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING |
5808                 CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING |
5809                 CPU_BASED_CR3_STORE_EXITING |
5810 #ifdef CONFIG_X86_64
5811                 CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING |
5812 #endif
5813                 CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING |
5814                 CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG |
5815                 CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING |
5816                 CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING |
5817                 CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
5818         /*
5819          * We can allow some features even when not supported by the
5820          * hardware. For example, L1 can specify an MSR bitmap - and we
5821          * can use it to avoid exits to L1 - even when L0 runs L2
5822          * without MSR bitmaps.
5823          */
5824         msrs->procbased_ctls_high |=
5825                 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
5826                 CPU_BASED_USE_MSR_BITMAPS;
5827 
5828         /* We support free control of CR3 access interception. */
5829         msrs->procbased_ctls_low &=
5830                 ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING);
5831 
5832         /*
5833          * secondary cpu-based controls.  Do not include those that
5834          * depend on CPUID bits, they are added later by vmx_cpuid_update.
5835          */
5836         if (msrs->procbased_ctls_high & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)
5837                 rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2,
5838                       msrs->secondary_ctls_low,
5839                       msrs->secondary_ctls_high);
5840 
5841         msrs->secondary_ctls_low = 0;
5842         msrs->secondary_ctls_high &=
5843                 SECONDARY_EXEC_DESC |
5844                 SECONDARY_EXEC_RDTSCP |
5845                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
5846                 SECONDARY_EXEC_WBINVD_EXITING |
5847                 SECONDARY_EXEC_APIC_REGISTER_VIRT |
5848                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
5849                 SECONDARY_EXEC_RDRAND_EXITING |
5850                 SECONDARY_EXEC_ENABLE_INVPCID |
5851                 SECONDARY_EXEC_RDSEED_EXITING |
5852                 SECONDARY_EXEC_XSAVES;
5853 
5854         /*
5855          * We can emulate "VMCS shadowing," even if the hardware
5856          * doesn't support it.
5857          */
5858         msrs->secondary_ctls_high |=
5859                 SECONDARY_EXEC_SHADOW_VMCS;
5860 
5861         if (enable_ept) {
5862                 /* nested EPT: emulate EPT also to L1 */
5863                 msrs->secondary_ctls_high |=
5864                         SECONDARY_EXEC_ENABLE_EPT;
5865                 msrs->ept_caps = VMX_EPT_PAGE_WALK_4_BIT |
5866                          VMX_EPTP_WB_BIT | VMX_EPT_INVEPT_BIT;
5867                 if (cpu_has_vmx_ept_execute_only())
5868                         msrs->ept_caps |=
5869                                 VMX_EPT_EXECUTE_ONLY_BIT;
5870                 msrs->ept_caps &= ept_caps;
5871                 msrs->ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT |
5872                         VMX_EPT_EXTENT_CONTEXT_BIT | VMX_EPT_2MB_PAGE_BIT |
5873                         VMX_EPT_1GB_PAGE_BIT;
5874                 if (enable_ept_ad_bits) {
5875                         msrs->secondary_ctls_high |=
5876                                 SECONDARY_EXEC_ENABLE_PML;
5877                         msrs->ept_caps |= VMX_EPT_AD_BIT;
5878                 }
5879         }
5880 
5881         if (cpu_has_vmx_vmfunc()) {
5882                 msrs->secondary_ctls_high |=
5883                         SECONDARY_EXEC_ENABLE_VMFUNC;
5884                 /*
5885                  * Advertise EPTP switching unconditionally
5886                  * since we emulate it
5887                  */
5888                 if (enable_ept)
5889                         msrs->vmfunc_controls =
5890                                 VMX_VMFUNC_EPTP_SWITCHING;
5891         }
5892 
5893         /*
5894          * Old versions of KVM use the single-context version without
5895          * checking for support, so declare that it is supported even
5896          * though it is treated as global context.  The alternative is
5897          * not failing the single-context invvpid, and it is worse.
5898          */
5899         if (enable_vpid) {
5900                 msrs->secondary_ctls_high |=
5901                         SECONDARY_EXEC_ENABLE_VPID;
5902                 msrs->vpid_caps = VMX_VPID_INVVPID_BIT |
5903                         VMX_VPID_EXTENT_SUPPORTED_MASK;
5904         }
5905 
5906         if (enable_unrestricted_guest)
5907                 msrs->secondary_ctls_high |=
5908                         SECONDARY_EXEC_UNRESTRICTED_GUEST;
5909 
5910         if (flexpriority_enabled)
5911                 msrs->secondary_ctls_high |=
5912                         SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
5913 
5914         /* miscellaneous data */
5915         rdmsr(MSR_IA32_VMX_MISC,
5916                 msrs->misc_low,
5917                 msrs->misc_high);
5918         msrs->misc_low &= VMX_MISC_SAVE_EFER_LMA;
5919         msrs->misc_low |=
5920                 MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS |
5921                 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE |
5922                 VMX_MISC_ACTIVITY_HLT;
5923         msrs->misc_high = 0;
5924 
5925         /*
5926          * This MSR reports some information about VMX support. We
5927          * should return information about the VMX we emulate for the
5928          * guest, and the VMCS structure we give it - not about the
5929          * VMX support of the underlying hardware.
5930          */
5931         msrs->basic =
5932                 VMCS12_REVISION |
5933                 VMX_BASIC_TRUE_CTLS |
5934                 ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) |
5935                 (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT);
5936 
5937         if (cpu_has_vmx_basic_inout())
5938                 msrs->basic |= VMX_BASIC_INOUT;
5939 
5940         /*
5941          * These MSRs specify bits which the guest must keep fixed on
5942          * while L1 is in VMXON mode (in L1's root mode, or running an L2).
5943          * We picked the standard core2 setting.
5944          */
5945 #define VMXON_CR0_ALWAYSON     (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE)
5946 #define VMXON_CR4_ALWAYSON     X86_CR4_VMXE
5947         msrs->cr0_fixed0 = VMXON_CR0_ALWAYSON;
5948         msrs->cr4_fixed0 = VMXON_CR4_ALWAYSON;
5949 
5950         /* These MSRs specify bits which the guest must keep fixed off. */
5951         rdmsrl(MSR_IA32_VMX_CR0_FIXED1, msrs->cr0_fixed1);
5952         rdmsrl(MSR_IA32_VMX_CR4_FIXED1, msrs->cr4_fixed1);
5953 
5954         /* highest index: VMX_PREEMPTION_TIMER_VALUE */
5955         msrs->vmcs_enum = VMCS12_MAX_FIELD_INDEX << 1;
5956 }
5957 
5958 void nested_vmx_hardware_unsetup(void)
5959 {
5960         int i;
5961 
5962         if (enable_shadow_vmcs) {
5963                 for (i = 0; i < VMX_BITMAP_NR; i++)
5964                         free_page((unsigned long)vmx_bitmap[i]);
5965         }
5966 }
5967 
5968 __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *))
5969 {
5970         int i;
5971 
5972         if (!cpu_has_vmx_shadow_vmcs())
5973                 enable_shadow_vmcs = 0;
5974         if (enable_shadow_vmcs) {
5975                 for (i = 0; i < VMX_BITMAP_NR; i++) {
5976                         /*
5977                          * The vmx_bitmap is not tied to a VM and so should
5978                          * not be charged to a memcg.
5979                          */
5980                         vmx_bitmap[i] = (unsigned long *)
5981                                 __get_free_page(GFP_KERNEL);
5982                         if (!vmx_bitmap[i]) {
5983                                 nested_vmx_hardware_unsetup();
5984                                 return -ENOMEM;
5985                         }
5986                 }
5987 
5988                 init_vmcs_shadow_fields();
5989         }
5990 
5991         exit_handlers[EXIT_REASON_VMCLEAR]      = handle_vmclear,
5992         exit_handlers[EXIT_REASON_VMLAUNCH]     = handle_vmlaunch,
5993         exit_handlers[EXIT_REASON_VMPTRLD]      = handle_vmptrld,
5994         exit_handlers[EXIT_REASON_VMPTRST]      = handle_vmptrst,
5995         exit_handlers[EXIT_REASON_VMREAD]       = handle_vmread,
5996         exit_handlers[EXIT_REASON_VMRESUME]     = handle_vmresume,
5997         exit_handlers[EXIT_REASON_VMWRITE]      = handle_vmwrite,
5998         exit_handlers[EXIT_REASON_VMOFF]        = handle_vmoff,
5999         exit_handlers[EXIT_REASON_VMON]         = handle_vmon,
6000         exit_handlers[EXIT_REASON_INVEPT]       = handle_invept,
6001         exit_handlers[EXIT_REASON_INVVPID]      = handle_invvpid,
6002         exit_handlers[EXIT_REASON_VMFUNC]       = handle_vmfunc,
6003 
6004         kvm_x86_ops->check_nested_events = vmx_check_nested_events;
6005         kvm_x86_ops->get_nested_state = vmx_get_nested_state;
6006         kvm_x86_ops->set_nested_state = vmx_set_nested_state;
6007         kvm_x86_ops->get_vmcs12_pages = nested_get_vmcs12_pages,
6008         kvm_x86_ops->nested_enable_evmcs = nested_enable_evmcs;
6009         kvm_x86_ops->nested_get_evmcs_version = nested_get_evmcs_version;
6010 
6011         return 0;
6012 }
6013 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp