~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/kvm/vmx.c

Version: ~ [ linux-5.5-rc1 ] ~ [ linux-5.4.2 ] ~ [ linux-5.3.15 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.88 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.158 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.206 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.206 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.78 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Kernel-based Virtual Machine driver for Linux
  3  *
  4  * This module enables machines with Intel VT-x extensions to run virtual
  5  * machines without emulation or binary translation.
  6  *
  7  * Copyright (C) 2006 Qumranet, Inc.
  8  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  9  *
 10  * Authors:
 11  *   Avi Kivity   <avi@qumranet.com>
 12  *   Yaniv Kamay  <yaniv@qumranet.com>
 13  *
 14  * This work is licensed under the terms of the GNU GPL, version 2.  See
 15  * the COPYING file in the top-level directory.
 16  *
 17  */
 18 
 19 #include "irq.h"
 20 #include "mmu.h"
 21 #include "cpuid.h"
 22 
 23 #include <linux/kvm_host.h>
 24 #include <linux/module.h>
 25 #include <linux/kernel.h>
 26 #include <linux/mm.h>
 27 #include <linux/highmem.h>
 28 #include <linux/sched.h>
 29 #include <linux/moduleparam.h>
 30 #include <linux/mod_devicetable.h>
 31 #include <linux/ftrace_event.h>
 32 #include <linux/slab.h>
 33 #include <linux/tboot.h>
 34 #include "kvm_cache_regs.h"
 35 #include "x86.h"
 36 
 37 #include <asm/io.h>
 38 #include <asm/desc.h>
 39 #include <asm/vmx.h>
 40 #include <asm/virtext.h>
 41 #include <asm/mce.h>
 42 #include <asm/i387.h>
 43 #include <asm/xcr.h>
 44 #include <asm/perf_event.h>
 45 #include <asm/kexec.h>
 46 
 47 #include "trace.h"
 48 
 49 #define __ex(x) __kvm_handle_fault_on_reboot(x)
 50 #define __ex_clear(x, reg) \
 51         ____kvm_handle_fault_on_reboot(x, "xor " reg " , " reg)
 52 
 53 MODULE_AUTHOR("Qumranet");
 54 MODULE_LICENSE("GPL");
 55 
 56 static const struct x86_cpu_id vmx_cpu_id[] = {
 57         X86_FEATURE_MATCH(X86_FEATURE_VMX),
 58         {}
 59 };
 60 MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id);
 61 
 62 static bool __read_mostly enable_vpid = 1;
 63 module_param_named(vpid, enable_vpid, bool, 0444);
 64 
 65 static bool __read_mostly flexpriority_enabled = 1;
 66 module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
 67 
 68 static bool __read_mostly enable_ept = 1;
 69 module_param_named(ept, enable_ept, bool, S_IRUGO);
 70 
 71 static bool __read_mostly enable_unrestricted_guest = 1;
 72 module_param_named(unrestricted_guest,
 73                         enable_unrestricted_guest, bool, S_IRUGO);
 74 
 75 static bool __read_mostly enable_ept_ad_bits = 1;
 76 module_param_named(eptad, enable_ept_ad_bits, bool, S_IRUGO);
 77 
 78 static bool __read_mostly emulate_invalid_guest_state = true;
 79 module_param(emulate_invalid_guest_state, bool, S_IRUGO);
 80 
 81 static bool __read_mostly vmm_exclusive = 1;
 82 module_param(vmm_exclusive, bool, S_IRUGO);
 83 
 84 static bool __read_mostly fasteoi = 1;
 85 module_param(fasteoi, bool, S_IRUGO);
 86 
 87 static bool __read_mostly enable_apicv = 1;
 88 module_param(enable_apicv, bool, S_IRUGO);
 89 
 90 static bool __read_mostly enable_shadow_vmcs = 1;
 91 module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO);
 92 /*
 93  * If nested=1, nested virtualization is supported, i.e., guests may use
 94  * VMX and be a hypervisor for its own guests. If nested=0, guests may not
 95  * use VMX instructions.
 96  */
 97 static bool __read_mostly nested = 0;
 98 module_param(nested, bool, S_IRUGO);
 99 
100 #define KVM_GUEST_CR0_MASK (X86_CR0_NW | X86_CR0_CD)
101 #define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST (X86_CR0_WP | X86_CR0_NE)
102 #define KVM_VM_CR0_ALWAYS_ON                                            \
103         (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
104 #define KVM_CR4_GUEST_OWNED_BITS                                      \
105         (X86_CR4_PVI | X86_CR4_DE | X86_CR4_PCE | X86_CR4_OSFXSR      \
106          | X86_CR4_OSXMMEXCPT)
107 
108 #define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE)
109 #define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE)
110 
111 #define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM))
112 
113 /*
114  * These 2 parameters are used to config the controls for Pause-Loop Exiting:
115  * ple_gap:    upper bound on the amount of time between two successive
116  *             executions of PAUSE in a loop. Also indicate if ple enabled.
117  *             According to test, this time is usually smaller than 128 cycles.
118  * ple_window: upper bound on the amount of time a guest is allowed to execute
119  *             in a PAUSE loop. Tests indicate that most spinlocks are held for
120  *             less than 2^12 cycles
121  * Time is measured based on a counter that runs at the same rate as the TSC,
122  * refer SDM volume 3b section 21.6.13 & 22.1.3.
123  */
124 #define KVM_VMX_DEFAULT_PLE_GAP    128
125 #define KVM_VMX_DEFAULT_PLE_WINDOW 4096
126 static int ple_gap = KVM_VMX_DEFAULT_PLE_GAP;
127 module_param(ple_gap, int, S_IRUGO);
128 
129 static int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW;
130 module_param(ple_window, int, S_IRUGO);
131 
132 extern const ulong vmx_return;
133 
134 #define NR_AUTOLOAD_MSRS 8
135 #define VMCS02_POOL_SIZE 1
136 
137 struct vmcs {
138         u32 revision_id;
139         u32 abort;
140         char data[0];
141 };
142 
143 /*
144  * Track a VMCS that may be loaded on a certain CPU. If it is (cpu!=-1), also
145  * remember whether it was VMLAUNCHed, and maintain a linked list of all VMCSs
146  * loaded on this CPU (so we can clear them if the CPU goes down).
147  */
148 struct loaded_vmcs {
149         struct vmcs *vmcs;
150         int cpu;
151         int launched;
152         struct list_head loaded_vmcss_on_cpu_link;
153 };
154 
155 struct shared_msr_entry {
156         unsigned index;
157         u64 data;
158         u64 mask;
159 };
160 
161 /*
162  * struct vmcs12 describes the state that our guest hypervisor (L1) keeps for a
163  * single nested guest (L2), hence the name vmcs12. Any VMX implementation has
164  * a VMCS structure, and vmcs12 is our emulated VMX's VMCS. This structure is
165  * stored in guest memory specified by VMPTRLD, but is opaque to the guest,
166  * which must access it using VMREAD/VMWRITE/VMCLEAR instructions.
167  * More than one of these structures may exist, if L1 runs multiple L2 guests.
168  * nested_vmx_run() will use the data here to build a vmcs02: a VMCS for the
169  * underlying hardware which will be used to run L2.
170  * This structure is packed to ensure that its layout is identical across
171  * machines (necessary for live migration).
172  * If there are changes in this struct, VMCS12_REVISION must be changed.
173  */
174 typedef u64 natural_width;
175 struct __packed vmcs12 {
176         /* According to the Intel spec, a VMCS region must start with the
177          * following two fields. Then follow implementation-specific data.
178          */
179         u32 revision_id;
180         u32 abort;
181 
182         u32 launch_state; /* set to 0 by VMCLEAR, to 1 by VMLAUNCH */
183         u32 padding[7]; /* room for future expansion */
184 
185         u64 io_bitmap_a;
186         u64 io_bitmap_b;
187         u64 msr_bitmap;
188         u64 vm_exit_msr_store_addr;
189         u64 vm_exit_msr_load_addr;
190         u64 vm_entry_msr_load_addr;
191         u64 tsc_offset;
192         u64 virtual_apic_page_addr;
193         u64 apic_access_addr;
194         u64 ept_pointer;
195         u64 guest_physical_address;
196         u64 vmcs_link_pointer;
197         u64 guest_ia32_debugctl;
198         u64 guest_ia32_pat;
199         u64 guest_ia32_efer;
200         u64 guest_ia32_perf_global_ctrl;
201         u64 guest_pdptr0;
202         u64 guest_pdptr1;
203         u64 guest_pdptr2;
204         u64 guest_pdptr3;
205         u64 host_ia32_pat;
206         u64 host_ia32_efer;
207         u64 host_ia32_perf_global_ctrl;
208         u64 padding64[8]; /* room for future expansion */
209         /*
210          * To allow migration of L1 (complete with its L2 guests) between
211          * machines of different natural widths (32 or 64 bit), we cannot have
212          * unsigned long fields with no explict size. We use u64 (aliased
213          * natural_width) instead. Luckily, x86 is little-endian.
214          */
215         natural_width cr0_guest_host_mask;
216         natural_width cr4_guest_host_mask;
217         natural_width cr0_read_shadow;
218         natural_width cr4_read_shadow;
219         natural_width cr3_target_value0;
220         natural_width cr3_target_value1;
221         natural_width cr3_target_value2;
222         natural_width cr3_target_value3;
223         natural_width exit_qualification;
224         natural_width guest_linear_address;
225         natural_width guest_cr0;
226         natural_width guest_cr3;
227         natural_width guest_cr4;
228         natural_width guest_es_base;
229         natural_width guest_cs_base;
230         natural_width guest_ss_base;
231         natural_width guest_ds_base;
232         natural_width guest_fs_base;
233         natural_width guest_gs_base;
234         natural_width guest_ldtr_base;
235         natural_width guest_tr_base;
236         natural_width guest_gdtr_base;
237         natural_width guest_idtr_base;
238         natural_width guest_dr7;
239         natural_width guest_rsp;
240         natural_width guest_rip;
241         natural_width guest_rflags;
242         natural_width guest_pending_dbg_exceptions;
243         natural_width guest_sysenter_esp;
244         natural_width guest_sysenter_eip;
245         natural_width host_cr0;
246         natural_width host_cr3;
247         natural_width host_cr4;
248         natural_width host_fs_base;
249         natural_width host_gs_base;
250         natural_width host_tr_base;
251         natural_width host_gdtr_base;
252         natural_width host_idtr_base;
253         natural_width host_ia32_sysenter_esp;
254         natural_width host_ia32_sysenter_eip;
255         natural_width host_rsp;
256         natural_width host_rip;
257         natural_width paddingl[8]; /* room for future expansion */
258         u32 pin_based_vm_exec_control;
259         u32 cpu_based_vm_exec_control;
260         u32 exception_bitmap;
261         u32 page_fault_error_code_mask;
262         u32 page_fault_error_code_match;
263         u32 cr3_target_count;
264         u32 vm_exit_controls;
265         u32 vm_exit_msr_store_count;
266         u32 vm_exit_msr_load_count;
267         u32 vm_entry_controls;
268         u32 vm_entry_msr_load_count;
269         u32 vm_entry_intr_info_field;
270         u32 vm_entry_exception_error_code;
271         u32 vm_entry_instruction_len;
272         u32 tpr_threshold;
273         u32 secondary_vm_exec_control;
274         u32 vm_instruction_error;
275         u32 vm_exit_reason;
276         u32 vm_exit_intr_info;
277         u32 vm_exit_intr_error_code;
278         u32 idt_vectoring_info_field;
279         u32 idt_vectoring_error_code;
280         u32 vm_exit_instruction_len;
281         u32 vmx_instruction_info;
282         u32 guest_es_limit;
283         u32 guest_cs_limit;
284         u32 guest_ss_limit;
285         u32 guest_ds_limit;
286         u32 guest_fs_limit;
287         u32 guest_gs_limit;
288         u32 guest_ldtr_limit;
289         u32 guest_tr_limit;
290         u32 guest_gdtr_limit;
291         u32 guest_idtr_limit;
292         u32 guest_es_ar_bytes;
293         u32 guest_cs_ar_bytes;
294         u32 guest_ss_ar_bytes;
295         u32 guest_ds_ar_bytes;
296         u32 guest_fs_ar_bytes;
297         u32 guest_gs_ar_bytes;
298         u32 guest_ldtr_ar_bytes;
299         u32 guest_tr_ar_bytes;
300         u32 guest_interruptibility_info;
301         u32 guest_activity_state;
302         u32 guest_sysenter_cs;
303         u32 host_ia32_sysenter_cs;
304         u32 vmx_preemption_timer_value;
305         u32 padding32[7]; /* room for future expansion */
306         u16 virtual_processor_id;
307         u16 guest_es_selector;
308         u16 guest_cs_selector;
309         u16 guest_ss_selector;
310         u16 guest_ds_selector;
311         u16 guest_fs_selector;
312         u16 guest_gs_selector;
313         u16 guest_ldtr_selector;
314         u16 guest_tr_selector;
315         u16 host_es_selector;
316         u16 host_cs_selector;
317         u16 host_ss_selector;
318         u16 host_ds_selector;
319         u16 host_fs_selector;
320         u16 host_gs_selector;
321         u16 host_tr_selector;
322 };
323 
324 /*
325  * VMCS12_REVISION is an arbitrary id that should be changed if the content or
326  * layout of struct vmcs12 is changed. MSR_IA32_VMX_BASIC returns this id, and
327  * VMPTRLD verifies that the VMCS region that L1 is loading contains this id.
328  */
329 #define VMCS12_REVISION 0x11e57ed0
330 
331 /*
332  * VMCS12_SIZE is the number of bytes L1 should allocate for the VMXON region
333  * and any VMCS region. Although only sizeof(struct vmcs12) are used by the
334  * current implementation, 4K are reserved to avoid future complications.
335  */
336 #define VMCS12_SIZE 0x1000
337 
338 /* Used to remember the last vmcs02 used for some recently used vmcs12s */
339 struct vmcs02_list {
340         struct list_head list;
341         gpa_t vmptr;
342         struct loaded_vmcs vmcs02;
343 };
344 
345 /*
346  * The nested_vmx structure is part of vcpu_vmx, and holds information we need
347  * for correct emulation of VMX (i.e., nested VMX) on this vcpu.
348  */
349 struct nested_vmx {
350         /* Has the level1 guest done vmxon? */
351         bool vmxon;
352 
353         /* The guest-physical address of the current VMCS L1 keeps for L2 */
354         gpa_t current_vmptr;
355         /* The host-usable pointer to the above */
356         struct page *current_vmcs12_page;
357         struct vmcs12 *current_vmcs12;
358         struct vmcs *current_shadow_vmcs;
359         /*
360          * Indicates if the shadow vmcs must be updated with the
361          * data hold by vmcs12
362          */
363         bool sync_shadow_vmcs;
364 
365         /* vmcs02_list cache of VMCSs recently used to run L2 guests */
366         struct list_head vmcs02_pool;
367         int vmcs02_num;
368         u64 vmcs01_tsc_offset;
369         bool change_vmcs01_virtual_x2apic_mode;
370         /* L2 must run next, and mustn't decide to exit to L1. */
371         bool nested_run_pending;
372         /*
373          * Guest pages referred to in vmcs02 with host-physical pointers, so
374          * we must keep them pinned while L2 runs.
375          */
376         struct page *apic_access_page;
377         u64 msr_ia32_feature_control;
378 };
379 
380 #define POSTED_INTR_ON  0
381 /* Posted-Interrupt Descriptor */
382 struct pi_desc {
383         u32 pir[8];     /* Posted interrupt requested */
384         u32 control;    /* bit 0 of control is outstanding notification bit */
385         u32 rsvd[7];
386 } __aligned(64);
387 
388 static bool pi_test_and_set_on(struct pi_desc *pi_desc)
389 {
390         return test_and_set_bit(POSTED_INTR_ON,
391                         (unsigned long *)&pi_desc->control);
392 }
393 
394 static bool pi_test_and_clear_on(struct pi_desc *pi_desc)
395 {
396         return test_and_clear_bit(POSTED_INTR_ON,
397                         (unsigned long *)&pi_desc->control);
398 }
399 
400 static int pi_test_and_set_pir(int vector, struct pi_desc *pi_desc)
401 {
402         return test_and_set_bit(vector, (unsigned long *)pi_desc->pir);
403 }
404 
405 struct vcpu_vmx {
406         struct kvm_vcpu       vcpu;
407         unsigned long         host_rsp;
408         u8                    fail;
409         u8                    cpl;
410         bool                  nmi_known_unmasked;
411         u32                   exit_intr_info;
412         u32                   idt_vectoring_info;
413         ulong                 rflags;
414         struct shared_msr_entry *guest_msrs;
415         int                   nmsrs;
416         int                   save_nmsrs;
417         unsigned long         host_idt_base;
418 #ifdef CONFIG_X86_64
419         u64                   msr_host_kernel_gs_base;
420         u64                   msr_guest_kernel_gs_base;
421 #endif
422         /*
423          * loaded_vmcs points to the VMCS currently used in this vcpu. For a
424          * non-nested (L1) guest, it always points to vmcs01. For a nested
425          * guest (L2), it points to a different VMCS.
426          */
427         struct loaded_vmcs    vmcs01;
428         struct loaded_vmcs   *loaded_vmcs;
429         bool                  __launched; /* temporary, used in vmx_vcpu_run */
430         struct msr_autoload {
431                 unsigned nr;
432                 struct vmx_msr_entry guest[NR_AUTOLOAD_MSRS];
433                 struct vmx_msr_entry host[NR_AUTOLOAD_MSRS];
434         } msr_autoload;
435         struct {
436                 int           loaded;
437                 u16           fs_sel, gs_sel, ldt_sel;
438 #ifdef CONFIG_X86_64
439                 u16           ds_sel, es_sel;
440 #endif
441                 int           gs_ldt_reload_needed;
442                 int           fs_reload_needed;
443                 unsigned long vmcs_host_cr4;    /* May not match real cr4 */
444         } host_state;
445         struct {
446                 int vm86_active;
447                 ulong save_rflags;
448                 struct kvm_segment segs[8];
449         } rmode;
450         struct {
451                 u32 bitmask; /* 4 bits per segment (1 bit per field) */
452                 struct kvm_save_segment {
453                         u16 selector;
454                         unsigned long base;
455                         u32 limit;
456                         u32 ar;
457                 } seg[8];
458         } segment_cache;
459         int vpid;
460         bool emulation_required;
461 
462         /* Support for vnmi-less CPUs */
463         int soft_vnmi_blocked;
464         ktime_t entry_time;
465         s64 vnmi_blocked_time;
466         u32 exit_reason;
467 
468         bool rdtscp_enabled;
469 
470         /* Posted interrupt descriptor */
471         struct pi_desc pi_desc;
472 
473         /* Support for a guest hypervisor (nested VMX) */
474         struct nested_vmx nested;
475 };
476 
477 enum segment_cache_field {
478         SEG_FIELD_SEL = 0,
479         SEG_FIELD_BASE = 1,
480         SEG_FIELD_LIMIT = 2,
481         SEG_FIELD_AR = 3,
482 
483         SEG_FIELD_NR = 4
484 };
485 
486 static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
487 {
488         return container_of(vcpu, struct vcpu_vmx, vcpu);
489 }
490 
491 #define VMCS12_OFFSET(x) offsetof(struct vmcs12, x)
492 #define FIELD(number, name)     [number] = VMCS12_OFFSET(name)
493 #define FIELD64(number, name)   [number] = VMCS12_OFFSET(name), \
494                                 [number##_HIGH] = VMCS12_OFFSET(name)+4
495 
496 
497 static const unsigned long shadow_read_only_fields[] = {
498         /*
499          * We do NOT shadow fields that are modified when L0
500          * traps and emulates any vmx instruction (e.g. VMPTRLD,
501          * VMXON...) executed by L1.
502          * For example, VM_INSTRUCTION_ERROR is read
503          * by L1 if a vmx instruction fails (part of the error path).
504          * Note the code assumes this logic. If for some reason
505          * we start shadowing these fields then we need to
506          * force a shadow sync when L0 emulates vmx instructions
507          * (e.g. force a sync if VM_INSTRUCTION_ERROR is modified
508          * by nested_vmx_failValid)
509          */
510         VM_EXIT_REASON,
511         VM_EXIT_INTR_INFO,
512         VM_EXIT_INSTRUCTION_LEN,
513         IDT_VECTORING_INFO_FIELD,
514         IDT_VECTORING_ERROR_CODE,
515         VM_EXIT_INTR_ERROR_CODE,
516         EXIT_QUALIFICATION,
517         GUEST_LINEAR_ADDRESS,
518         GUEST_PHYSICAL_ADDRESS
519 };
520 static const int max_shadow_read_only_fields =
521         ARRAY_SIZE(shadow_read_only_fields);
522 
523 static const unsigned long shadow_read_write_fields[] = {
524         GUEST_RIP,
525         GUEST_RSP,
526         GUEST_CR0,
527         GUEST_CR3,
528         GUEST_CR4,
529         GUEST_INTERRUPTIBILITY_INFO,
530         GUEST_RFLAGS,
531         GUEST_CS_SELECTOR,
532         GUEST_CS_AR_BYTES,
533         GUEST_CS_LIMIT,
534         GUEST_CS_BASE,
535         GUEST_ES_BASE,
536         CR0_GUEST_HOST_MASK,
537         CR0_READ_SHADOW,
538         CR4_READ_SHADOW,
539         TSC_OFFSET,
540         EXCEPTION_BITMAP,
541         CPU_BASED_VM_EXEC_CONTROL,
542         VM_ENTRY_EXCEPTION_ERROR_CODE,
543         VM_ENTRY_INTR_INFO_FIELD,
544         VM_ENTRY_INSTRUCTION_LEN,
545         VM_ENTRY_EXCEPTION_ERROR_CODE,
546         HOST_FS_BASE,
547         HOST_GS_BASE,
548         HOST_FS_SELECTOR,
549         HOST_GS_SELECTOR
550 };
551 static const int max_shadow_read_write_fields =
552         ARRAY_SIZE(shadow_read_write_fields);
553 
554 static const unsigned short vmcs_field_to_offset_table[] = {
555         FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id),
556         FIELD(GUEST_ES_SELECTOR, guest_es_selector),
557         FIELD(GUEST_CS_SELECTOR, guest_cs_selector),
558         FIELD(GUEST_SS_SELECTOR, guest_ss_selector),
559         FIELD(GUEST_DS_SELECTOR, guest_ds_selector),
560         FIELD(GUEST_FS_SELECTOR, guest_fs_selector),
561         FIELD(GUEST_GS_SELECTOR, guest_gs_selector),
562         FIELD(GUEST_LDTR_SELECTOR, guest_ldtr_selector),
563         FIELD(GUEST_TR_SELECTOR, guest_tr_selector),
564         FIELD(HOST_ES_SELECTOR, host_es_selector),
565         FIELD(HOST_CS_SELECTOR, host_cs_selector),
566         FIELD(HOST_SS_SELECTOR, host_ss_selector),
567         FIELD(HOST_DS_SELECTOR, host_ds_selector),
568         FIELD(HOST_FS_SELECTOR, host_fs_selector),
569         FIELD(HOST_GS_SELECTOR, host_gs_selector),
570         FIELD(HOST_TR_SELECTOR, host_tr_selector),
571         FIELD64(IO_BITMAP_A, io_bitmap_a),
572         FIELD64(IO_BITMAP_B, io_bitmap_b),
573         FIELD64(MSR_BITMAP, msr_bitmap),
574         FIELD64(VM_EXIT_MSR_STORE_ADDR, vm_exit_msr_store_addr),
575         FIELD64(VM_EXIT_MSR_LOAD_ADDR, vm_exit_msr_load_addr),
576         FIELD64(VM_ENTRY_MSR_LOAD_ADDR, vm_entry_msr_load_addr),
577         FIELD64(TSC_OFFSET, tsc_offset),
578         FIELD64(VIRTUAL_APIC_PAGE_ADDR, virtual_apic_page_addr),
579         FIELD64(APIC_ACCESS_ADDR, apic_access_addr),
580         FIELD64(EPT_POINTER, ept_pointer),
581         FIELD64(GUEST_PHYSICAL_ADDRESS, guest_physical_address),
582         FIELD64(VMCS_LINK_POINTER, vmcs_link_pointer),
583         FIELD64(GUEST_IA32_DEBUGCTL, guest_ia32_debugctl),
584         FIELD64(GUEST_IA32_PAT, guest_ia32_pat),
585         FIELD64(GUEST_IA32_EFER, guest_ia32_efer),
586         FIELD64(GUEST_IA32_PERF_GLOBAL_CTRL, guest_ia32_perf_global_ctrl),
587         FIELD64(GUEST_PDPTR0, guest_pdptr0),
588         FIELD64(GUEST_PDPTR1, guest_pdptr1),
589         FIELD64(GUEST_PDPTR2, guest_pdptr2),
590         FIELD64(GUEST_PDPTR3, guest_pdptr3),
591         FIELD64(HOST_IA32_PAT, host_ia32_pat),
592         FIELD64(HOST_IA32_EFER, host_ia32_efer),
593         FIELD64(HOST_IA32_PERF_GLOBAL_CTRL, host_ia32_perf_global_ctrl),
594         FIELD(PIN_BASED_VM_EXEC_CONTROL, pin_based_vm_exec_control),
595         FIELD(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control),
596         FIELD(EXCEPTION_BITMAP, exception_bitmap),
597         FIELD(PAGE_FAULT_ERROR_CODE_MASK, page_fault_error_code_mask),
598         FIELD(PAGE_FAULT_ERROR_CODE_MATCH, page_fault_error_code_match),
599         FIELD(CR3_TARGET_COUNT, cr3_target_count),
600         FIELD(VM_EXIT_CONTROLS, vm_exit_controls),
601         FIELD(VM_EXIT_MSR_STORE_COUNT, vm_exit_msr_store_count),
602         FIELD(VM_EXIT_MSR_LOAD_COUNT, vm_exit_msr_load_count),
603         FIELD(VM_ENTRY_CONTROLS, vm_entry_controls),
604         FIELD(VM_ENTRY_MSR_LOAD_COUNT, vm_entry_msr_load_count),
605         FIELD(VM_ENTRY_INTR_INFO_FIELD, vm_entry_intr_info_field),
606         FIELD(VM_ENTRY_EXCEPTION_ERROR_CODE, vm_entry_exception_error_code),
607         FIELD(VM_ENTRY_INSTRUCTION_LEN, vm_entry_instruction_len),
608         FIELD(TPR_THRESHOLD, tpr_threshold),
609         FIELD(SECONDARY_VM_EXEC_CONTROL, secondary_vm_exec_control),
610         FIELD(VM_INSTRUCTION_ERROR, vm_instruction_error),
611         FIELD(VM_EXIT_REASON, vm_exit_reason),
612         FIELD(VM_EXIT_INTR_INFO, vm_exit_intr_info),
613         FIELD(VM_EXIT_INTR_ERROR_CODE, vm_exit_intr_error_code),
614         FIELD(IDT_VECTORING_INFO_FIELD, idt_vectoring_info_field),
615         FIELD(IDT_VECTORING_ERROR_CODE, idt_vectoring_error_code),
616         FIELD(VM_EXIT_INSTRUCTION_LEN, vm_exit_instruction_len),
617         FIELD(VMX_INSTRUCTION_INFO, vmx_instruction_info),
618         FIELD(GUEST_ES_LIMIT, guest_es_limit),
619         FIELD(GUEST_CS_LIMIT, guest_cs_limit),
620         FIELD(GUEST_SS_LIMIT, guest_ss_limit),
621         FIELD(GUEST_DS_LIMIT, guest_ds_limit),
622         FIELD(GUEST_FS_LIMIT, guest_fs_limit),
623         FIELD(GUEST_GS_LIMIT, guest_gs_limit),
624         FIELD(GUEST_LDTR_LIMIT, guest_ldtr_limit),
625         FIELD(GUEST_TR_LIMIT, guest_tr_limit),
626         FIELD(GUEST_GDTR_LIMIT, guest_gdtr_limit),
627         FIELD(GUEST_IDTR_LIMIT, guest_idtr_limit),
628         FIELD(GUEST_ES_AR_BYTES, guest_es_ar_bytes),
629         FIELD(GUEST_CS_AR_BYTES, guest_cs_ar_bytes),
630         FIELD(GUEST_SS_AR_BYTES, guest_ss_ar_bytes),
631         FIELD(GUEST_DS_AR_BYTES, guest_ds_ar_bytes),
632         FIELD(GUEST_FS_AR_BYTES, guest_fs_ar_bytes),
633         FIELD(GUEST_GS_AR_BYTES, guest_gs_ar_bytes),
634         FIELD(GUEST_LDTR_AR_BYTES, guest_ldtr_ar_bytes),
635         FIELD(GUEST_TR_AR_BYTES, guest_tr_ar_bytes),
636         FIELD(GUEST_INTERRUPTIBILITY_INFO, guest_interruptibility_info),
637         FIELD(GUEST_ACTIVITY_STATE, guest_activity_state),
638         FIELD(GUEST_SYSENTER_CS, guest_sysenter_cs),
639         FIELD(HOST_IA32_SYSENTER_CS, host_ia32_sysenter_cs),
640         FIELD(VMX_PREEMPTION_TIMER_VALUE, vmx_preemption_timer_value),
641         FIELD(CR0_GUEST_HOST_MASK, cr0_guest_host_mask),
642         FIELD(CR4_GUEST_HOST_MASK, cr4_guest_host_mask),
643         FIELD(CR0_READ_SHADOW, cr0_read_shadow),
644         FIELD(CR4_READ_SHADOW, cr4_read_shadow),
645         FIELD(CR3_TARGET_VALUE0, cr3_target_value0),
646         FIELD(CR3_TARGET_VALUE1, cr3_target_value1),
647         FIELD(CR3_TARGET_VALUE2, cr3_target_value2),
648         FIELD(CR3_TARGET_VALUE3, cr3_target_value3),
649         FIELD(EXIT_QUALIFICATION, exit_qualification),
650         FIELD(GUEST_LINEAR_ADDRESS, guest_linear_address),
651         FIELD(GUEST_CR0, guest_cr0),
652         FIELD(GUEST_CR3, guest_cr3),
653         FIELD(GUEST_CR4, guest_cr4),
654         FIELD(GUEST_ES_BASE, guest_es_base),
655         FIELD(GUEST_CS_BASE, guest_cs_base),
656         FIELD(GUEST_SS_BASE, guest_ss_base),
657         FIELD(GUEST_DS_BASE, guest_ds_base),
658         FIELD(GUEST_FS_BASE, guest_fs_base),
659         FIELD(GUEST_GS_BASE, guest_gs_base),
660         FIELD(GUEST_LDTR_BASE, guest_ldtr_base),
661         FIELD(GUEST_TR_BASE, guest_tr_base),
662         FIELD(GUEST_GDTR_BASE, guest_gdtr_base),
663         FIELD(GUEST_IDTR_BASE, guest_idtr_base),
664         FIELD(GUEST_DR7, guest_dr7),
665         FIELD(GUEST_RSP, guest_rsp),
666         FIELD(GUEST_RIP, guest_rip),
667         FIELD(GUEST_RFLAGS, guest_rflags),
668         FIELD(GUEST_PENDING_DBG_EXCEPTIONS, guest_pending_dbg_exceptions),
669         FIELD(GUEST_SYSENTER_ESP, guest_sysenter_esp),
670         FIELD(GUEST_SYSENTER_EIP, guest_sysenter_eip),
671         FIELD(HOST_CR0, host_cr0),
672         FIELD(HOST_CR3, host_cr3),
673         FIELD(HOST_CR4, host_cr4),
674         FIELD(HOST_FS_BASE, host_fs_base),
675         FIELD(HOST_GS_BASE, host_gs_base),
676         FIELD(HOST_TR_BASE, host_tr_base),
677         FIELD(HOST_GDTR_BASE, host_gdtr_base),
678         FIELD(HOST_IDTR_BASE, host_idtr_base),
679         FIELD(HOST_IA32_SYSENTER_ESP, host_ia32_sysenter_esp),
680         FIELD(HOST_IA32_SYSENTER_EIP, host_ia32_sysenter_eip),
681         FIELD(HOST_RSP, host_rsp),
682         FIELD(HOST_RIP, host_rip),
683 };
684 static const int max_vmcs_field = ARRAY_SIZE(vmcs_field_to_offset_table);
685 
686 static inline short vmcs_field_to_offset(unsigned long field)
687 {
688         if (field >= max_vmcs_field || vmcs_field_to_offset_table[field] == 0)
689                 return -1;
690         return vmcs_field_to_offset_table[field];
691 }
692 
693 static inline struct vmcs12 *get_vmcs12(struct kvm_vcpu *vcpu)
694 {
695         return to_vmx(vcpu)->nested.current_vmcs12;
696 }
697 
698 static struct page *nested_get_page(struct kvm_vcpu *vcpu, gpa_t addr)
699 {
700         struct page *page = gfn_to_page(vcpu->kvm, addr >> PAGE_SHIFT);
701         if (is_error_page(page))
702                 return NULL;
703 
704         return page;
705 }
706 
707 static void nested_release_page(struct page *page)
708 {
709         kvm_release_page_dirty(page);
710 }
711 
712 static void nested_release_page_clean(struct page *page)
713 {
714         kvm_release_page_clean(page);
715 }
716 
717 static unsigned long nested_ept_get_cr3(struct kvm_vcpu *vcpu);
718 static u64 construct_eptp(unsigned long root_hpa);
719 static void kvm_cpu_vmxon(u64 addr);
720 static void kvm_cpu_vmxoff(void);
721 static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr);
722 static void vmx_set_segment(struct kvm_vcpu *vcpu,
723                             struct kvm_segment *var, int seg);
724 static void vmx_get_segment(struct kvm_vcpu *vcpu,
725                             struct kvm_segment *var, int seg);
726 static bool guest_state_valid(struct kvm_vcpu *vcpu);
727 static u32 vmx_segment_access_rights(struct kvm_segment *var);
728 static void vmx_sync_pir_to_irr_dummy(struct kvm_vcpu *vcpu);
729 static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx);
730 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx);
731 
732 static DEFINE_PER_CPU(struct vmcs *, vmxarea);
733 static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
734 /*
735  * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed
736  * when a CPU is brought down, and we need to VMCLEAR all VMCSs loaded on it.
737  */
738 static DEFINE_PER_CPU(struct list_head, loaded_vmcss_on_cpu);
739 static DEFINE_PER_CPU(struct desc_ptr, host_gdt);
740 
741 static unsigned long *vmx_io_bitmap_a;
742 static unsigned long *vmx_io_bitmap_b;
743 static unsigned long *vmx_msr_bitmap_legacy;
744 static unsigned long *vmx_msr_bitmap_longmode;
745 static unsigned long *vmx_msr_bitmap_legacy_x2apic;
746 static unsigned long *vmx_msr_bitmap_longmode_x2apic;
747 static unsigned long *vmx_vmread_bitmap;
748 static unsigned long *vmx_vmwrite_bitmap;
749 
750 static bool cpu_has_load_ia32_efer;
751 static bool cpu_has_load_perf_global_ctrl;
752 
753 static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
754 static DEFINE_SPINLOCK(vmx_vpid_lock);
755 
756 static struct vmcs_config {
757         int size;
758         int order;
759         u32 revision_id;
760         u32 pin_based_exec_ctrl;
761         u32 cpu_based_exec_ctrl;
762         u32 cpu_based_2nd_exec_ctrl;
763         u32 vmexit_ctrl;
764         u32 vmentry_ctrl;
765 } vmcs_config;
766 
767 static struct vmx_capability {
768         u32 ept;
769         u32 vpid;
770 } vmx_capability;
771 
772 #define VMX_SEGMENT_FIELD(seg)                                  \
773         [VCPU_SREG_##seg] = {                                   \
774                 .selector = GUEST_##seg##_SELECTOR,             \
775                 .base = GUEST_##seg##_BASE,                     \
776                 .limit = GUEST_##seg##_LIMIT,                   \
777                 .ar_bytes = GUEST_##seg##_AR_BYTES,             \
778         }
779 
780 static const struct kvm_vmx_segment_field {
781         unsigned selector;
782         unsigned base;
783         unsigned limit;
784         unsigned ar_bytes;
785 } kvm_vmx_segment_fields[] = {
786         VMX_SEGMENT_FIELD(CS),
787         VMX_SEGMENT_FIELD(DS),
788         VMX_SEGMENT_FIELD(ES),
789         VMX_SEGMENT_FIELD(FS),
790         VMX_SEGMENT_FIELD(GS),
791         VMX_SEGMENT_FIELD(SS),
792         VMX_SEGMENT_FIELD(TR),
793         VMX_SEGMENT_FIELD(LDTR),
794 };
795 
796 static u64 host_efer;
797 
798 static void ept_save_pdptrs(struct kvm_vcpu *vcpu);
799 
800 /*
801  * Keep MSR_STAR at the end, as setup_msrs() will try to optimize it
802  * away by decrementing the array size.
803  */
804 static const u32 vmx_msr_index[] = {
805 #ifdef CONFIG_X86_64
806         MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR,
807 #endif
808         MSR_EFER, MSR_TSC_AUX, MSR_STAR,
809 };
810 #define NR_VMX_MSR ARRAY_SIZE(vmx_msr_index)
811 
812 static inline bool is_page_fault(u32 intr_info)
813 {
814         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
815                              INTR_INFO_VALID_MASK)) ==
816                 (INTR_TYPE_HARD_EXCEPTION | PF_VECTOR | INTR_INFO_VALID_MASK);
817 }
818 
819 static inline bool is_no_device(u32 intr_info)
820 {
821         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
822                              INTR_INFO_VALID_MASK)) ==
823                 (INTR_TYPE_HARD_EXCEPTION | NM_VECTOR | INTR_INFO_VALID_MASK);
824 }
825 
826 static inline bool is_invalid_opcode(u32 intr_info)
827 {
828         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
829                              INTR_INFO_VALID_MASK)) ==
830                 (INTR_TYPE_HARD_EXCEPTION | UD_VECTOR | INTR_INFO_VALID_MASK);
831 }
832 
833 static inline bool is_external_interrupt(u32 intr_info)
834 {
835         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
836                 == (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
837 }
838 
839 static inline bool is_machine_check(u32 intr_info)
840 {
841         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
842                              INTR_INFO_VALID_MASK)) ==
843                 (INTR_TYPE_HARD_EXCEPTION | MC_VECTOR | INTR_INFO_VALID_MASK);
844 }
845 
846 static inline bool cpu_has_vmx_msr_bitmap(void)
847 {
848         return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_USE_MSR_BITMAPS;
849 }
850 
851 static inline bool cpu_has_vmx_tpr_shadow(void)
852 {
853         return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW;
854 }
855 
856 static inline bool vm_need_tpr_shadow(struct kvm *kvm)
857 {
858         return (cpu_has_vmx_tpr_shadow()) && (irqchip_in_kernel(kvm));
859 }
860 
861 static inline bool cpu_has_secondary_exec_ctrls(void)
862 {
863         return vmcs_config.cpu_based_exec_ctrl &
864                 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
865 }
866 
867 static inline bool cpu_has_vmx_virtualize_apic_accesses(void)
868 {
869         return vmcs_config.cpu_based_2nd_exec_ctrl &
870                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
871 }
872 
873 static inline bool cpu_has_vmx_virtualize_x2apic_mode(void)
874 {
875         return vmcs_config.cpu_based_2nd_exec_ctrl &
876                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
877 }
878 
879 static inline bool cpu_has_vmx_apic_register_virt(void)
880 {
881         return vmcs_config.cpu_based_2nd_exec_ctrl &
882                 SECONDARY_EXEC_APIC_REGISTER_VIRT;
883 }
884 
885 static inline bool cpu_has_vmx_virtual_intr_delivery(void)
886 {
887         return vmcs_config.cpu_based_2nd_exec_ctrl &
888                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY;
889 }
890 
891 static inline bool cpu_has_vmx_posted_intr(void)
892 {
893         return vmcs_config.pin_based_exec_ctrl & PIN_BASED_POSTED_INTR;
894 }
895 
896 static inline bool cpu_has_vmx_apicv(void)
897 {
898         return cpu_has_vmx_apic_register_virt() &&
899                 cpu_has_vmx_virtual_intr_delivery() &&
900                 cpu_has_vmx_posted_intr();
901 }
902 
903 static inline bool cpu_has_vmx_flexpriority(void)
904 {
905         return cpu_has_vmx_tpr_shadow() &&
906                 cpu_has_vmx_virtualize_apic_accesses();
907 }
908 
909 static inline bool cpu_has_vmx_ept_execute_only(void)
910 {
911         return vmx_capability.ept & VMX_EPT_EXECUTE_ONLY_BIT;
912 }
913 
914 static inline bool cpu_has_vmx_eptp_uncacheable(void)
915 {
916         return vmx_capability.ept & VMX_EPTP_UC_BIT;
917 }
918 
919 static inline bool cpu_has_vmx_eptp_writeback(void)
920 {
921         return vmx_capability.ept & VMX_EPTP_WB_BIT;
922 }
923 
924 static inline bool cpu_has_vmx_ept_2m_page(void)
925 {
926         return vmx_capability.ept & VMX_EPT_2MB_PAGE_BIT;
927 }
928 
929 static inline bool cpu_has_vmx_ept_1g_page(void)
930 {
931         return vmx_capability.ept & VMX_EPT_1GB_PAGE_BIT;
932 }
933 
934 static inline bool cpu_has_vmx_ept_4levels(void)
935 {
936         return vmx_capability.ept & VMX_EPT_PAGE_WALK_4_BIT;
937 }
938 
939 static inline bool cpu_has_vmx_ept_ad_bits(void)
940 {
941         return vmx_capability.ept & VMX_EPT_AD_BIT;
942 }
943 
944 static inline bool cpu_has_vmx_invept_context(void)
945 {
946         return vmx_capability.ept & VMX_EPT_EXTENT_CONTEXT_BIT;
947 }
948 
949 static inline bool cpu_has_vmx_invept_global(void)
950 {
951         return vmx_capability.ept & VMX_EPT_EXTENT_GLOBAL_BIT;
952 }
953 
954 static inline bool cpu_has_vmx_invvpid_single(void)
955 {
956         return vmx_capability.vpid & VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT;
957 }
958 
959 static inline bool cpu_has_vmx_invvpid_global(void)
960 {
961         return vmx_capability.vpid & VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT;
962 }
963 
964 static inline bool cpu_has_vmx_ept(void)
965 {
966         return vmcs_config.cpu_based_2nd_exec_ctrl &
967                 SECONDARY_EXEC_ENABLE_EPT;
968 }
969 
970 static inline bool cpu_has_vmx_unrestricted_guest(void)
971 {
972         return vmcs_config.cpu_based_2nd_exec_ctrl &
973                 SECONDARY_EXEC_UNRESTRICTED_GUEST;
974 }
975 
976 static inline bool cpu_has_vmx_ple(void)
977 {
978         return vmcs_config.cpu_based_2nd_exec_ctrl &
979                 SECONDARY_EXEC_PAUSE_LOOP_EXITING;
980 }
981 
982 static inline bool vm_need_virtualize_apic_accesses(struct kvm *kvm)
983 {
984         return flexpriority_enabled && irqchip_in_kernel(kvm);
985 }
986 
987 static inline bool cpu_has_vmx_vpid(void)
988 {
989         return vmcs_config.cpu_based_2nd_exec_ctrl &
990                 SECONDARY_EXEC_ENABLE_VPID;
991 }
992 
993 static inline bool cpu_has_vmx_rdtscp(void)
994 {
995         return vmcs_config.cpu_based_2nd_exec_ctrl &
996                 SECONDARY_EXEC_RDTSCP;
997 }
998 
999 static inline bool cpu_has_vmx_invpcid(void)
1000 {
1001         return vmcs_config.cpu_based_2nd_exec_ctrl &
1002                 SECONDARY_EXEC_ENABLE_INVPCID;
1003 }
1004 
1005 static inline bool cpu_has_virtual_nmis(void)
1006 {
1007         return vmcs_config.pin_based_exec_ctrl & PIN_BASED_VIRTUAL_NMIS;
1008 }
1009 
1010 static inline bool cpu_has_vmx_wbinvd_exit(void)
1011 {
1012         return vmcs_config.cpu_based_2nd_exec_ctrl &
1013                 SECONDARY_EXEC_WBINVD_EXITING;
1014 }
1015 
1016 static inline bool cpu_has_vmx_shadow_vmcs(void)
1017 {
1018         u64 vmx_msr;
1019         rdmsrl(MSR_IA32_VMX_MISC, vmx_msr);
1020         /* check if the cpu supports writing r/o exit information fields */
1021         if (!(vmx_msr & MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS))
1022                 return false;
1023 
1024         return vmcs_config.cpu_based_2nd_exec_ctrl &
1025                 SECONDARY_EXEC_SHADOW_VMCS;
1026 }
1027 
1028 static inline bool report_flexpriority(void)
1029 {
1030         return flexpriority_enabled;
1031 }
1032 
1033 static inline bool nested_cpu_has(struct vmcs12 *vmcs12, u32 bit)
1034 {
1035         return vmcs12->cpu_based_vm_exec_control & bit;
1036 }
1037 
1038 static inline bool nested_cpu_has2(struct vmcs12 *vmcs12, u32 bit)
1039 {
1040         return (vmcs12->cpu_based_vm_exec_control &
1041                         CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) &&
1042                 (vmcs12->secondary_vm_exec_control & bit);
1043 }
1044 
1045 static inline bool nested_cpu_has_virtual_nmis(struct vmcs12 *vmcs12)
1046 {
1047         return vmcs12->pin_based_vm_exec_control & PIN_BASED_VIRTUAL_NMIS;
1048 }
1049 
1050 static inline int nested_cpu_has_ept(struct vmcs12 *vmcs12)
1051 {
1052         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_EPT);
1053 }
1054 
1055 static inline bool is_nmi(u32 intr_info)
1056 {
1057         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
1058                 == (INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK);
1059 }
1060 
1061 static void nested_vmx_vmexit(struct kvm_vcpu *vcpu);
1062 static void nested_vmx_entry_failure(struct kvm_vcpu *vcpu,
1063                         struct vmcs12 *vmcs12,
1064                         u32 reason, unsigned long qualification);
1065 
1066 static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr)
1067 {
1068         int i;
1069 
1070         for (i = 0; i < vmx->nmsrs; ++i)
1071                 if (vmx_msr_index[vmx->guest_msrs[i].index] == msr)
1072                         return i;
1073         return -1;
1074 }
1075 
1076 static inline void __invvpid(int ext, u16 vpid, gva_t gva)
1077 {
1078     struct {
1079         u64 vpid : 16;
1080         u64 rsvd : 48;
1081         u64 gva;
1082     } operand = { vpid, 0, gva };
1083 
1084     asm volatile (__ex(ASM_VMX_INVVPID)
1085                   /* CF==1 or ZF==1 --> rc = -1 */
1086                   "; ja 1f ; ud2 ; 1:"
1087                   : : "a"(&operand), "c"(ext) : "cc", "memory");
1088 }
1089 
1090 static inline void __invept(int ext, u64 eptp, gpa_t gpa)
1091 {
1092         struct {
1093                 u64 eptp, gpa;
1094         } operand = {eptp, gpa};
1095 
1096         asm volatile (__ex(ASM_VMX_INVEPT)
1097                         /* CF==1 or ZF==1 --> rc = -1 */
1098                         "; ja 1f ; ud2 ; 1:\n"
1099                         : : "a" (&operand), "c" (ext) : "cc", "memory");
1100 }
1101 
1102 static struct shared_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr)
1103 {
1104         int i;
1105 
1106         i = __find_msr_index(vmx, msr);
1107         if (i >= 0)
1108                 return &vmx->guest_msrs[i];
1109         return NULL;
1110 }
1111 
1112 static void vmcs_clear(struct vmcs *vmcs)
1113 {
1114         u64 phys_addr = __pa(vmcs);
1115         u8 error;
1116 
1117         asm volatile (__ex(ASM_VMX_VMCLEAR_RAX) "; setna %0"
1118                       : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
1119                       : "cc", "memory");
1120         if (error)
1121                 printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
1122                        vmcs, phys_addr);
1123 }
1124 
1125 static inline void loaded_vmcs_init(struct loaded_vmcs *loaded_vmcs)
1126 {
1127         vmcs_clear(loaded_vmcs->vmcs);
1128         loaded_vmcs->cpu = -1;
1129         loaded_vmcs->launched = 0;
1130 }
1131 
1132 static void vmcs_load(struct vmcs *vmcs)
1133 {
1134         u64 phys_addr = __pa(vmcs);
1135         u8 error;
1136 
1137         asm volatile (__ex(ASM_VMX_VMPTRLD_RAX) "; setna %0"
1138                         : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
1139                         : "cc", "memory");
1140         if (error)
1141                 printk(KERN_ERR "kvm: vmptrld %p/%llx failed\n",
1142                        vmcs, phys_addr);
1143 }
1144 
1145 #ifdef CONFIG_KEXEC
1146 /*
1147  * This bitmap is used to indicate whether the vmclear
1148  * operation is enabled on all cpus. All disabled by
1149  * default.
1150  */
1151 static cpumask_t crash_vmclear_enabled_bitmap = CPU_MASK_NONE;
1152 
1153 static inline void crash_enable_local_vmclear(int cpu)
1154 {
1155         cpumask_set_cpu(cpu, &crash_vmclear_enabled_bitmap);
1156 }
1157 
1158 static inline void crash_disable_local_vmclear(int cpu)
1159 {
1160         cpumask_clear_cpu(cpu, &crash_vmclear_enabled_bitmap);
1161 }
1162 
1163 static inline int crash_local_vmclear_enabled(int cpu)
1164 {
1165         return cpumask_test_cpu(cpu, &crash_vmclear_enabled_bitmap);
1166 }
1167 
1168 static void crash_vmclear_local_loaded_vmcss(void)
1169 {
1170         int cpu = raw_smp_processor_id();
1171         struct loaded_vmcs *v;
1172 
1173         if (!crash_local_vmclear_enabled(cpu))
1174                 return;
1175 
1176         list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu),
1177                             loaded_vmcss_on_cpu_link)
1178                 vmcs_clear(v->vmcs);
1179 }
1180 #else
1181 static inline void crash_enable_local_vmclear(int cpu) { }
1182 static inline void crash_disable_local_vmclear(int cpu) { }
1183 #endif /* CONFIG_KEXEC */
1184 
1185 static void __loaded_vmcs_clear(void *arg)
1186 {
1187         struct loaded_vmcs *loaded_vmcs = arg;
1188         int cpu = raw_smp_processor_id();
1189 
1190         if (loaded_vmcs->cpu != cpu)
1191                 return; /* vcpu migration can race with cpu offline */
1192         if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs)
1193                 per_cpu(current_vmcs, cpu) = NULL;
1194         crash_disable_local_vmclear(cpu);
1195         list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link);
1196 
1197         /*
1198          * we should ensure updating loaded_vmcs->loaded_vmcss_on_cpu_link
1199          * is before setting loaded_vmcs->vcpu to -1 which is done in
1200          * loaded_vmcs_init. Otherwise, other cpu can see vcpu = -1 fist
1201          * then adds the vmcs into percpu list before it is deleted.
1202          */
1203         smp_wmb();
1204 
1205         loaded_vmcs_init(loaded_vmcs);
1206         crash_enable_local_vmclear(cpu);
1207 }
1208 
1209 static void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs)
1210 {
1211         int cpu = loaded_vmcs->cpu;
1212 
1213         if (cpu != -1)
1214                 smp_call_function_single(cpu,
1215                          __loaded_vmcs_clear, loaded_vmcs, 1);
1216 }
1217 
1218 static inline void vpid_sync_vcpu_single(struct vcpu_vmx *vmx)
1219 {
1220         if (vmx->vpid == 0)
1221                 return;
1222 
1223         if (cpu_has_vmx_invvpid_single())
1224                 __invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vmx->vpid, 0);
1225 }
1226 
1227 static inline void vpid_sync_vcpu_global(void)
1228 {
1229         if (cpu_has_vmx_invvpid_global())
1230                 __invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0);
1231 }
1232 
1233 static inline void vpid_sync_context(struct vcpu_vmx *vmx)
1234 {
1235         if (cpu_has_vmx_invvpid_single())
1236                 vpid_sync_vcpu_single(vmx);
1237         else
1238                 vpid_sync_vcpu_global();
1239 }
1240 
1241 static inline void ept_sync_global(void)
1242 {
1243         if (cpu_has_vmx_invept_global())
1244                 __invept(VMX_EPT_EXTENT_GLOBAL, 0, 0);
1245 }
1246 
1247 static inline void ept_sync_context(u64 eptp)
1248 {
1249         if (enable_ept) {
1250                 if (cpu_has_vmx_invept_context())
1251                         __invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0);
1252                 else
1253                         ept_sync_global();
1254         }
1255 }
1256 
1257 static __always_inline unsigned long vmcs_readl(unsigned long field)
1258 {
1259         unsigned long value;
1260 
1261         asm volatile (__ex_clear(ASM_VMX_VMREAD_RDX_RAX, "%0")
1262                       : "=a"(value) : "d"(field) : "cc");
1263         return value;
1264 }
1265 
1266 static __always_inline u16 vmcs_read16(unsigned long field)
1267 {
1268         return vmcs_readl(field);
1269 }
1270 
1271 static __always_inline u32 vmcs_read32(unsigned long field)
1272 {
1273         return vmcs_readl(field);
1274 }
1275 
1276 static __always_inline u64 vmcs_read64(unsigned long field)
1277 {
1278 #ifdef CONFIG_X86_64
1279         return vmcs_readl(field);
1280 #else
1281         return vmcs_readl(field) | ((u64)vmcs_readl(field+1) << 32);
1282 #endif
1283 }
1284 
1285 static noinline void vmwrite_error(unsigned long field, unsigned long value)
1286 {
1287         printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n",
1288                field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
1289         dump_stack();
1290 }
1291 
1292 static void vmcs_writel(unsigned long field, unsigned long value)
1293 {
1294         u8 error;
1295 
1296         asm volatile (__ex(ASM_VMX_VMWRITE_RAX_RDX) "; setna %0"
1297                        : "=q"(error) : "a"(value), "d"(field) : "cc");
1298         if (unlikely(error))
1299                 vmwrite_error(field, value);
1300 }
1301 
1302 static void vmcs_write16(unsigned long field, u16 value)
1303 {
1304         vmcs_writel(field, value);
1305 }
1306 
1307 static void vmcs_write32(unsigned long field, u32 value)
1308 {
1309         vmcs_writel(field, value);
1310 }
1311 
1312 static void vmcs_write64(unsigned long field, u64 value)
1313 {
1314         vmcs_writel(field, value);
1315 #ifndef CONFIG_X86_64
1316         asm volatile ("");
1317         vmcs_writel(field+1, value >> 32);
1318 #endif
1319 }
1320 
1321 static void vmcs_clear_bits(unsigned long field, u32 mask)
1322 {
1323         vmcs_writel(field, vmcs_readl(field) & ~mask);
1324 }
1325 
1326 static void vmcs_set_bits(unsigned long field, u32 mask)
1327 {
1328         vmcs_writel(field, vmcs_readl(field) | mask);
1329 }
1330 
1331 static void vmx_segment_cache_clear(struct vcpu_vmx *vmx)
1332 {
1333         vmx->segment_cache.bitmask = 0;
1334 }
1335 
1336 static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg,
1337                                        unsigned field)
1338 {
1339         bool ret;
1340         u32 mask = 1 << (seg * SEG_FIELD_NR + field);
1341 
1342         if (!(vmx->vcpu.arch.regs_avail & (1 << VCPU_EXREG_SEGMENTS))) {
1343                 vmx->vcpu.arch.regs_avail |= (1 << VCPU_EXREG_SEGMENTS);
1344                 vmx->segment_cache.bitmask = 0;
1345         }
1346         ret = vmx->segment_cache.bitmask & mask;
1347         vmx->segment_cache.bitmask |= mask;
1348         return ret;
1349 }
1350 
1351 static u16 vmx_read_guest_seg_selector(struct vcpu_vmx *vmx, unsigned seg)
1352 {
1353         u16 *p = &vmx->segment_cache.seg[seg].selector;
1354 
1355         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_SEL))
1356                 *p = vmcs_read16(kvm_vmx_segment_fields[seg].selector);
1357         return *p;
1358 }
1359 
1360 static ulong vmx_read_guest_seg_base(struct vcpu_vmx *vmx, unsigned seg)
1361 {
1362         ulong *p = &vmx->segment_cache.seg[seg].base;
1363 
1364         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_BASE))
1365                 *p = vmcs_readl(kvm_vmx_segment_fields[seg].base);
1366         return *p;
1367 }
1368 
1369 static u32 vmx_read_guest_seg_limit(struct vcpu_vmx *vmx, unsigned seg)
1370 {
1371         u32 *p = &vmx->segment_cache.seg[seg].limit;
1372 
1373         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_LIMIT))
1374                 *p = vmcs_read32(kvm_vmx_segment_fields[seg].limit);
1375         return *p;
1376 }
1377 
1378 static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg)
1379 {
1380         u32 *p = &vmx->segment_cache.seg[seg].ar;
1381 
1382         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_AR))
1383                 *p = vmcs_read32(kvm_vmx_segment_fields[seg].ar_bytes);
1384         return *p;
1385 }
1386 
1387 static void update_exception_bitmap(struct kvm_vcpu *vcpu)
1388 {
1389         u32 eb;
1390 
1391         eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) |
1392              (1u << NM_VECTOR) | (1u << DB_VECTOR) | (1u << AC_VECTOR);
1393         if ((vcpu->guest_debug &
1394              (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) ==
1395             (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP))
1396                 eb |= 1u << BP_VECTOR;
1397         if (to_vmx(vcpu)->rmode.vm86_active)
1398                 eb = ~0;
1399         if (enable_ept)
1400                 eb &= ~(1u << PF_VECTOR); /* bypass_guest_pf = 0 */
1401         if (vcpu->fpu_active)
1402                 eb &= ~(1u << NM_VECTOR);
1403 
1404         /* When we are running a nested L2 guest and L1 specified for it a
1405          * certain exception bitmap, we must trap the same exceptions and pass
1406          * them to L1. When running L2, we will only handle the exceptions
1407          * specified above if L1 did not want them.
1408          */
1409         if (is_guest_mode(vcpu))
1410                 eb |= get_vmcs12(vcpu)->exception_bitmap;
1411 
1412         vmcs_write32(EXCEPTION_BITMAP, eb);
1413 }
1414 
1415 static void clear_atomic_switch_msr_special(unsigned long entry,
1416                 unsigned long exit)
1417 {
1418         vmcs_clear_bits(VM_ENTRY_CONTROLS, entry);
1419         vmcs_clear_bits(VM_EXIT_CONTROLS, exit);
1420 }
1421 
1422 static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr)
1423 {
1424         unsigned i;
1425         struct msr_autoload *m = &vmx->msr_autoload;
1426 
1427         switch (msr) {
1428         case MSR_EFER:
1429                 if (cpu_has_load_ia32_efer) {
1430                         clear_atomic_switch_msr_special(VM_ENTRY_LOAD_IA32_EFER,
1431                                         VM_EXIT_LOAD_IA32_EFER);
1432                         return;
1433                 }
1434                 break;
1435         case MSR_CORE_PERF_GLOBAL_CTRL:
1436                 if (cpu_has_load_perf_global_ctrl) {
1437                         clear_atomic_switch_msr_special(
1438                                         VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
1439                                         VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);
1440                         return;
1441                 }
1442                 break;
1443         }
1444 
1445         for (i = 0; i < m->nr; ++i)
1446                 if (m->guest[i].index == msr)
1447                         break;
1448 
1449         if (i == m->nr)
1450                 return;
1451         --m->nr;
1452         m->guest[i] = m->guest[m->nr];
1453         m->host[i] = m->host[m->nr];
1454         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr);
1455         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr);
1456 }
1457 
1458 static void add_atomic_switch_msr_special(unsigned long entry,
1459                 unsigned long exit, unsigned long guest_val_vmcs,
1460                 unsigned long host_val_vmcs, u64 guest_val, u64 host_val)
1461 {
1462         vmcs_write64(guest_val_vmcs, guest_val);
1463         vmcs_write64(host_val_vmcs, host_val);
1464         vmcs_set_bits(VM_ENTRY_CONTROLS, entry);
1465         vmcs_set_bits(VM_EXIT_CONTROLS, exit);
1466 }
1467 
1468 static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr,
1469                                   u64 guest_val, u64 host_val)
1470 {
1471         unsigned i;
1472         struct msr_autoload *m = &vmx->msr_autoload;
1473 
1474         switch (msr) {
1475         case MSR_EFER:
1476                 if (cpu_has_load_ia32_efer) {
1477                         add_atomic_switch_msr_special(VM_ENTRY_LOAD_IA32_EFER,
1478                                         VM_EXIT_LOAD_IA32_EFER,
1479                                         GUEST_IA32_EFER,
1480                                         HOST_IA32_EFER,
1481                                         guest_val, host_val);
1482                         return;
1483                 }
1484                 break;
1485         case MSR_CORE_PERF_GLOBAL_CTRL:
1486                 if (cpu_has_load_perf_global_ctrl) {
1487                         add_atomic_switch_msr_special(
1488                                         VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
1489                                         VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL,
1490                                         GUEST_IA32_PERF_GLOBAL_CTRL,
1491                                         HOST_IA32_PERF_GLOBAL_CTRL,
1492                                         guest_val, host_val);
1493                         return;
1494                 }
1495                 break;
1496         case MSR_IA32_PEBS_ENABLE:
1497                 /* PEBS needs a quiescent period after being disabled (to write
1498                  * a record).  Disabling PEBS through VMX MSR swapping doesn't
1499                  * provide that period, so a CPU could write host's record into
1500                  * guest's memory.
1501                  */
1502                 wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
1503         }
1504 
1505         for (i = 0; i < m->nr; ++i)
1506                 if (m->guest[i].index == msr)
1507                         break;
1508 
1509         if (i == NR_AUTOLOAD_MSRS) {
1510                 printk_once(KERN_WARNING"Not enough mst switch entries. "
1511                                 "Can't add msr %x\n", msr);
1512                 return;
1513         } else if (i == m->nr) {
1514                 ++m->nr;
1515                 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr);
1516                 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr);
1517         }
1518 
1519         m->guest[i].index = msr;
1520         m->guest[i].value = guest_val;
1521         m->host[i].index = msr;
1522         m->host[i].value = host_val;
1523 }
1524 
1525 static void reload_tss(void)
1526 {
1527         /*
1528          * VT restores TR but not its size.  Useless.
1529          */
1530         struct desc_ptr *gdt = &__get_cpu_var(host_gdt);
1531         struct desc_struct *descs;
1532 
1533         descs = (void *)gdt->address;
1534         descs[GDT_ENTRY_TSS].type = 9; /* available TSS */
1535         load_TR_desc();
1536 }
1537 
1538 static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset)
1539 {
1540         u64 guest_efer;
1541         u64 ignore_bits;
1542 
1543         guest_efer = vmx->vcpu.arch.efer;
1544 
1545         /*
1546          * NX is emulated; LMA and LME handled by hardware; SCE meaningless
1547          * outside long mode
1548          */
1549         ignore_bits = EFER_NX | EFER_SCE;
1550 #ifdef CONFIG_X86_64
1551         ignore_bits |= EFER_LMA | EFER_LME;
1552         /* SCE is meaningful only in long mode on Intel */
1553         if (guest_efer & EFER_LMA)
1554                 ignore_bits &= ~(u64)EFER_SCE;
1555 #endif
1556         guest_efer &= ~ignore_bits;
1557         guest_efer |= host_efer & ignore_bits;
1558         vmx->guest_msrs[efer_offset].data = guest_efer;
1559         vmx->guest_msrs[efer_offset].mask = ~ignore_bits;
1560 
1561         clear_atomic_switch_msr(vmx, MSR_EFER);
1562         /* On ept, can't emulate nx, and must switch nx atomically */
1563         if (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX)) {
1564                 guest_efer = vmx->vcpu.arch.efer;
1565                 if (!(guest_efer & EFER_LMA))
1566                         guest_efer &= ~EFER_LME;
1567                 add_atomic_switch_msr(vmx, MSR_EFER, guest_efer, host_efer);
1568                 return false;
1569         }
1570 
1571         return true;
1572 }
1573 
1574 static unsigned long segment_base(u16 selector)
1575 {
1576         struct desc_ptr *gdt = &__get_cpu_var(host_gdt);
1577         struct desc_struct *d;
1578         unsigned long table_base;
1579         unsigned long v;
1580 
1581         if (!(selector & ~3))
1582                 return 0;
1583 
1584         table_base = gdt->address;
1585 
1586         if (selector & 4) {           /* from ldt */
1587                 u16 ldt_selector = kvm_read_ldt();
1588 
1589                 if (!(ldt_selector & ~3))
1590                         return 0;
1591 
1592                 table_base = segment_base(ldt_selector);
1593         }
1594         d = (struct desc_struct *)(table_base + (selector & ~7));
1595         v = get_desc_base(d);
1596 #ifdef CONFIG_X86_64
1597        if (d->s == 0 && (d->type == 2 || d->type == 9 || d->type == 11))
1598                v |= ((unsigned long)((struct ldttss_desc64 *)d)->base3) << 32;
1599 #endif
1600         return v;
1601 }
1602 
1603 static inline unsigned long kvm_read_tr_base(void)
1604 {
1605         u16 tr;
1606         asm("str %0" : "=g"(tr));
1607         return segment_base(tr);
1608 }
1609 
1610 static void vmx_save_host_state(struct kvm_vcpu *vcpu)
1611 {
1612         struct vcpu_vmx *vmx = to_vmx(vcpu);
1613         int i;
1614 
1615         if (vmx->host_state.loaded)
1616                 return;
1617 
1618         vmx->host_state.loaded = 1;
1619         /*
1620          * Set host fs and gs selectors.  Unfortunately, 22.2.3 does not
1621          * allow segment selectors with cpl > 0 or ti == 1.
1622          */
1623         vmx->host_state.ldt_sel = kvm_read_ldt();
1624         vmx->host_state.gs_ldt_reload_needed = vmx->host_state.ldt_sel;
1625         savesegment(fs, vmx->host_state.fs_sel);
1626         if (!(vmx->host_state.fs_sel & 7)) {
1627                 vmcs_write16(HOST_FS_SELECTOR, vmx->host_state.fs_sel);
1628                 vmx->host_state.fs_reload_needed = 0;
1629         } else {
1630                 vmcs_write16(HOST_FS_SELECTOR, 0);
1631                 vmx->host_state.fs_reload_needed = 1;
1632         }
1633         savesegment(gs, vmx->host_state.gs_sel);
1634         if (!(vmx->host_state.gs_sel & 7))
1635                 vmcs_write16(HOST_GS_SELECTOR, vmx->host_state.gs_sel);
1636         else {
1637                 vmcs_write16(HOST_GS_SELECTOR, 0);
1638                 vmx->host_state.gs_ldt_reload_needed = 1;
1639         }
1640 
1641 #ifdef CONFIG_X86_64
1642         savesegment(ds, vmx->host_state.ds_sel);
1643         savesegment(es, vmx->host_state.es_sel);
1644 #endif
1645 
1646 #ifdef CONFIG_X86_64
1647         vmcs_writel(HOST_FS_BASE, read_msr(MSR_FS_BASE));
1648         vmcs_writel(HOST_GS_BASE, read_msr(MSR_GS_BASE));
1649 #else
1650         vmcs_writel(HOST_FS_BASE, segment_base(vmx->host_state.fs_sel));
1651         vmcs_writel(HOST_GS_BASE, segment_base(vmx->host_state.gs_sel));
1652 #endif
1653 
1654 #ifdef CONFIG_X86_64
1655         rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
1656         if (is_long_mode(&vmx->vcpu))
1657                 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
1658 #endif
1659         for (i = 0; i < vmx->save_nmsrs; ++i)
1660                 kvm_set_shared_msr(vmx->guest_msrs[i].index,
1661                                    vmx->guest_msrs[i].data,
1662                                    vmx->guest_msrs[i].mask);
1663 }
1664 
1665 static void __vmx_load_host_state(struct vcpu_vmx *vmx)
1666 {
1667         if (!vmx->host_state.loaded)
1668                 return;
1669 
1670         ++vmx->vcpu.stat.host_state_reload;
1671         vmx->host_state.loaded = 0;
1672 #ifdef CONFIG_X86_64
1673         if (is_long_mode(&vmx->vcpu))
1674                 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
1675 #endif
1676         if (vmx->host_state.gs_ldt_reload_needed) {
1677                 kvm_load_ldt(vmx->host_state.ldt_sel);
1678 #ifdef CONFIG_X86_64
1679                 load_gs_index(vmx->host_state.gs_sel);
1680 #else
1681                 loadsegment(gs, vmx->host_state.gs_sel);
1682 #endif
1683         }
1684         if (vmx->host_state.fs_reload_needed)
1685                 loadsegment(fs, vmx->host_state.fs_sel);
1686 #ifdef CONFIG_X86_64
1687         if (unlikely(vmx->host_state.ds_sel | vmx->host_state.es_sel)) {
1688                 loadsegment(ds, vmx->host_state.ds_sel);
1689                 loadsegment(es, vmx->host_state.es_sel);
1690         }
1691 #endif
1692         reload_tss();
1693 #ifdef CONFIG_X86_64
1694         wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
1695 #endif
1696         /*
1697          * If the FPU is not active (through the host task or
1698          * the guest vcpu), then restore the cr0.TS bit.
1699          */
1700         if (!user_has_fpu() && !vmx->vcpu.guest_fpu_loaded)
1701                 stts();
1702         load_gdt(&__get_cpu_var(host_gdt));
1703 }
1704 
1705 static void vmx_load_host_state(struct vcpu_vmx *vmx)
1706 {
1707         preempt_disable();
1708         __vmx_load_host_state(vmx);
1709         preempt_enable();
1710 }
1711 
1712 /*
1713  * Switches to specified vcpu, until a matching vcpu_put(), but assumes
1714  * vcpu mutex is already taken.
1715  */
1716 static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1717 {
1718         struct vcpu_vmx *vmx = to_vmx(vcpu);
1719         u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
1720 
1721         if (!vmm_exclusive)
1722                 kvm_cpu_vmxon(phys_addr);
1723         else if (vmx->loaded_vmcs->cpu != cpu)
1724                 loaded_vmcs_clear(vmx->loaded_vmcs);
1725 
1726         if (per_cpu(current_vmcs, cpu) != vmx->loaded_vmcs->vmcs) {
1727                 per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;
1728                 vmcs_load(vmx->loaded_vmcs->vmcs);
1729         }
1730 
1731         if (vmx->loaded_vmcs->cpu != cpu) {
1732                 struct desc_ptr *gdt = &__get_cpu_var(host_gdt);
1733                 unsigned long sysenter_esp;
1734 
1735                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1736                 local_irq_disable();
1737                 crash_disable_local_vmclear(cpu);
1738 
1739                 /*
1740                  * Read loaded_vmcs->cpu should be before fetching
1741                  * loaded_vmcs->loaded_vmcss_on_cpu_link.
1742                  * See the comments in __loaded_vmcs_clear().
1743                  */
1744                 smp_rmb();
1745 
1746                 list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link,
1747                          &per_cpu(loaded_vmcss_on_cpu, cpu));
1748                 crash_enable_local_vmclear(cpu);
1749                 local_irq_enable();
1750 
1751                 /*
1752                  * Linux uses per-cpu TSS and GDT, so set these when switching
1753                  * processors.
1754                  */
1755                 vmcs_writel(HOST_TR_BASE, kvm_read_tr_base()); /* 22.2.4 */
1756                 vmcs_writel(HOST_GDTR_BASE, gdt->address);   /* 22.2.4 */
1757 
1758                 rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
1759                 vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
1760                 vmx->loaded_vmcs->cpu = cpu;
1761         }
1762 }
1763 
1764 static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
1765 {
1766         __vmx_load_host_state(to_vmx(vcpu));
1767         if (!vmm_exclusive) {
1768                 __loaded_vmcs_clear(to_vmx(vcpu)->loaded_vmcs);
1769                 vcpu->cpu = -1;
1770                 kvm_cpu_vmxoff();
1771         }
1772 }
1773 
1774 static void vmx_fpu_activate(struct kvm_vcpu *vcpu)
1775 {
1776         ulong cr0;
1777 
1778         if (vcpu->fpu_active)
1779                 return;
1780         vcpu->fpu_active = 1;
1781         cr0 = vmcs_readl(GUEST_CR0);
1782         cr0 &= ~(X86_CR0_TS | X86_CR0_MP);
1783         cr0 |= kvm_read_cr0_bits(vcpu, X86_CR0_TS | X86_CR0_MP);
1784         vmcs_writel(GUEST_CR0, cr0);
1785         update_exception_bitmap(vcpu);
1786         vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS;
1787         if (is_guest_mode(vcpu))
1788                 vcpu->arch.cr0_guest_owned_bits &=
1789                         ~get_vmcs12(vcpu)->cr0_guest_host_mask;
1790         vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
1791 }
1792 
1793 static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu);
1794 
1795 /*
1796  * Return the cr0 value that a nested guest would read. This is a combination
1797  * of the real cr0 used to run the guest (guest_cr0), and the bits shadowed by
1798  * its hypervisor (cr0_read_shadow).
1799  */
1800 static inline unsigned long nested_read_cr0(struct vmcs12 *fields)
1801 {
1802         return (fields->guest_cr0 & ~fields->cr0_guest_host_mask) |
1803                 (fields->cr0_read_shadow & fields->cr0_guest_host_mask);
1804 }
1805 static inline unsigned long nested_read_cr4(struct vmcs12 *fields)
1806 {
1807         return (fields->guest_cr4 & ~fields->cr4_guest_host_mask) |
1808                 (fields->cr4_read_shadow & fields->cr4_guest_host_mask);
1809 }
1810 
1811 static void vmx_fpu_deactivate(struct kvm_vcpu *vcpu)
1812 {
1813         /* Note that there is no vcpu->fpu_active = 0 here. The caller must
1814          * set this *before* calling this function.
1815          */
1816         vmx_decache_cr0_guest_bits(vcpu);
1817         vmcs_set_bits(GUEST_CR0, X86_CR0_TS | X86_CR0_MP);
1818         update_exception_bitmap(vcpu);
1819         vcpu->arch.cr0_guest_owned_bits = 0;
1820         vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
1821         if (is_guest_mode(vcpu)) {
1822                 /*
1823                  * L1's specified read shadow might not contain the TS bit,
1824                  * so now that we turned on shadowing of this bit, we need to
1825                  * set this bit of the shadow. Like in nested_vmx_run we need
1826                  * nested_read_cr0(vmcs12), but vmcs12->guest_cr0 is not yet
1827                  * up-to-date here because we just decached cr0.TS (and we'll
1828                  * only update vmcs12->guest_cr0 on nested exit).
1829                  */
1830                 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
1831                 vmcs12->guest_cr0 = (vmcs12->guest_cr0 & ~X86_CR0_TS) |
1832                         (vcpu->arch.cr0 & X86_CR0_TS);
1833                 vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12));
1834         } else
1835                 vmcs_writel(CR0_READ_SHADOW, vcpu->arch.cr0);
1836 }
1837 
1838 static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
1839 {
1840         unsigned long rflags, save_rflags;
1841 
1842         if (!test_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail)) {
1843                 __set_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail);
1844                 rflags = vmcs_readl(GUEST_RFLAGS);
1845                 if (to_vmx(vcpu)->rmode.vm86_active) {
1846                         rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
1847                         save_rflags = to_vmx(vcpu)->rmode.save_rflags;
1848                         rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
1849                 }
1850                 to_vmx(vcpu)->rflags = rflags;
1851         }
1852         return to_vmx(vcpu)->rflags;
1853 }
1854 
1855 static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1856 {
1857         __set_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail);
1858         to_vmx(vcpu)->rflags = rflags;
1859         if (to_vmx(vcpu)->rmode.vm86_active) {
1860                 to_vmx(vcpu)->rmode.save_rflags = rflags;
1861                 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
1862         }
1863         vmcs_writel(GUEST_RFLAGS, rflags);
1864 }
1865 
1866 static u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
1867 {
1868         u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
1869         int ret = 0;
1870 
1871         if (interruptibility & GUEST_INTR_STATE_STI)
1872                 ret |= KVM_X86_SHADOW_INT_STI;
1873         if (interruptibility & GUEST_INTR_STATE_MOV_SS)
1874                 ret |= KVM_X86_SHADOW_INT_MOV_SS;
1875 
1876         return ret & mask;
1877 }
1878 
1879 static void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
1880 {
1881         u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
1882         u32 interruptibility = interruptibility_old;
1883 
1884         interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS);
1885 
1886         if (mask & KVM_X86_SHADOW_INT_MOV_SS)
1887                 interruptibility |= GUEST_INTR_STATE_MOV_SS;
1888         else if (mask & KVM_X86_SHADOW_INT_STI)
1889                 interruptibility |= GUEST_INTR_STATE_STI;
1890 
1891         if ((interruptibility != interruptibility_old))
1892                 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility);
1893 }
1894 
1895 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
1896 {
1897         unsigned long rip;
1898 
1899         rip = kvm_rip_read(vcpu);
1900         rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
1901         kvm_rip_write(vcpu, rip);
1902 
1903         /* skipping an emulated instruction also counts */
1904         vmx_set_interrupt_shadow(vcpu, 0);
1905 }
1906 
1907 /*
1908  * KVM wants to inject page-faults which it got to the guest. This function
1909  * checks whether in a nested guest, we need to inject them to L1 or L2.
1910  * This function assumes it is called with the exit reason in vmcs02 being
1911  * a #PF exception (this is the only case in which KVM injects a #PF when L2
1912  * is running).
1913  */
1914 static int nested_pf_handled(struct kvm_vcpu *vcpu)
1915 {
1916         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
1917 
1918         /* TODO: also check PFEC_MATCH/MASK, not just EB.PF. */
1919         if (!(vmcs12->exception_bitmap & (1u << PF_VECTOR)))
1920                 return 0;
1921 
1922         nested_vmx_vmexit(vcpu);
1923         return 1;
1924 }
1925 
1926 static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
1927                                 bool has_error_code, u32 error_code,
1928                                 bool reinject)
1929 {
1930         struct vcpu_vmx *vmx = to_vmx(vcpu);
1931         u32 intr_info = nr | INTR_INFO_VALID_MASK;
1932 
1933         if (nr == PF_VECTOR && is_guest_mode(vcpu) &&
1934             !vmx->nested.nested_run_pending && nested_pf_handled(vcpu))
1935                 return;
1936 
1937         if (has_error_code) {
1938                 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
1939                 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
1940         }
1941 
1942         if (vmx->rmode.vm86_active) {
1943                 int inc_eip = 0;
1944                 if (kvm_exception_is_soft(nr))
1945                         inc_eip = vcpu->arch.event_exit_inst_len;
1946                 if (kvm_inject_realmode_interrupt(vcpu, nr, inc_eip) != EMULATE_DONE)
1947                         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
1948                 return;
1949         }
1950 
1951         if (kvm_exception_is_soft(nr)) {
1952                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
1953                              vmx->vcpu.arch.event_exit_inst_len);
1954                 intr_info |= INTR_TYPE_SOFT_EXCEPTION;
1955         } else
1956                 intr_info |= INTR_TYPE_HARD_EXCEPTION;
1957 
1958         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
1959 }
1960 
1961 static bool vmx_rdtscp_supported(void)
1962 {
1963         return cpu_has_vmx_rdtscp();
1964 }
1965 
1966 static bool vmx_invpcid_supported(void)
1967 {
1968         return cpu_has_vmx_invpcid() && enable_ept;
1969 }
1970 
1971 /*
1972  * Swap MSR entry in host/guest MSR entry array.
1973  */
1974 static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
1975 {
1976         struct shared_msr_entry tmp;
1977 
1978         tmp = vmx->guest_msrs[to];
1979         vmx->guest_msrs[to] = vmx->guest_msrs[from];
1980         vmx->guest_msrs[from] = tmp;
1981 }
1982 
1983 static void vmx_set_msr_bitmap(struct kvm_vcpu *vcpu)
1984 {
1985         unsigned long *msr_bitmap;
1986 
1987         if (irqchip_in_kernel(vcpu->kvm) && apic_x2apic_mode(vcpu->arch.apic)) {
1988                 if (is_long_mode(vcpu))
1989                         msr_bitmap = vmx_msr_bitmap_longmode_x2apic;
1990                 else
1991                         msr_bitmap = vmx_msr_bitmap_legacy_x2apic;
1992         } else {
1993                 if (is_long_mode(vcpu))
1994                         msr_bitmap = vmx_msr_bitmap_longmode;
1995                 else
1996                         msr_bitmap = vmx_msr_bitmap_legacy;
1997         }
1998 
1999         vmcs_write64(MSR_BITMAP, __pa(msr_bitmap));
2000 }
2001 
2002 /*
2003  * Set up the vmcs to automatically save and restore system
2004  * msrs.  Don't touch the 64-bit msrs if the guest is in legacy
2005  * mode, as fiddling with msrs is very expensive.
2006  */
2007 static void setup_msrs(struct vcpu_vmx *vmx)
2008 {
2009         int save_nmsrs, index;
2010 
2011         save_nmsrs = 0;
2012 #ifdef CONFIG_X86_64
2013         if (is_long_mode(&vmx->vcpu)) {
2014                 index = __find_msr_index(vmx, MSR_SYSCALL_MASK);
2015                 if (index >= 0)
2016                         move_msr_up(vmx, index, save_nmsrs++);
2017                 index = __find_msr_index(vmx, MSR_LSTAR);
2018                 if (index >= 0)
2019                         move_msr_up(vmx, index, save_nmsrs++);
2020                 index = __find_msr_index(vmx, MSR_CSTAR);
2021                 if (index >= 0)
2022                         move_msr_up(vmx, index, save_nmsrs++);
2023                 index = __find_msr_index(vmx, MSR_TSC_AUX);
2024                 if (index >= 0 && vmx->rdtscp_enabled)
2025                         move_msr_up(vmx, index, save_nmsrs++);
2026                 /*
2027                  * MSR_STAR is only needed on long mode guests, and only
2028                  * if efer.sce is enabled.
2029                  */
2030                 index = __find_msr_index(vmx, MSR_STAR);
2031                 if ((index >= 0) && (vmx->vcpu.arch.efer & EFER_SCE))
2032                         move_msr_up(vmx, index, save_nmsrs++);
2033         }
2034 #endif
2035         index = __find_msr_index(vmx, MSR_EFER);
2036         if (index >= 0 && update_transition_efer(vmx, index))
2037                 move_msr_up(vmx, index, save_nmsrs++);
2038 
2039         vmx->save_nmsrs = save_nmsrs;
2040 
2041         if (cpu_has_vmx_msr_bitmap())
2042                 vmx_set_msr_bitmap(&vmx->vcpu);
2043 }
2044 
2045 /*
2046  * reads and returns guest's timestamp counter "register"
2047  * guest_tsc = host_tsc + tsc_offset    -- 21.3
2048  */
2049 static u64 guest_read_tsc(void)
2050 {
2051         u64 host_tsc, tsc_offset;
2052 
2053         rdtscll(host_tsc);
2054         tsc_offset = vmcs_read64(TSC_OFFSET);
2055         return host_tsc + tsc_offset;
2056 }
2057 
2058 /*
2059  * Like guest_read_tsc, but always returns L1's notion of the timestamp
2060  * counter, even if a nested guest (L2) is currently running.
2061  */
2062 u64 vmx_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
2063 {
2064         u64 tsc_offset;
2065 
2066         tsc_offset = is_guest_mode(vcpu) ?
2067                 to_vmx(vcpu)->nested.vmcs01_tsc_offset :
2068                 vmcs_read64(TSC_OFFSET);
2069         return host_tsc + tsc_offset;
2070 }
2071 
2072 /*
2073  * Engage any workarounds for mis-matched TSC rates.  Currently limited to
2074  * software catchup for faster rates on slower CPUs.
2075  */
2076 static void vmx_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)
2077 {
2078         if (!scale)
2079                 return;
2080 
2081         if (user_tsc_khz > tsc_khz) {
2082                 vcpu->arch.tsc_catchup = 1;
2083                 vcpu->arch.tsc_always_catchup = 1;
2084         } else
2085                 WARN(1, "user requested TSC rate below hardware speed\n");
2086 }
2087 
2088 static u64 vmx_read_tsc_offset(struct kvm_vcpu *vcpu)
2089 {
2090         return vmcs_read64(TSC_OFFSET);
2091 }
2092 
2093 /*
2094  * writes 'offset' into guest's timestamp counter offset register
2095  */
2096 static void vmx_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
2097 {
2098         if (is_guest_mode(vcpu)) {
2099                 /*
2100                  * We're here if L1 chose not to trap WRMSR to TSC. According
2101                  * to the spec, this should set L1's TSC; The offset that L1
2102                  * set for L2 remains unchanged, and still needs to be added
2103                  * to the newly set TSC to get L2's TSC.
2104                  */
2105                 struct vmcs12 *vmcs12;
2106                 to_vmx(vcpu)->nested.vmcs01_tsc_offset = offset;
2107                 /* recalculate vmcs02.TSC_OFFSET: */
2108                 vmcs12 = get_vmcs12(vcpu);
2109                 vmcs_write64(TSC_OFFSET, offset +
2110                         (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETING) ?
2111                          vmcs12->tsc_offset : 0));
2112         } else {
2113                 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
2114                                            vmcs_read64(TSC_OFFSET), offset);
2115                 vmcs_write64(TSC_OFFSET, offset);
2116         }
2117 }
2118 
2119 static void vmx_adjust_tsc_offset(struct kvm_vcpu *vcpu, s64 adjustment, bool host)
2120 {
2121         u64 offset = vmcs_read64(TSC_OFFSET);
2122 
2123         vmcs_write64(TSC_OFFSET, offset + adjustment);
2124         if (is_guest_mode(vcpu)) {
2125                 /* Even when running L2, the adjustment needs to apply to L1 */
2126                 to_vmx(vcpu)->nested.vmcs01_tsc_offset += adjustment;
2127         } else
2128                 trace_kvm_write_tsc_offset(vcpu->vcpu_id, offset,
2129                                            offset + adjustment);
2130 }
2131 
2132 static u64 vmx_compute_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
2133 {
2134         return target_tsc - native_read_tsc();
2135 }
2136 
2137 static bool guest_cpuid_has_vmx(struct kvm_vcpu *vcpu)
2138 {
2139         struct kvm_cpuid_entry2 *best = kvm_find_cpuid_entry(vcpu, 1, 0);
2140         return best && (best->ecx & (1 << (X86_FEATURE_VMX & 31)));
2141 }
2142 
2143 /*
2144  * nested_vmx_allowed() checks whether a guest should be allowed to use VMX
2145  * instructions and MSRs (i.e., nested VMX). Nested VMX is disabled for
2146  * all guests if the "nested" module option is off, and can also be disabled
2147  * for a single guest by disabling its VMX cpuid bit.
2148  */
2149 static inline bool nested_vmx_allowed(struct kvm_vcpu *vcpu)
2150 {
2151         return nested && guest_cpuid_has_vmx(vcpu);
2152 }
2153 
2154 /*
2155  * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be
2156  * returned for the various VMX controls MSRs when nested VMX is enabled.
2157  * The same values should also be used to verify that vmcs12 control fields are
2158  * valid during nested entry from L1 to L2.
2159  * Each of these control msrs has a low and high 32-bit half: A low bit is on
2160  * if the corresponding bit in the (32-bit) control field *must* be on, and a
2161  * bit in the high half is on if the corresponding bit in the control field
2162  * may be on. See also vmx_control_verify().
2163  * TODO: allow these variables to be modified (downgraded) by module options
2164  * or other means.
2165  */
2166 static u32 nested_vmx_procbased_ctls_low, nested_vmx_procbased_ctls_high;
2167 static u32 nested_vmx_secondary_ctls_low, nested_vmx_secondary_ctls_high;
2168 static u32 nested_vmx_pinbased_ctls_low, nested_vmx_pinbased_ctls_high;
2169 static u32 nested_vmx_exit_ctls_low, nested_vmx_exit_ctls_high;
2170 static u32 nested_vmx_entry_ctls_low, nested_vmx_entry_ctls_high;
2171 static u32 nested_vmx_misc_low, nested_vmx_misc_high;
2172 static u32 nested_vmx_ept_caps;
2173 static __init void nested_vmx_setup_ctls_msrs(void)
2174 {
2175         /*
2176          * Note that as a general rule, the high half of the MSRs (bits in
2177          * the control fields which may be 1) should be initialized by the
2178          * intersection of the underlying hardware's MSR (i.e., features which
2179          * can be supported) and the list of features we want to expose -
2180          * because they are known to be properly supported in our code.
2181          * Also, usually, the low half of the MSRs (bits which must be 1) can
2182          * be set to 0, meaning that L1 may turn off any of these bits. The
2183          * reason is that if one of these bits is necessary, it will appear
2184          * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control
2185          * fields of vmcs01 and vmcs02, will turn these bits off - and
2186          * nested_vmx_exit_handled() will not pass related exits to L1.
2187          * These rules have exceptions below.
2188          */
2189 
2190         /* pin-based controls */
2191         rdmsr(MSR_IA32_VMX_PINBASED_CTLS,
2192               nested_vmx_pinbased_ctls_low, nested_vmx_pinbased_ctls_high);
2193         /*
2194          * According to the Intel spec, if bit 55 of VMX_BASIC is off (as it is
2195          * in our case), bits 1, 2 and 4 (i.e., 0x16) must be 1 in this MSR.
2196          */
2197         nested_vmx_pinbased_ctls_low |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
2198         nested_vmx_pinbased_ctls_high &= PIN_BASED_EXT_INTR_MASK |
2199                 PIN_BASED_NMI_EXITING | PIN_BASED_VIRTUAL_NMIS |
2200                 PIN_BASED_VMX_PREEMPTION_TIMER;
2201         nested_vmx_pinbased_ctls_high |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
2202 
2203         /*
2204          * Exit controls
2205          * If bit 55 of VMX_BASIC is off, bits 0-8 and 10, 11, 13, 14, 16 and
2206          * 17 must be 1.
2207          */
2208         rdmsr(MSR_IA32_VMX_EXIT_CTLS,
2209                 nested_vmx_exit_ctls_low, nested_vmx_exit_ctls_high);
2210         nested_vmx_exit_ctls_low = VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
2211         /* Note that guest use of VM_EXIT_ACK_INTR_ON_EXIT is not supported. */
2212         nested_vmx_exit_ctls_high &=
2213 #ifdef CONFIG_X86_64
2214                 VM_EXIT_HOST_ADDR_SPACE_SIZE |
2215 #endif
2216                 VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT;
2217         nested_vmx_exit_ctls_high |= (VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |
2218                                       VM_EXIT_LOAD_IA32_EFER);
2219 
2220         /* entry controls */
2221         rdmsr(MSR_IA32_VMX_ENTRY_CTLS,
2222                 nested_vmx_entry_ctls_low, nested_vmx_entry_ctls_high);
2223         /* If bit 55 of VMX_BASIC is off, bits 0-8 and 12 must be 1. */
2224         nested_vmx_entry_ctls_low = VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
2225         nested_vmx_entry_ctls_high &=
2226 #ifdef CONFIG_X86_64
2227                 VM_ENTRY_IA32E_MODE |
2228 #endif
2229                 VM_ENTRY_LOAD_IA32_PAT;
2230         nested_vmx_entry_ctls_high |= (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR |
2231                                        VM_ENTRY_LOAD_IA32_EFER);
2232 
2233         /* cpu-based controls */
2234         rdmsr(MSR_IA32_VMX_PROCBASED_CTLS,
2235                 nested_vmx_procbased_ctls_low, nested_vmx_procbased_ctls_high);
2236         nested_vmx_procbased_ctls_low = 0;
2237         nested_vmx_procbased_ctls_high &=
2238                 CPU_BASED_VIRTUAL_INTR_PENDING | CPU_BASED_USE_TSC_OFFSETING |
2239                 CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING |
2240                 CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING |
2241                 CPU_BASED_CR3_STORE_EXITING |
2242 #ifdef CONFIG_X86_64
2243                 CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING |
2244 #endif
2245                 CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING |
2246                 CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_EXITING |
2247                 CPU_BASED_RDPMC_EXITING | CPU_BASED_RDTSC_EXITING |
2248                 CPU_BASED_PAUSE_EXITING |
2249                 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
2250         /*
2251          * We can allow some features even when not supported by the
2252          * hardware. For example, L1 can specify an MSR bitmap - and we
2253          * can use it to avoid exits to L1 - even when L0 runs L2
2254          * without MSR bitmaps.
2255          */
2256         nested_vmx_procbased_ctls_high |= CPU_BASED_USE_MSR_BITMAPS;
2257 
2258         /* secondary cpu-based controls */
2259         rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2,
2260                 nested_vmx_secondary_ctls_low, nested_vmx_secondary_ctls_high);
2261         nested_vmx_secondary_ctls_low = 0;
2262         nested_vmx_secondary_ctls_high &=
2263                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
2264                 SECONDARY_EXEC_WBINVD_EXITING;
2265 
2266         if (enable_ept) {
2267                 /* nested EPT: emulate EPT also to L1 */
2268                 nested_vmx_secondary_ctls_high |= SECONDARY_EXEC_ENABLE_EPT;
2269                 nested_vmx_ept_caps = VMX_EPT_PAGE_WALK_4_BIT |
2270                          VMX_EPTP_WB_BIT | VMX_EPT_INVEPT_BIT;
2271                 nested_vmx_ept_caps &= vmx_capability.ept;
2272                 /*
2273                  * Since invept is completely emulated we support both global
2274                  * and context invalidation independent of what host cpu
2275                  * supports
2276                  */
2277                 nested_vmx_ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT |
2278                         VMX_EPT_EXTENT_CONTEXT_BIT;
2279         } else
2280                 nested_vmx_ept_caps = 0;
2281 
2282         /* miscellaneous data */
2283         rdmsr(MSR_IA32_VMX_MISC, nested_vmx_misc_low, nested_vmx_misc_high);
2284         nested_vmx_misc_low &= VMX_MISC_PREEMPTION_TIMER_RATE_MASK |
2285                 VMX_MISC_SAVE_EFER_LMA;
2286         nested_vmx_misc_high = 0;
2287 }
2288 
2289 static inline bool vmx_control_verify(u32 control, u32 low, u32 high)
2290 {
2291         /*
2292          * Bits 0 in high must be 0, and bits 1 in low must be 1.
2293          */
2294         return ((control & high) | low) == control;
2295 }
2296 
2297 static inline u64 vmx_control_msr(u32 low, u32 high)
2298 {
2299         return low | ((u64)high << 32);
2300 }
2301 
2302 /*
2303  * If we allow our guest to use VMX instructions (i.e., nested VMX), we should
2304  * also let it use VMX-specific MSRs.
2305  * vmx_get_vmx_msr() and vmx_set_vmx_msr() return 1 when we handled a
2306  * VMX-specific MSR, or 0 when we haven't (and the caller should handle it
2307  * like all other MSRs).
2308  */
2309 static int vmx_get_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
2310 {
2311         if (!nested_vmx_allowed(vcpu) && msr_index >= MSR_IA32_VMX_BASIC &&
2312                      msr_index <= MSR_IA32_VMX_TRUE_ENTRY_CTLS) {
2313                 /*
2314                  * According to the spec, processors which do not support VMX
2315                  * should throw a #GP(0) when VMX capability MSRs are read.
2316                  */
2317                 kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
2318                 return 1;
2319         }
2320 
2321         switch (msr_index) {
2322         case MSR_IA32_FEATURE_CONTROL:
2323                 if (nested_vmx_allowed(vcpu)) {
2324                         *pdata = to_vmx(vcpu)->nested.msr_ia32_feature_control;
2325                         break;
2326                 }
2327                 return 0;
2328         case MSR_IA32_VMX_BASIC:
2329                 /*
2330                  * This MSR reports some information about VMX support. We
2331                  * should return information about the VMX we emulate for the
2332                  * guest, and the VMCS structure we give it - not about the
2333                  * VMX support of the underlying hardware.
2334                  */
2335                 *pdata = VMCS12_REVISION |
2336                            ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) |
2337                            (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT);
2338                 break;
2339         case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
2340         case MSR_IA32_VMX_PINBASED_CTLS:
2341                 *pdata = vmx_control_msr(nested_vmx_pinbased_ctls_low,
2342                                         nested_vmx_pinbased_ctls_high);
2343                 break;
2344         case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
2345         case MSR_IA32_VMX_PROCBASED_CTLS:
2346                 *pdata = vmx_control_msr(nested_vmx_procbased_ctls_low,
2347                                         nested_vmx_procbased_ctls_high);
2348                 break;
2349         case MSR_IA32_VMX_TRUE_EXIT_CTLS:
2350         case MSR_IA32_VMX_EXIT_CTLS:
2351                 *pdata = vmx_control_msr(nested_vmx_exit_ctls_low,
2352                                         nested_vmx_exit_ctls_high);
2353                 break;
2354         case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
2355         case MSR_IA32_VMX_ENTRY_CTLS:
2356                 *pdata = vmx_control_msr(nested_vmx_entry_ctls_low,
2357                                         nested_vmx_entry_ctls_high);
2358                 break;
2359         case MSR_IA32_VMX_MISC:
2360                 *pdata = vmx_control_msr(nested_vmx_misc_low,
2361                                          nested_vmx_misc_high);
2362                 break;
2363         /*
2364          * These MSRs specify bits which the guest must keep fixed (on or off)
2365          * while L1 is in VMXON mode (in L1's root mode, or running an L2).
2366          * We picked the standard core2 setting.
2367          */
2368 #define VMXON_CR0_ALWAYSON      (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE)
2369 #define VMXON_CR4_ALWAYSON      X86_CR4_VMXE
2370         case MSR_IA32_VMX_CR0_FIXED0:
2371                 *pdata = VMXON_CR0_ALWAYSON;
2372                 break;
2373         case MSR_IA32_VMX_CR0_FIXED1:
2374                 *pdata = -1ULL;
2375                 break;
2376         case MSR_IA32_VMX_CR4_FIXED0:
2377                 *pdata = VMXON_CR4_ALWAYSON;
2378                 break;
2379         case MSR_IA32_VMX_CR4_FIXED1:
2380                 *pdata = -1ULL;
2381                 break;
2382         case MSR_IA32_VMX_VMCS_ENUM:
2383                 *pdata = 0x1f;
2384                 break;
2385         case MSR_IA32_VMX_PROCBASED_CTLS2:
2386                 *pdata = vmx_control_msr(nested_vmx_secondary_ctls_low,
2387                                         nested_vmx_secondary_ctls_high);
2388                 break;
2389         case MSR_IA32_VMX_EPT_VPID_CAP:
2390                 /* Currently, no nested vpid support */
2391                 *pdata = nested_vmx_ept_caps;
2392                 break;
2393         default:
2394                 return 0;
2395         }
2396 
2397         return 1;
2398 }
2399 
2400 static int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2401 {
2402         u32 msr_index = msr_info->index;
2403         u64 data = msr_info->data;
2404         bool host_initialized = msr_info->host_initiated;
2405 
2406         if (!nested_vmx_allowed(vcpu))
2407                 return 0;
2408 
2409         if (msr_index == MSR_IA32_FEATURE_CONTROL) {
2410                 if (!host_initialized &&
2411                                 to_vmx(vcpu)->nested.msr_ia32_feature_control
2412                                 & FEATURE_CONTROL_LOCKED)
2413                         return 0;
2414                 to_vmx(vcpu)->nested.msr_ia32_feature_control = data;
2415                 return 1;
2416         }
2417 
2418         /*
2419          * No need to treat VMX capability MSRs specially: If we don't handle
2420          * them, handle_wrmsr will #GP(0), which is correct (they are readonly)
2421          */
2422         return 0;
2423 }
2424 
2425 /*
2426  * Reads an msr value (of 'msr_index') into 'pdata'.
2427  * Returns 0 on success, non-0 otherwise.
2428  * Assumes vcpu_load() was already called.
2429  */
2430 static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
2431 {
2432         u64 data;
2433         struct shared_msr_entry *msr;
2434 
2435         if (!pdata) {
2436                 printk(KERN_ERR "BUG: get_msr called with NULL pdata\n");
2437                 return -EINVAL;
2438         }
2439 
2440         switch (msr_index) {
2441 #ifdef CONFIG_X86_64
2442         case MSR_FS_BASE:
2443                 data = vmcs_readl(GUEST_FS_BASE);
2444                 break;
2445         case MSR_GS_BASE:
2446                 data = vmcs_readl(GUEST_GS_BASE);
2447                 break;
2448         case MSR_KERNEL_GS_BASE:
2449                 vmx_load_host_state(to_vmx(vcpu));
2450                 data = to_vmx(vcpu)->msr_guest_kernel_gs_base;
2451                 break;
2452 #endif
2453         case MSR_EFER:
2454                 return kvm_get_msr_common(vcpu, msr_index, pdata);
2455         case MSR_IA32_TSC:
2456                 data = guest_read_tsc();
2457                 break;
2458         case MSR_IA32_SYSENTER_CS:
2459                 data = vmcs_read32(GUEST_SYSENTER_CS);
2460                 break;
2461         case MSR_IA32_SYSENTER_EIP:
2462                 data = vmcs_readl(GUEST_SYSENTER_EIP);
2463                 break;
2464         case MSR_IA32_SYSENTER_ESP:
2465                 data = vmcs_readl(GUEST_SYSENTER_ESP);
2466                 break;
2467         case MSR_TSC_AUX:
2468                 if (!to_vmx(vcpu)->rdtscp_enabled)
2469                         return 1;
2470                 /* Otherwise falls through */
2471         default:
2472                 if (vmx_get_vmx_msr(vcpu, msr_index, pdata))
2473                         return 0;
2474                 msr = find_msr_entry(to_vmx(vcpu), msr_index);
2475                 if (msr) {
2476                         data = msr->data;
2477                         break;
2478                 }
2479                 return kvm_get_msr_common(vcpu, msr_index, pdata);
2480         }
2481 
2482         *pdata = data;
2483         return 0;
2484 }
2485 
2486 /*
2487  * Writes msr value into into the appropriate "register".
2488  * Returns 0 on success, non-0 otherwise.
2489  * Assumes vcpu_load() was already called.
2490  */
2491 static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2492 {
2493         struct vcpu_vmx *vmx = to_vmx(vcpu);
2494         struct shared_msr_entry *msr;
2495         int ret = 0;
2496         u32 msr_index = msr_info->index;
2497         u64 data = msr_info->data;
2498 
2499         switch (msr_index) {
2500         case MSR_EFER:
2501                 ret = kvm_set_msr_common(vcpu, msr_info);
2502                 break;
2503 #ifdef CONFIG_X86_64
2504         case MSR_FS_BASE:
2505                 vmx_segment_cache_clear(vmx);
2506                 vmcs_writel(GUEST_FS_BASE, data);
2507                 break;
2508         case MSR_GS_BASE:
2509                 vmx_segment_cache_clear(vmx);
2510                 vmcs_writel(GUEST_GS_BASE, data);
2511                 break;
2512         case MSR_KERNEL_GS_BASE:
2513                 vmx_load_host_state(vmx);
2514                 vmx->msr_guest_kernel_gs_base = data;
2515                 break;
2516 #endif
2517         case MSR_IA32_SYSENTER_CS:
2518                 vmcs_write32(GUEST_SYSENTER_CS, data);
2519                 break;
2520         case MSR_IA32_SYSENTER_EIP:
2521                 vmcs_writel(GUEST_SYSENTER_EIP, data);
2522                 break;
2523         case MSR_IA32_SYSENTER_ESP:
2524                 vmcs_writel(GUEST_SYSENTER_ESP, data);
2525                 break;
2526         case MSR_IA32_TSC:
2527                 kvm_write_tsc(vcpu, msr_info);
2528                 break;
2529         case MSR_IA32_CR_PAT:
2530                 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
2531                         if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
2532                                 return 1;
2533                         vmcs_write64(GUEST_IA32_PAT, data);
2534                         vcpu->arch.pat = data;
2535                         break;
2536                 }
2537                 ret = kvm_set_msr_common(vcpu, msr_info);
2538                 break;
2539         case MSR_IA32_TSC_ADJUST:
2540                 ret = kvm_set_msr_common(vcpu, msr_info);
2541                 break;
2542         case MSR_TSC_AUX:
2543                 if (!vmx->rdtscp_enabled)
2544                         return 1;
2545                 /* Check reserved bit, higher 32 bits should be zero */
2546                 if ((data >> 32) != 0)
2547                         return 1;
2548                 /* Otherwise falls through */
2549         default:
2550                 if (vmx_set_vmx_msr(vcpu, msr_info))
2551                         break;
2552                 msr = find_msr_entry(vmx, msr_index);
2553                 if (msr) {
2554                         u64 old_msr_data = msr->data;
2555                         msr->data = data;
2556                         if (msr - vmx->guest_msrs < vmx->save_nmsrs) {
2557                                 preempt_disable();
2558                                 ret = kvm_set_shared_msr(msr->index, msr->data,
2559                                                          msr->mask);
2560                                 preempt_enable();
2561                                 if (ret)
2562                                         msr->data = old_msr_data;
2563                         }
2564                         break;
2565                 }
2566                 ret = kvm_set_msr_common(vcpu, msr_info);
2567         }
2568 
2569         return ret;
2570 }
2571 
2572 static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
2573 {
2574         __set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
2575         switch (reg) {
2576         case VCPU_REGS_RSP:
2577                 vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
2578                 break;
2579         case VCPU_REGS_RIP:
2580                 vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);
2581                 break;
2582         case VCPU_EXREG_PDPTR:
2583                 if (enable_ept)
2584                         ept_save_pdptrs(vcpu);
2585                 break;
2586         default:
2587                 break;
2588         }
2589 }
2590 
2591 static __init int cpu_has_kvm_support(void)
2592 {
2593         return cpu_has_vmx();
2594 }
2595 
2596 static __init int vmx_disabled_by_bios(void)
2597 {
2598         u64 msr;
2599 
2600         rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
2601         if (msr & FEATURE_CONTROL_LOCKED) {
2602                 /* launched w/ TXT and VMX disabled */
2603                 if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
2604                         && tboot_enabled())
2605                         return 1;
2606                 /* launched w/o TXT and VMX only enabled w/ TXT */
2607                 if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
2608                         && (msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
2609                         && !tboot_enabled()) {
2610                         printk(KERN_WARNING "kvm: disable TXT in the BIOS or "
2611                                 "activate TXT before enabling KVM\n");
2612                         return 1;
2613                 }
2614                 /* launched w/o TXT and VMX disabled */
2615                 if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
2616                         && !tboot_enabled())
2617                         return 1;
2618         }
2619 
2620         return 0;
2621 }
2622 
2623 static void kvm_cpu_vmxon(u64 addr)
2624 {
2625         asm volatile (ASM_VMX_VMXON_RAX
2626                         : : "a"(&addr), "m"(addr)
2627                         : "memory", "cc");
2628 }
2629 
2630 static int hardware_enable(void *garbage)
2631 {
2632         int cpu = raw_smp_processor_id();
2633         u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
2634         u64 old, test_bits;
2635 
2636         if (read_cr4() & X86_CR4_VMXE)
2637                 return -EBUSY;
2638 
2639         INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu));
2640 
2641         /*
2642          * Now we can enable the vmclear operation in kdump
2643          * since the loaded_vmcss_on_cpu list on this cpu
2644          * has been initialized.
2645          *
2646          * Though the cpu is not in VMX operation now, there
2647          * is no problem to enable the vmclear operation
2648          * for the loaded_vmcss_on_cpu list is empty!
2649          */
2650         crash_enable_local_vmclear(cpu);
2651 
2652         rdmsrl(MSR_IA32_FEATURE_CONTROL, old);
2653 
2654         test_bits = FEATURE_CONTROL_LOCKED;
2655         test_bits |= FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
2656         if (tboot_enabled())
2657                 test_bits |= FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX;
2658 
2659         if ((old & test_bits) != test_bits) {
2660                 /* enable and lock */
2661                 wrmsrl(MSR_IA32_FEATURE_CONTROL, old | test_bits);
2662         }
2663         write_cr4(read_cr4() | X86_CR4_VMXE); /* FIXME: not cpu hotplug safe */
2664 
2665         if (vmm_exclusive) {
2666                 kvm_cpu_vmxon(phys_addr);
2667                 ept_sync_global();
2668         }
2669 
2670         native_store_gdt(&__get_cpu_var(host_gdt));
2671 
2672         return 0;
2673 }
2674 
2675 static void vmclear_local_loaded_vmcss(void)
2676 {
2677         int cpu = raw_smp_processor_id();
2678         struct loaded_vmcs *v, *n;
2679 
2680         list_for_each_entry_safe(v, n, &per_cpu(loaded_vmcss_on_cpu, cpu),
2681                                  loaded_vmcss_on_cpu_link)
2682                 __loaded_vmcs_clear(v);
2683 }
2684 
2685 
2686 /* Just like cpu_vmxoff(), but with the __kvm_handle_fault_on_reboot()
2687  * tricks.
2688  */
2689 static void kvm_cpu_vmxoff(void)
2690 {
2691         asm volatile (__ex(ASM_VMX_VMXOFF) : : : "cc");
2692 }
2693 
2694 static void hardware_disable(void *garbage)
2695 {
2696         if (vmm_exclusive) {
2697                 vmclear_local_loaded_vmcss();
2698                 kvm_cpu_vmxoff();
2699         }
2700         write_cr4(read_cr4() & ~X86_CR4_VMXE);
2701 }
2702 
2703 static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
2704                                       u32 msr, u32 *result)
2705 {
2706         u32 vmx_msr_low, vmx_msr_high;
2707         u32 ctl = ctl_min | ctl_opt;
2708 
2709         rdmsr(msr, vmx_msr_low, vmx_msr_high);
2710 
2711         ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */
2712         ctl |= vmx_msr_low;  /* bit == 1 in low word  ==> must be one  */
2713 
2714         /* Ensure minimum (required) set of control bits are supported. */
2715         if (ctl_min & ~ctl)
2716                 return -EIO;
2717 
2718         *result = ctl;
2719         return 0;
2720 }
2721 
2722 static __init bool allow_1_setting(u32 msr, u32 ctl)
2723 {
2724         u32 vmx_msr_low, vmx_msr_high;
2725 
2726         rdmsr(msr, vmx_msr_low, vmx_msr_high);
2727         return vmx_msr_high & ctl;
2728 }
2729 
2730 static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
2731 {
2732         u32 vmx_msr_low, vmx_msr_high;
2733         u32 min, opt, min2, opt2;
2734         u32 _pin_based_exec_control = 0;
2735         u32 _cpu_based_exec_control = 0;
2736         u32 _cpu_based_2nd_exec_control = 0;
2737         u32 _vmexit_control = 0;
2738         u32 _vmentry_control = 0;
2739 
2740         min = CPU_BASED_HLT_EXITING |
2741 #ifdef CONFIG_X86_64
2742               CPU_BASED_CR8_LOAD_EXITING |
2743               CPU_BASED_CR8_STORE_EXITING |
2744 #endif
2745               CPU_BASED_CR3_LOAD_EXITING |
2746               CPU_BASED_CR3_STORE_EXITING |
2747               CPU_BASED_USE_IO_BITMAPS |
2748               CPU_BASED_MOV_DR_EXITING |
2749               CPU_BASED_USE_TSC_OFFSETING |
2750               CPU_BASED_MWAIT_EXITING |
2751               CPU_BASED_MONITOR_EXITING |
2752               CPU_BASED_INVLPG_EXITING |
2753               CPU_BASED_RDPMC_EXITING;
2754 
2755         opt = CPU_BASED_TPR_SHADOW |
2756               CPU_BASED_USE_MSR_BITMAPS |
2757               CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
2758         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
2759                                 &_cpu_based_exec_control) < 0)
2760                 return -EIO;
2761 #ifdef CONFIG_X86_64
2762         if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
2763                 _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING &
2764                                            ~CPU_BASED_CR8_STORE_EXITING;
2765 #endif
2766         if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) {
2767                 min2 = 0;
2768                 opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
2769                         SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
2770                         SECONDARY_EXEC_WBINVD_EXITING |
2771                         SECONDARY_EXEC_ENABLE_VPID |
2772                         SECONDARY_EXEC_ENABLE_EPT |
2773                         SECONDARY_EXEC_UNRESTRICTED_GUEST |
2774                         SECONDARY_EXEC_PAUSE_LOOP_EXITING |
2775                         SECONDARY_EXEC_RDTSCP |
2776                         SECONDARY_EXEC_ENABLE_INVPCID |
2777                         SECONDARY_EXEC_APIC_REGISTER_VIRT |
2778                         SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
2779                         SECONDARY_EXEC_SHADOW_VMCS;
2780                 if (adjust_vmx_controls(min2, opt2,
2781                                         MSR_IA32_VMX_PROCBASED_CTLS2,
2782                                         &_cpu_based_2nd_exec_control) < 0)
2783                         return -EIO;
2784         }
2785 #ifndef CONFIG_X86_64
2786         if (!(_cpu_based_2nd_exec_control &
2787                                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
2788                 _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW;
2789 #endif
2790 
2791         if (!(_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
2792                 _cpu_based_2nd_exec_control &= ~(
2793                                 SECONDARY_EXEC_APIC_REGISTER_VIRT |
2794                                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
2795                                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
2796 
2797         if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) {
2798                 /* CR3 accesses and invlpg don't need to cause VM Exits when EPT
2799                    enabled */
2800                 _cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING |
2801                                              CPU_BASED_CR3_STORE_EXITING |
2802                                              CPU_BASED_INVLPG_EXITING);
2803                 rdmsr(MSR_IA32_VMX_EPT_VPID_CAP,
2804                       vmx_capability.ept, vmx_capability.vpid);
2805         }
2806 
2807         min = 0;
2808 #ifdef CONFIG_X86_64
2809         min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
2810 #endif
2811         opt = VM_EXIT_SAVE_IA32_PAT | VM_EXIT_LOAD_IA32_PAT |
2812                 VM_EXIT_ACK_INTR_ON_EXIT;
2813         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,
2814                                 &_vmexit_control) < 0)
2815                 return -EIO;
2816 
2817         min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
2818         opt = PIN_BASED_VIRTUAL_NMIS | PIN_BASED_POSTED_INTR;
2819         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS,
2820                                 &_pin_based_exec_control) < 0)
2821                 return -EIO;
2822 
2823         if (!(_cpu_based_2nd_exec_control &
2824                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) ||
2825                 !(_vmexit_control & VM_EXIT_ACK_INTR_ON_EXIT))
2826                 _pin_based_exec_control &= ~PIN_BASED_POSTED_INTR;
2827 
2828         min = 0;
2829         opt = VM_ENTRY_LOAD_IA32_PAT;
2830         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
2831                                 &_vmentry_control) < 0)
2832                 return -EIO;
2833 
2834         rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
2835 
2836         /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
2837         if ((vmx_msr_high & 0x1fff) > PAGE_SIZE)
2838                 return -EIO;
2839 
2840 #ifdef CONFIG_X86_64
2841         /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */
2842         if (vmx_msr_high & (1u<<16))
2843                 return -EIO;
2844 #endif
2845 
2846         /* Require Write-Back (WB) memory type for VMCS accesses. */
2847         if (((vmx_msr_high >> 18) & 15) != 6)
2848                 return -EIO;
2849 
2850         vmcs_conf->size = vmx_msr_high & 0x1fff;
2851         vmcs_conf->order = get_order(vmcs_config.size);
2852         vmcs_conf->revision_id = vmx_msr_low;
2853 
2854         vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control;
2855         vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control;
2856         vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control;
2857         vmcs_conf->vmexit_ctrl         = _vmexit_control;
2858         vmcs_conf->vmentry_ctrl        = _vmentry_control;
2859 
2860         cpu_has_load_ia32_efer =
2861                 allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS,
2862                                 VM_ENTRY_LOAD_IA32_EFER)
2863                 && allow_1_setting(MSR_IA32_VMX_EXIT_CTLS,
2864                                    VM_EXIT_LOAD_IA32_EFER);
2865 
2866         cpu_has_load_perf_global_ctrl =
2867                 allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS,
2868                                 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL)
2869                 && allow_1_setting(MSR_IA32_VMX_EXIT_CTLS,
2870                                    VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);
2871 
2872         /*
2873          * Some cpus support VM_ENTRY_(LOAD|SAVE)_IA32_PERF_GLOBAL_CTRL
2874          * but due to arrata below it can't be used. Workaround is to use
2875          * msr load mechanism to switch IA32_PERF_GLOBAL_CTRL.
2876          *
2877          * VM Exit May Incorrectly Clear IA32_PERF_GLOBAL_CTRL [34:32]
2878          *
2879          * AAK155             (model 26)
2880          * AAP115             (model 30)
2881          * AAT100             (model 37)
2882          * BC86,AAY89,BD102   (model 44)
2883          * BA97               (model 46)
2884          *
2885          */
2886         if (cpu_has_load_perf_global_ctrl && boot_cpu_data.x86 == 0x6) {
2887                 switch (boot_cpu_data.x86_model) {
2888                 case 26:
2889                 case 30:
2890                 case 37:
2891                 case 44:
2892                 case 46:
2893                         cpu_has_load_perf_global_ctrl = false;
2894                         printk_once(KERN_WARNING"kvm: VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL "
2895                                         "does not work properly. Using workaround\n");
2896                         break;
2897                 default:
2898                         break;
2899                 }
2900         }
2901 
2902         return 0;
2903 }
2904 
2905 static struct vmcs *alloc_vmcs_cpu(int cpu)
2906 {
2907         int node = cpu_to_node(cpu);
2908         struct page *pages;
2909         struct vmcs *vmcs;
2910 
2911         pages = alloc_pages_exact_node(node, GFP_KERNEL, vmcs_config.order);
2912         if (!pages)
2913                 return NULL;
2914         vmcs = page_address(pages);
2915         memset(vmcs, 0, vmcs_config.size);
2916         vmcs->revision_id = vmcs_config.revision_id; /* vmcs revision id */
2917         return vmcs;
2918 }
2919 
2920 static struct vmcs *alloc_vmcs(void)
2921 {
2922         return alloc_vmcs_cpu(raw_smp_processor_id());
2923 }
2924 
2925 static void free_vmcs(struct vmcs *vmcs)
2926 {
2927         free_pages((unsigned long)vmcs, vmcs_config.order);
2928 }
2929 
2930 /*
2931  * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded
2932  */
2933 static void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
2934 {
2935         if (!loaded_vmcs->vmcs)
2936                 return;
2937         loaded_vmcs_clear(loaded_vmcs);
2938         free_vmcs(loaded_vmcs->vmcs);
2939         loaded_vmcs->vmcs = NULL;
2940 }
2941 
2942 static void free_kvm_area(void)
2943 {
2944         int cpu;
2945 
2946         for_each_possible_cpu(cpu) {
2947                 free_vmcs(per_cpu(vmxarea, cpu));
2948                 per_cpu(vmxarea, cpu) = NULL;
2949         }
2950 }
2951 
2952 static __init int alloc_kvm_area(void)
2953 {
2954         int cpu;
2955 
2956         for_each_possible_cpu(cpu) {
2957                 struct vmcs *vmcs;
2958 
2959                 vmcs = alloc_vmcs_cpu(cpu);
2960                 if (!vmcs) {
2961                         free_kvm_area();
2962                         return -ENOMEM;
2963                 }
2964 
2965                 per_cpu(vmxarea, cpu) = vmcs;
2966         }
2967         return 0;
2968 }
2969 
2970 static __init int hardware_setup(void)
2971 {
2972         if (setup_vmcs_config(&vmcs_config) < 0)
2973                 return -EIO;
2974 
2975         if (boot_cpu_has(X86_FEATURE_NX))
2976                 kvm_enable_efer_bits(EFER_NX);
2977 
2978         if (!cpu_has_vmx_vpid())
2979                 enable_vpid = 0;
2980         if (!cpu_has_vmx_shadow_vmcs())
2981                 enable_shadow_vmcs = 0;
2982 
2983         if (!cpu_has_vmx_ept() ||
2984             !cpu_has_vmx_ept_4levels()) {
2985                 enable_ept = 0;
2986                 enable_unrestricted_guest = 0;
2987                 enable_ept_ad_bits = 0;
2988         }
2989 
2990         if (!cpu_has_vmx_ept_ad_bits())
2991                 enable_ept_ad_bits = 0;
2992 
2993         if (!cpu_has_vmx_unrestricted_guest())
2994                 enable_unrestricted_guest = 0;
2995 
2996         if (!cpu_has_vmx_flexpriority())
2997                 flexpriority_enabled = 0;
2998 
2999         if (!cpu_has_vmx_tpr_shadow())
3000                 kvm_x86_ops->update_cr8_intercept = NULL;
3001 
3002         if (enable_ept && !cpu_has_vmx_ept_2m_page())
3003                 kvm_disable_largepages();
3004 
3005         if (!cpu_has_vmx_ple())
3006                 ple_gap = 0;
3007 
3008         if (!cpu_has_vmx_apicv())
3009                 enable_apicv = 0;
3010 
3011         if (enable_apicv)
3012                 kvm_x86_ops->update_cr8_intercept = NULL;
3013         else {
3014                 kvm_x86_ops->hwapic_irr_update = NULL;
3015                 kvm_x86_ops->deliver_posted_interrupt = NULL;
3016                 kvm_x86_ops->sync_pir_to_irr = vmx_sync_pir_to_irr_dummy;
3017         }
3018 
3019         if (nested)
3020                 nested_vmx_setup_ctls_msrs();
3021 
3022         return alloc_kvm_area();
3023 }
3024 
3025 static __exit void hardware_unsetup(void)
3026 {
3027         free_kvm_area();
3028 }
3029 
3030 static bool emulation_required(struct kvm_vcpu *vcpu)
3031 {
3032         return emulate_invalid_guest_state && !guest_state_valid(vcpu);
3033 }
3034 
3035 static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg,
3036                 struct kvm_segment *save)
3037 {
3038         if (!emulate_invalid_guest_state) {
3039                 /*
3040                  * CS and SS RPL should be equal during guest entry according
3041                  * to VMX spec, but in reality it is not always so. Since vcpu
3042                  * is in the middle of the transition from real mode to
3043                  * protected mode it is safe to assume that RPL 0 is a good
3044                  * default value.
3045                  */
3046                 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS)
3047                         save->selector &= ~SELECTOR_RPL_MASK;
3048                 save->dpl = save->selector & SELECTOR_RPL_MASK;
3049                 save->s = 1;
3050         }
3051         vmx_set_segment(vcpu, save, seg);
3052 }
3053 
3054 static void enter_pmode(struct kvm_vcpu *vcpu)
3055 {
3056         unsigned long flags;
3057         struct vcpu_vmx *vmx = to_vmx(vcpu);
3058 
3059         /*
3060          * Update real mode segment cache. It may be not up-to-date if sement
3061          * register was written while vcpu was in a guest mode.
3062          */
3063         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
3064         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
3065         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
3066         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
3067         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
3068         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
3069 
3070         vmx->rmode.vm86_active = 0;
3071 
3072         vmx_segment_cache_clear(vmx);
3073 
3074         vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
3075 
3076         flags = vmcs_readl(GUEST_RFLAGS);
3077         flags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
3078         flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
3079         vmcs_writel(GUEST_RFLAGS, flags);
3080 
3081         vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
3082                         (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
3083 
3084         update_exception_bitmap(vcpu);
3085 
3086         fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
3087         fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
3088         fix_pmode_seg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
3089         fix_pmode_seg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
3090         fix_pmode_seg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
3091         fix_pmode_seg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
3092 
3093         /* CPL is always 0 when CPU enters protected mode */
3094         __set_bit(VCPU_EXREG_CPL, (ulong *)&vcpu->arch.regs_avail);
3095         vmx->cpl = 0;
3096 }
3097 
3098 static void fix_rmode_seg(int seg, struct kvm_segment *save)
3099 {
3100         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
3101         struct kvm_segment var = *save;
3102 
3103         var.dpl = 0x3;
3104         if (seg == VCPU_SREG_CS)
3105                 var.type = 0x3;
3106 
3107         if (!emulate_invalid_guest_state) {
3108                 var.selector = var.base >> 4;
3109                 var.base = var.base & 0xffff0;
3110                 var.limit = 0xffff;
3111                 var.g = 0;
3112                 var.db = 0;
3113                 var.present = 1;
3114                 var.s = 1;
3115                 var.l = 0;
3116                 var.unusable = 0;
3117                 var.type = 0x3;
3118                 var.avl = 0;
3119                 if (save->base & 0xf)
3120                         printk_once(KERN_WARNING "kvm: segment base is not "
3121                                         "paragraph aligned when entering "
3122                                         "protected mode (seg=%d)", seg);
3123         }
3124 
3125         vmcs_write16(sf->selector, var.selector);
3126         vmcs_writel(sf->base, var.base);
3127         vmcs_write32(sf->limit, var.limit);
3128         vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(&var));
3129 }
3130 
3131 static void enter_rmode(struct kvm_vcpu *vcpu)
3132 {
3133         unsigned long flags;
3134         struct vcpu_vmx *vmx = to_vmx(vcpu);
3135 
3136         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
3137         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
3138         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
3139         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
3140         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
3141         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
3142         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
3143 
3144         vmx->rmode.vm86_active = 1;
3145 
3146         /*
3147          * Very old userspace does not call KVM_SET_TSS_ADDR before entering
3148          * vcpu. Warn the user that an update is overdue.
3149          */
3150         if (!vcpu->kvm->arch.tss_addr)
3151                 printk_once(KERN_WARNING "kvm: KVM_SET_TSS_ADDR need to be "
3152                              "called before entering vcpu\n");
3153 
3154         vmx_segment_cache_clear(vmx);
3155 
3156         vmcs_writel(GUEST_TR_BASE, vcpu->kvm->arch.tss_addr);
3157         vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
3158         vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
3159 
3160         flags = vmcs_readl(GUEST_RFLAGS);
3161         vmx->rmode.save_rflags = flags;
3162 
3163         flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
3164 
3165         vmcs_writel(GUEST_RFLAGS, flags);
3166         vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
3167         update_exception_bitmap(vcpu);
3168 
3169         fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
3170         fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
3171         fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
3172         fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
3173         fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
3174         fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
3175 
3176         kvm_mmu_reset_context(vcpu);
3177 }
3178 
3179 static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
3180 {
3181         struct vcpu_vmx *vmx = to_vmx(vcpu);
3182         struct shared_msr_entry *msr = find_msr_entry(vmx, MSR_EFER);
3183 
3184         if (!msr)
3185                 return;
3186 
3187         /*
3188          * Force kernel_gs_base reloading before EFER changes, as control
3189          * of this msr depends on is_long_mode().
3190          */
3191         vmx_load_host_state(to_vmx(vcpu));
3192         vcpu->arch.efer = efer;
3193         if (efer & EFER_LMA) {
3194                 vmcs_write32(VM_ENTRY_CONTROLS,
3195                              vmcs_read32(VM_ENTRY_CONTROLS) |
3196                              VM_ENTRY_IA32E_MODE);
3197                 msr->data = efer;
3198         } else {
3199                 vmcs_write32(VM_ENTRY_CONTROLS,
3200                              vmcs_read32(VM_ENTRY_CONTROLS) &
3201                              ~VM_ENTRY_IA32E_MODE);
3202 
3203                 msr->data = efer & ~EFER_LME;
3204         }
3205         setup_msrs(vmx);
3206 }
3207 
3208 #ifdef CONFIG_X86_64
3209 
3210 static void enter_lmode(struct kvm_vcpu *vcpu)
3211 {
3212         u32 guest_tr_ar;
3213 
3214         vmx_segment_cache_clear(to_vmx(vcpu));
3215 
3216         guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
3217         if ((guest_tr_ar & AR_TYPE_MASK) != AR_TYPE_BUSY_64_TSS) {
3218                 pr_debug_ratelimited("%s: tss fixup for long mode. \n",
3219                                      __func__);
3220                 vmcs_write32(GUEST_TR_AR_BYTES,
3221                              (guest_tr_ar & ~AR_TYPE_MASK)
3222                              | AR_TYPE_BUSY_64_TSS);
3223         }
3224         vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA);
3225 }
3226 
3227 static void exit_lmode(struct kvm_vcpu *vcpu)
3228 {
3229         vmcs_write32(VM_ENTRY_CONTROLS,
3230                      vmcs_read32(VM_ENTRY_CONTROLS)
3231                      & ~VM_ENTRY_IA32E_MODE);
3232         vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA);
3233 }
3234 
3235 #endif
3236 
3237 static void vmx_flush_tlb(struct kvm_vcpu *vcpu)
3238 {
3239         vpid_sync_context(to_vmx(vcpu));
3240         if (enable_ept) {
3241                 if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
3242                         return;
3243                 ept_sync_context(construct_eptp(vcpu->arch.mmu.root_hpa));
3244         }
3245 }
3246 
3247 static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
3248 {
3249         ulong cr0_guest_owned_bits = vcpu->arch.cr0_guest_owned_bits;
3250 
3251         vcpu->arch.cr0 &= ~cr0_guest_owned_bits;
3252         vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & cr0_guest_owned_bits;
3253 }
3254 
3255 static void vmx_decache_cr3(struct kvm_vcpu *vcpu)
3256 {
3257         if (enable_ept && is_paging(vcpu))
3258                 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
3259         __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
3260 }
3261 
3262 static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
3263 {
3264         ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;
3265 
3266         vcpu->arch.cr4 &= ~cr4_guest_owned_bits;
3267         vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & cr4_guest_owned_bits;
3268 }
3269 
3270 static void ept_load_pdptrs(struct kvm_vcpu *vcpu)
3271 {
3272         struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
3273 
3274         if (!test_bit(VCPU_EXREG_PDPTR,
3275                       (unsigned long *)&vcpu->arch.regs_dirty))
3276                 return;
3277 
3278         if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
3279                 vmcs_write64(GUEST_PDPTR0, mmu->pdptrs[0]);
3280                 vmcs_write64(GUEST_PDPTR1, mmu->pdptrs[1]);
3281                 vmcs_write64(GUEST_PDPTR2, mmu->pdptrs[2]);
3282                 vmcs_write64(GUEST_PDPTR3, mmu->pdptrs[3]);
3283         }
3284 }
3285 
3286 static void ept_save_pdptrs(struct kvm_vcpu *vcpu)
3287 {
3288         struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
3289 
3290         if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
3291                 mmu->pdptrs[0] = vmcs_read64(GUEST_PDPTR0);
3292                 mmu->pdptrs[1] = vmcs_read64(GUEST_PDPTR1);
3293                 mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2);
3294                 mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3);
3295         }
3296 
3297         __set_bit(VCPU_EXREG_PDPTR,
3298                   (unsigned long *)&vcpu->arch.regs_avail);
3299         __set_bit(VCPU_EXREG_PDPTR,
3300                   (unsigned long *)&vcpu->arch.regs_dirty);
3301 }
3302 
3303 static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
3304 
3305 static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
3306                                         unsigned long cr0,
3307                                         struct kvm_vcpu *vcpu)
3308 {
3309         if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail))
3310                 vmx_decache_cr3(vcpu);
3311         if (!(cr0 & X86_CR0_PG)) {
3312                 /* From paging/starting to nonpaging */
3313                 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
3314                              vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) |
3315                              (CPU_BASED_CR3_LOAD_EXITING |
3316                               CPU_BASED_CR3_STORE_EXITING));
3317                 vcpu->arch.cr0 = cr0;
3318                 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
3319         } else if (!is_paging(vcpu)) {
3320                 /* From nonpaging to paging */
3321                 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
3322                              vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) &
3323                              ~(CPU_BASED_CR3_LOAD_EXITING |
3324                                CPU_BASED_CR3_STORE_EXITING));
3325                 vcpu->arch.cr0 = cr0;
3326                 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
3327         }
3328 
3329         if (!(cr0 & X86_CR0_WP))
3330                 *hw_cr0 &= ~X86_CR0_WP;
3331 }
3332 
3333 static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
3334 {
3335         struct vcpu_vmx *vmx = to_vmx(vcpu);
3336         unsigned long hw_cr0;
3337 
3338         hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK);
3339         if (enable_unrestricted_guest)
3340                 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
3341         else {
3342                 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON;
3343 
3344                 if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE))
3345                         enter_pmode(vcpu);
3346 
3347                 if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE))
3348                         enter_rmode(vcpu);
3349         }
3350 
3351 #ifdef CONFIG_X86_64
3352         if (vcpu->arch.efer & EFER_LME) {
3353                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG))
3354                         enter_lmode(vcpu);
3355                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG))
3356                         exit_lmode(vcpu);
3357         }
3358 #endif
3359 
3360         if (enable_ept)
3361                 ept_update_paging_mode_cr0(&hw_cr0, cr0, vcpu);
3362 
3363         if (!vcpu->fpu_active)
3364                 hw_cr0 |= X86_CR0_TS | X86_CR0_MP;
3365 
3366         vmcs_writel(CR0_READ_SHADOW, cr0);
3367         vmcs_writel(GUEST_CR0, hw_cr0);
3368         vcpu->arch.cr0 = cr0;
3369 
3370         /* depends on vcpu->arch.cr0 to be set to a new value */
3371         vmx->emulation_required = emulation_required(vcpu);
3372 }
3373 
3374 static u64 construct_eptp(unsigned long root_hpa)
3375 {
3376         u64 eptp;
3377 
3378         /* TODO write the value reading from MSR */
3379         eptp = VMX_EPT_DEFAULT_MT |
3380                 VMX_EPT_DEFAULT_GAW << VMX_EPT_GAW_EPTP_SHIFT;
3381         if (enable_ept_ad_bits)
3382                 eptp |= VMX_EPT_AD_ENABLE_BIT;
3383         eptp |= (root_hpa & PAGE_MASK);
3384 
3385         return eptp;
3386 }
3387 
3388 static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
3389 {
3390         unsigned long guest_cr3;
3391         u64 eptp;
3392 
3393         guest_cr3 = cr3;
3394         if (enable_ept) {
3395                 eptp = construct_eptp(cr3);
3396                 vmcs_write64(EPT_POINTER, eptp);
3397                 guest_cr3 = is_paging(vcpu) ? kvm_read_cr3(vcpu) :
3398                         vcpu->kvm->arch.ept_identity_map_addr;
3399                 ept_load_pdptrs(vcpu);
3400         }
3401 
3402         vmx_flush_tlb(vcpu);
3403         vmcs_writel(GUEST_CR3, guest_cr3);
3404 }
3405 
3406 static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
3407 {
3408         unsigned long hw_cr4 = cr4 | (to_vmx(vcpu)->rmode.vm86_active ?
3409                     KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON);
3410 
3411         if (cr4 & X86_CR4_VMXE) {
3412                 /*
3413                  * To use VMXON (and later other VMX instructions), a guest
3414                  * must first be able to turn on cr4.VMXE (see handle_vmon()).
3415                  * So basically the check on whether to allow nested VMX
3416                  * is here.
3417                  */
3418                 if (!nested_vmx_allowed(vcpu))
3419                         return 1;
3420         }
3421         if (to_vmx(vcpu)->nested.vmxon &&
3422             ((cr4 & VMXON_CR4_ALWAYSON) != VMXON_CR4_ALWAYSON))
3423                 return 1;
3424 
3425         vcpu->arch.cr4 = cr4;
3426         if (enable_ept) {
3427                 if (!is_paging(vcpu)) {
3428                         hw_cr4 &= ~X86_CR4_PAE;
3429                         hw_cr4 |= X86_CR4_PSE;
3430                         /*
3431                          * SMEP is disabled if CPU is in non-paging mode in
3432                          * hardware. However KVM always uses paging mode to
3433                          * emulate guest non-paging mode with TDP.
3434                          * To emulate this behavior, SMEP needs to be manually
3435                          * disabled when guest switches to non-paging mode.
3436                          */
3437                         hw_cr4 &= ~X86_CR4_SMEP;
3438                 } else if (!(cr4 & X86_CR4_PAE)) {
3439                         hw_cr4 &= ~X86_CR4_PAE;
3440                 }
3441         }
3442 
3443         vmcs_writel(CR4_READ_SHADOW, cr4);
3444         vmcs_writel(GUEST_CR4, hw_cr4);
3445         return 0;
3446 }
3447 
3448 static void vmx_get_segment(struct kvm_vcpu *vcpu,
3449                             struct kvm_segment *var, int seg)
3450 {
3451         struct vcpu_vmx *vmx = to_vmx(vcpu);
3452         u32 ar;
3453 
3454         if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
3455                 *var = vmx->rmode.segs[seg];
3456                 if (seg == VCPU_SREG_TR
3457                     || var->selector == vmx_read_guest_seg_selector(vmx, seg))
3458                         return;
3459                 var->base = vmx_read_guest_seg_base(vmx, seg);
3460                 var->selector = vmx_read_guest_seg_selector(vmx, seg);
3461                 return;
3462         }
3463         var->base = vmx_read_guest_seg_base(vmx, seg);
3464         var->limit = vmx_read_guest_seg_limit(vmx, seg);
3465         var->selector = vmx_read_guest_seg_selector(vmx, seg);
3466         ar = vmx_read_guest_seg_ar(vmx, seg);
3467         var->unusable = (ar >> 16) & 1;
3468         var->type = ar & 15;
3469         var->s = (ar >> 4) & 1;
3470         var->dpl = (ar >> 5) & 3;
3471         /*
3472          * Some userspaces do not preserve unusable property. Since usable
3473          * segment has to be present according to VMX spec we can use present
3474          * property to amend userspace bug by making unusable segment always
3475          * nonpresent. vmx_segment_access_rights() already marks nonpresent
3476          * segment as unusable.
3477          */
3478         var->present = !var->unusable;
3479         var->avl = (ar >> 12) & 1;
3480         var->l = (ar >> 13) & 1;
3481         var->db = (ar >> 14) & 1;
3482         var->g = (ar >> 15) & 1;
3483 }
3484 
3485 static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
3486 {
3487         struct kvm_segment s;
3488 
3489         if (to_vmx(vcpu)->rmode.vm86_active) {
3490                 vmx_get_segment(vcpu, &s, seg);
3491                 return s.base;
3492         }
3493         return vmx_read_guest_seg_base(to_vmx(vcpu), seg);
3494 }
3495 
3496 static int vmx_get_cpl(struct kvm_vcpu *vcpu)
3497 {
3498         struct vcpu_vmx *vmx = to_vmx(vcpu);
3499 
3500         if (!is_protmode(vcpu))
3501                 return 0;
3502 
3503         if (!is_long_mode(vcpu)
3504             && (kvm_get_rflags(vcpu) & X86_EFLAGS_VM)) /* if virtual 8086 */
3505                 return 3;
3506 
3507         if (!test_bit(VCPU_EXREG_CPL, (ulong *)&vcpu->arch.regs_avail)) {
3508                 __set_bit(VCPU_EXREG_CPL, (ulong *)&vcpu->arch.regs_avail);
3509                 vmx->cpl = vmx_read_guest_seg_selector(vmx, VCPU_SREG_CS) & 3;
3510         }
3511 
3512         return vmx->cpl;
3513 }
3514 
3515 
3516 static u32 vmx_segment_access_rights(struct kvm_segment *var)
3517 {
3518         u32 ar;
3519 
3520         if (var->unusable || !var->present)
3521                 ar = 1 << 16;
3522         else {
3523                 ar = var->type & 15;
3524                 ar |= (var->s & 1) << 4;
3525                 ar |= (var->dpl & 3) << 5;
3526                 ar |= (var->present & 1) << 7;
3527                 ar |= (var->avl & 1) << 12;
3528                 ar |= (var->l & 1) << 13;
3529                 ar |= (var->db & 1) << 14;
3530                 ar |= (var->g & 1) << 15;
3531         }
3532 
3533         return ar;
3534 }
3535 
3536 static void vmx_set_segment(struct kvm_vcpu *vcpu,
3537                             struct kvm_segment *var, int seg)
3538 {
3539         struct vcpu_vmx *vmx = to_vmx(vcpu);
3540         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
3541 
3542         vmx_segment_cache_clear(vmx);
3543         if (seg == VCPU_SREG_CS)
3544                 __clear_bit(VCPU_EXREG_CPL, (ulong *)&vcpu->arch.regs_avail);
3545 
3546         if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
3547                 vmx->rmode.segs[seg] = *var;
3548                 if (seg == VCPU_SREG_TR)
3549                         vmcs_write16(sf->selector, var->selector);
3550                 else if (var->s)
3551                         fix_rmode_seg(seg, &vmx->rmode.segs[seg]);
3552                 goto out;
3553         }
3554 
3555         vmcs_writel(sf->base, var->base);
3556         vmcs_write32(sf->limit, var->limit);
3557         vmcs_write16(sf->selector, var->selector);
3558 
3559         /*
3560          *   Fix the "Accessed" bit in AR field of segment registers for older
3561          * qemu binaries.
3562          *   IA32 arch specifies that at the time of processor reset the
3563          * "Accessed" bit in the AR field of segment registers is 1. And qemu
3564          * is setting it to 0 in the userland code. This causes invalid guest
3565          * state vmexit when "unrestricted guest" mode is turned on.
3566          *    Fix for this setup issue in cpu_reset is being pushed in the qemu
3567          * tree. Newer qemu binaries with that qemu fix would not need this
3568          * kvm hack.
3569          */
3570         if (enable_unrestricted_guest && (seg != VCPU_SREG_LDTR))
3571                 var->type |= 0x1; /* Accessed */
3572 
3573         vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var));
3574 
3575 out:
3576         vmx->emulation_required |= emulation_required(vcpu);
3577 }
3578 
3579 static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
3580 {
3581         u32 ar = vmx_read_guest_seg_ar(to_vmx(vcpu), VCPU_SREG_CS);
3582 
3583         *db = (ar >> 14) & 1;
3584         *l = (ar >> 13) & 1;
3585 }
3586 
3587 static void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
3588 {
3589         dt->size = vmcs_read32(GUEST_IDTR_LIMIT);
3590         dt->address = vmcs_readl(GUEST_IDTR_BASE);
3591 }
3592 
3593 static void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
3594 {
3595         vmcs_write32(GUEST_IDTR_LIMIT, dt->size);
3596         vmcs_writel(GUEST_IDTR_BASE, dt->address);
3597 }
3598 
3599 static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
3600 {
3601         dt->size = vmcs_read32(GUEST_GDTR_LIMIT);
3602         dt->address = vmcs_readl(GUEST_GDTR_BASE);
3603 }
3604 
3605 static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
3606 {
3607         vmcs_write32(GUEST_GDTR_LIMIT, dt->size);
3608         vmcs_writel(GUEST_GDTR_BASE, dt->address);
3609 }
3610 
3611 static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
3612 {
3613         struct kvm_segment var;
3614         u32 ar;
3615 
3616         vmx_get_segment(vcpu, &var, seg);
3617         var.dpl = 0x3;
3618         if (seg == VCPU_SREG_CS)
3619                 var.type = 0x3;
3620         ar = vmx_segment_access_rights(&var);
3621 
3622         if (var.base != (var.selector << 4))
3623                 return false;
3624         if (var.limit != 0xffff)
3625                 return false;
3626         if (ar != 0xf3)
3627                 return false;
3628 
3629         return true;
3630 }
3631 
3632 static bool code_segment_valid(struct kvm_vcpu *vcpu)
3633 {
3634         struct kvm_segment cs;
3635         unsigned int cs_rpl;
3636 
3637         vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
3638         cs_rpl = cs.selector & SELECTOR_RPL_MASK;
3639 
3640         if (cs.unusable)
3641                 return false;
3642         if (~cs.type & (AR_TYPE_CODE_MASK|AR_TYPE_ACCESSES_MASK))
3643                 return false;
3644         if (!cs.s)
3645                 return false;
3646         if (cs.type & AR_TYPE_WRITEABLE_MASK) {
3647                 if (cs.dpl > cs_rpl)
3648                         return false;
3649         } else {
3650                 if (cs.dpl != cs_rpl)
3651                         return false;
3652         }
3653         if (!cs.present)
3654                 return false;
3655 
3656         /* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */
3657         return true;
3658 }
3659 
3660 static bool stack_segment_valid(struct kvm_vcpu *vcpu)
3661 {
3662         struct kvm_segment ss;
3663         unsigned int ss_rpl;
3664 
3665         vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
3666         ss_rpl = ss.selector & SELECTOR_RPL_MASK;
3667 
3668         if (ss.unusable)
3669                 return true;
3670         if (ss.type != 3 && ss.type != 7)
3671                 return false;
3672         if (!ss.s)
3673                 return false;
3674         if (ss.dpl != ss_rpl) /* DPL != RPL */
3675                 return false;
3676         if (!ss.present)
3677                 return false;
3678 
3679         return true;
3680 }
3681 
3682 static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg)
3683 {
3684         struct kvm_segment var;
3685         unsigned int rpl;
3686 
3687         vmx_get_segment(vcpu, &var, seg);
3688         rpl = var.selector & SELECTOR_RPL_MASK;
3689 
3690         if (var.unusable)
3691                 return true;
3692         if (!var.s)
3693                 return false;
3694         if (!var.present)
3695                 return false;
3696         if (~var.type & (AR_TYPE_CODE_MASK|AR_TYPE_WRITEABLE_MASK)) {
3697                 if (var.dpl < rpl) /* DPL < RPL */
3698                         return false;
3699         }
3700 
3701         /* TODO: Add other members to kvm_segment_field to allow checking for other access
3702          * rights flags
3703          */
3704         return true;
3705 }
3706 
3707 static bool tr_valid(struct kvm_vcpu *vcpu)
3708 {
3709         struct kvm_segment tr;
3710 
3711         vmx_get_segment(vcpu, &tr, VCPU_SREG_TR);
3712 
3713         if (tr.unusable)
3714                 return false;
3715         if (tr.selector & SELECTOR_TI_MASK)     /* TI = 1 */
3716                 return false;
3717         if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */
3718                 return false;
3719         if (!tr.present)
3720                 return false;
3721 
3722         return true;
3723 }
3724 
3725 static bool ldtr_valid(struct kvm_vcpu *vcpu)
3726 {
3727         struct kvm_segment ldtr;
3728 
3729         vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR);
3730 
3731         if (ldtr.unusable)
3732                 return true;
3733         if (ldtr.selector & SELECTOR_TI_MASK)   /* TI = 1 */
3734                 return false;
3735         if (ldtr.type != 2)
3736                 return false;
3737         if (!ldtr.present)
3738                 return false;
3739 
3740         return true;
3741 }
3742 
3743 static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu)
3744 {
3745         struct kvm_segment cs, ss;
3746 
3747         vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
3748         vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
3749 
3750         return ((cs.selector & SELECTOR_RPL_MASK) ==
3751                  (ss.selector & SELECTOR_RPL_MASK));
3752 }
3753 
3754 /*
3755  * Check if guest state is valid. Returns true if valid, false if
3756  * not.
3757  * We assume that registers are always usable
3758  */
3759 static bool guest_state_valid(struct kvm_vcpu *vcpu)
3760 {
3761         if (enable_unrestricted_guest)
3762                 return true;
3763 
3764         /* real mode guest state checks */
3765         if (!is_protmode(vcpu) || (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) {
3766                 if (!rmode_segment_valid(vcpu, VCPU_SREG_CS))
3767                         return false;
3768                 if (!rmode_segment_valid(vcpu, VCPU_SREG_SS))
3769                         return false;
3770                 if (!rmode_segment_valid(vcpu, VCPU_SREG_DS))
3771                         return false;
3772                 if (!rmode_segment_valid(vcpu, VCPU_SREG_ES))
3773                         return false;
3774                 if (!rmode_segment_valid(vcpu, VCPU_SREG_FS))
3775                         return false;
3776                 if (!rmode_segment_valid(vcpu, VCPU_SREG_GS))
3777                         return false;
3778         } else {
3779         /* protected mode guest state checks */
3780                 if (!cs_ss_rpl_check(vcpu))
3781                         return false;
3782                 if (!code_segment_valid(vcpu))
3783                         return false;
3784                 if (!stack_segment_valid(vcpu))
3785                         return false;
3786                 if (!data_segment_valid(vcpu, VCPU_SREG_DS))
3787                         return false;
3788                 if (!data_segment_valid(vcpu, VCPU_SREG_ES))
3789                         return false;
3790                 if (!data_segment_valid(vcpu, VCPU_SREG_FS))
3791                         return false;
3792                 if (!data_segment_valid(vcpu, VCPU_SREG_GS))
3793                         return false;
3794                 if (!tr_valid(vcpu))
3795                         return false;
3796                 if (!ldtr_valid(vcpu))
3797                         return false;
3798         }
3799         /* TODO:
3800          * - Add checks on RIP
3801          * - Add checks on RFLAGS
3802          */
3803 
3804         return true;
3805 }
3806 
3807 static int init_rmode_tss(struct kvm *kvm)
3808 {
3809         gfn_t fn;
3810         u16 data = 0;
3811         int r, idx, ret = 0;
3812 
3813         idx = srcu_read_lock(&kvm->srcu);
3814         fn = kvm->arch.tss_addr >> PAGE_SHIFT;
3815         r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
3816         if (r < 0)
3817                 goto out;
3818         data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
3819         r = kvm_write_guest_page(kvm, fn++, &data,
3820                         TSS_IOPB_BASE_OFFSET, sizeof(u16));
3821         if (r < 0)
3822                 goto out;
3823         r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE);
3824         if (r < 0)
3825                 goto out;
3826         r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
3827         if (r < 0)
3828                 goto out;
3829         data = ~0;
3830         r = kvm_write_guest_page(kvm, fn, &data,
3831                                  RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1,
3832                                  sizeof(u8));
3833         if (r < 0)
3834                 goto out;
3835 
3836         ret = 1;
3837 out:
3838         srcu_read_unlock(&kvm->srcu, idx);
3839         return ret;
3840 }
3841 
3842 static int init_rmode_identity_map(struct kvm *kvm)
3843 {
3844         int i, idx, r, ret;
3845         pfn_t identity_map_pfn;
3846         u32 tmp;
3847 
3848         if (!enable_ept)
3849                 return 1;
3850         if (unlikely(!kvm->arch.ept_identity_pagetable)) {
3851                 printk(KERN_ERR "EPT: identity-mapping pagetable "
3852                         "haven't been allocated!\n");
3853                 return 0;
3854         }
3855         if (likely(kvm->arch.ept_identity_pagetable_done))
3856                 return 1;
3857         ret = 0;
3858         identity_map_pfn = kvm->arch.ept_identity_map_addr >> PAGE_SHIFT;
3859         idx = srcu_read_lock(&kvm->srcu);
3860         r = kvm_clear_guest_page(kvm, identity_map_pfn, 0, PAGE_SIZE);
3861         if (r < 0)
3862                 goto out;
3863         /* Set up identity-mapping pagetable for EPT in real mode */
3864         for (i = 0; i < PT32_ENT_PER_PAGE; i++) {
3865                 tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |
3866                         _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE);
3867                 r = kvm_write_guest_page(kvm, identity_map_pfn,
3868                                 &tmp, i * sizeof(tmp), sizeof(tmp));
3869                 if (r < 0)
3870                         goto out;
3871         }
3872         kvm->arch.ept_identity_pagetable_done = true;
3873         ret = 1;
3874 out:
3875         srcu_read_unlock(&kvm->srcu, idx);
3876         return ret;
3877 }
3878 
3879 static void seg_setup(int seg)
3880 {
3881         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
3882         unsigned int ar;
3883 
3884         vmcs_write16(sf->selector, 0);
3885         vmcs_writel(sf->base, 0);
3886         vmcs_write32(sf->limit, 0xffff);
3887         ar = 0x93;
3888         if (seg == VCPU_SREG_CS)
3889                 ar |= 0x08; /* code segment */
3890 
3891         vmcs_write32(sf->ar_bytes, ar);
3892 }
3893 
3894 static int alloc_apic_access_page(struct kvm *kvm)
3895 {
3896         struct page *page;
3897         struct kvm_userspace_memory_region kvm_userspace_mem;
3898         int r = 0;
3899 
3900         mutex_lock(&kvm->slots_lock);
3901         if (kvm->arch.apic_access_page)
3902                 goto out;
3903         kvm_userspace_mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT;
3904         kvm_userspace_mem.flags = 0;
3905         kvm_userspace_mem.guest_phys_addr = 0xfee00000ULL;
3906         kvm_userspace_mem.memory_size = PAGE_SIZE;
3907         r = __kvm_set_memory_region(kvm, &kvm_userspace_mem);
3908         if (r)
3909                 goto out;
3910 
3911         page = gfn_to_page(kvm, 0xfee00);
3912         if (is_error_page(page)) {
3913                 r = -EFAULT;
3914                 goto out;
3915         }
3916 
3917         kvm->arch.apic_access_page = page;
3918 out:
3919         mutex_unlock(&kvm->slots_lock);
3920         return r;
3921 }
3922 
3923 static int alloc_identity_pagetable(struct kvm *kvm)
3924 {
3925         struct page *page;
3926         struct kvm_userspace_memory_region kvm_userspace_mem;
3927         int r = 0;
3928 
3929         mutex_lock(&kvm->slots_lock);
3930         if (kvm->arch.ept_identity_pagetable)
3931                 goto out;
3932         kvm_userspace_mem.slot = IDENTITY_PAGETABLE_PRIVATE_MEMSLOT;
3933         kvm_userspace_mem.flags = 0;
3934         kvm_userspace_mem.guest_phys_addr =
3935                 kvm->arch.ept_identity_map_addr;
3936         kvm_userspace_mem.memory_size = PAGE_SIZE;
3937         r = __kvm_set_memory_region(kvm, &kvm_userspace_mem);
3938         if (r)
3939                 goto out;
3940 
3941         page = gfn_to_page(kvm, kvm->arch.ept_identity_map_addr >> PAGE_SHIFT);
3942         if (is_error_page(page)) {
3943                 r = -EFAULT;
3944                 goto out;
3945         }
3946 
3947         kvm->arch.ept_identity_pagetable = page;
3948 out:
3949         mutex_unlock(&kvm->slots_lock);
3950         return r;
3951 }
3952 
3953 static void allocate_vpid(struct vcpu_vmx *vmx)
3954 {
3955         int vpid;
3956 
3957         vmx->vpid = 0;
3958         if (!enable_vpid)
3959                 return;
3960         spin_lock(&vmx_vpid_lock);
3961         vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS);
3962         if (vpid < VMX_NR_VPIDS) {
3963                 vmx->vpid = vpid;
3964                 __set_bit(vpid, vmx_vpid_bitmap);
3965         }
3966         spin_unlock(&vmx_vpid_lock);
3967 }
3968 
3969 static void free_vpid(struct vcpu_vmx *vmx)
3970 {
3971         if (!enable_vpid)
3972                 return;
3973         spin_lock(&vmx_vpid_lock);
3974         if (vmx->vpid != 0)
3975                 __clear_bit(vmx->vpid, vmx_vpid_bitmap);
3976         spin_unlock(&vmx_vpid_lock);
3977 }
3978 
3979 #define MSR_TYPE_R      1
3980 #define MSR_TYPE_W      2
3981 static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
3982                                                 u32 msr, int type)
3983 {
3984         int f = sizeof(unsigned long);
3985 
3986         if (!cpu_has_vmx_msr_bitmap())
3987                 return;
3988 
3989         /*
3990          * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
3991          * have the write-low and read-high bitmap offsets the wrong way round.
3992          * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
3993          */
3994         if (msr <= 0x1fff) {
3995                 if (type & MSR_TYPE_R)
3996                         /* read-low */
3997                         __clear_bit(msr, msr_bitmap + 0x000 / f);
3998 
3999                 if (type & MSR_TYPE_W)
4000                         /* write-low */
4001                         __clear_bit(msr, msr_bitmap + 0x800 / f);
4002 
4003         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
4004                 msr &= 0x1fff;
4005                 if (type & MSR_TYPE_R)
4006                         /* read-high */
4007                         __clear_bit(msr, msr_bitmap + 0x400 / f);
4008 
4009                 if (type & MSR_TYPE_W)
4010                         /* write-high */
4011                         __clear_bit(msr, msr_bitmap + 0xc00 / f);
4012 
4013         }
4014 }
4015 
4016 static void __vmx_enable_intercept_for_msr(unsigned long *msr_bitmap,
4017                                                 u32 msr, int type)
4018 {
4019         int f = sizeof(unsigned long);
4020 
4021         if (!cpu_has_vmx_msr_bitmap())
4022                 return;
4023 
4024         /*
4025          * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
4026          * have the write-low and read-high bitmap offsets the wrong way round.
4027          * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
4028          */
4029         if (msr <= 0x1fff) {
4030                 if (type & MSR_TYPE_R)
4031                         /* read-low */
4032                         __set_bit(msr, msr_bitmap + 0x000 / f);
4033 
4034                 if (type & MSR_TYPE_W)
4035                         /* write-low */
4036                         __set_bit(msr, msr_bitmap + 0x800 / f);
4037 
4038         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
4039                 msr &= 0x1fff;
4040                 if (type & MSR_TYPE_R)
4041                         /* read-high */
4042                         __set_bit(msr, msr_bitmap + 0x400 / f);
4043 
4044                 if (type & MSR_TYPE_W)
4045                         /* write-high */
4046                         __set_bit(msr, msr_bitmap + 0xc00 / f);
4047 
4048         }
4049 }
4050 
4051 static void vmx_disable_intercept_for_msr(u32 msr, bool longmode_only)
4052 {
4053         if (!longmode_only)
4054                 __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy,
4055                                                 msr, MSR_TYPE_R | MSR_TYPE_W);
4056         __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode,
4057                                                 msr, MSR_TYPE_R | MSR_TYPE_W);
4058 }
4059 
4060 static void vmx_enable_intercept_msr_read_x2apic(u32 msr)
4061 {
4062         __vmx_enable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic,
4063                         msr, MSR_TYPE_R);
4064         __vmx_enable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic,
4065                         msr, MSR_TYPE_R);
4066 }
4067 
4068 static void vmx_disable_intercept_msr_read_x2apic(u32 msr)
4069 {
4070         __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic,
4071                         msr, MSR_TYPE_R);
4072         __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic,
4073                         msr, MSR_TYPE_R);
4074 }
4075 
4076 static void vmx_disable_intercept_msr_write_x2apic(u32 msr)
4077 {
4078         __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic,
4079                         msr, MSR_TYPE_W);
4080         __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic,
4081                         msr, MSR_TYPE_W);
4082 }
4083 
4084 static int vmx_vm_has_apicv(struct kvm *kvm)
4085 {
4086         return enable_apicv && irqchip_in_kernel(kvm);
4087 }
4088 
4089 /*
4090  * Send interrupt to vcpu via posted interrupt way.
4091  * 1. If target vcpu is running(non-root mode), send posted interrupt
4092  * notification to vcpu and hardware will sync PIR to vIRR atomically.
4093  * 2. If target vcpu isn't running(root mode), kick it to pick up the
4094  * interrupt from PIR in next vmentry.
4095  */
4096 static void vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector)
4097 {
4098         struct vcpu_vmx *vmx = to_vmx(vcpu);
4099         int r;
4100 
4101         if (pi_test_and_set_pir(vector, &vmx->pi_desc))
4102                 return;
4103 
4104         r = pi_test_and_set_on(&vmx->pi_desc);
4105         kvm_make_request(KVM_REQ_EVENT, vcpu);
4106 #ifdef CONFIG_SMP
4107         if (!r && (vcpu->mode == IN_GUEST_MODE))
4108                 apic->send_IPI_mask(get_cpu_mask(vcpu->cpu),
4109                                 POSTED_INTR_VECTOR);
4110         else
4111 #endif
4112                 kvm_vcpu_kick(vcpu);
4113 }
4114 
4115 static void vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
4116 {
4117         struct vcpu_vmx *vmx = to_vmx(vcpu);
4118 
4119         if (!pi_test_and_clear_on(&vmx->pi_desc))
4120                 return;
4121 
4122         kvm_apic_update_irr(vcpu, vmx->pi_desc.pir);
4123 }
4124 
4125 static void vmx_sync_pir_to_irr_dummy(struct kvm_vcpu *vcpu)
4126 {
4127         return;
4128 }
4129 
4130 /*
4131  * Set up the vmcs's constant host-state fields, i.e., host-state fields that
4132  * will not change in the lifetime of the guest.
4133  * Note that host-state that does change is set elsewhere. E.g., host-state
4134  * that is set differently for each CPU is set in vmx_vcpu_load(), not here.
4135  */
4136 static void vmx_set_constant_host_state(struct vcpu_vmx *vmx)
4137 {
4138         u32 low32, high32;
4139         unsigned long tmpl;
4140         struct desc_ptr dt;
4141         unsigned long cr4;
4142 
4143         vmcs_writel(HOST_CR0, read_cr0() & ~X86_CR0_TS);  /* 22.2.3 */
4144         vmcs_writel(HOST_CR3, read_cr3());  /* 22.2.3  FIXME: shadow tables */
4145 
4146         /* Save the most likely value for this task's CR4 in the VMCS. */
4147         cr4 = read_cr4();
4148         vmcs_writel(HOST_CR4, cr4);                     /* 22.2.3, 22.2.5 */
4149         vmx->host_state.vmcs_host_cr4 = cr4;
4150 
4151         vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS);  /* 22.2.4 */
4152 #ifdef CONFIG_X86_64
4153         /*
4154          * Load null selectors, so we can avoid reloading them in
4155          * __vmx_load_host_state(), in case userspace uses the null selectors
4156          * too (the expected case).
4157          */
4158         vmcs_write16(HOST_DS_SELECTOR, 0);
4159         vmcs_write16(HOST_ES_SELECTOR, 0);
4160 #else
4161         vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
4162         vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
4163 #endif
4164         vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
4165         vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8);  /* 22.2.4 */
4166 
4167         native_store_idt(&dt);
4168         vmcs_writel(HOST_IDTR_BASE, dt.address);   /* 22.2.4 */
4169         vmx->host_idt_base = dt.address;
4170 
4171         vmcs_writel(HOST_RIP, vmx_return); /* 22.2.5 */
4172 
4173         rdmsr(MSR_IA32_SYSENTER_CS, low32, high32);
4174         vmcs_write32(HOST_IA32_SYSENTER_CS, low32);
4175         rdmsrl(MSR_IA32_SYSENTER_EIP, tmpl);
4176         vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl);   /* 22.2.3 */
4177 
4178         if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) {
4179                 rdmsr(MSR_IA32_CR_PAT, low32, high32);
4180                 vmcs_write64(HOST_IA32_PAT, low32 | ((u64) high32 << 32));
4181         }
4182 }
4183 
4184 static void set_cr4_guest_host_mask(struct vcpu_vmx *vmx)
4185 {
4186         vmx->vcpu.arch.cr4_guest_owned_bits = KVM_CR4_GUEST_OWNED_BITS;
4187         if (enable_ept)
4188                 vmx->vcpu.arch.cr4_guest_owned_bits |= X86_CR4_PGE;
4189         if (is_guest_mode(&vmx->vcpu))
4190                 vmx->vcpu.arch.cr4_guest_owned_bits &=
4191                         ~get_vmcs12(&vmx->vcpu)->cr4_guest_host_mask;
4192         vmcs_writel(CR4_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr4_guest_owned_bits);
4193 }
4194 
4195 static u32 vmx_pin_based_exec_ctrl(struct vcpu_vmx *vmx)
4196 {
4197         u32 pin_based_exec_ctrl = vmcs_config.pin_based_exec_ctrl;
4198 
4199         if (!vmx_vm_has_apicv(vmx->vcpu.kvm))
4200                 pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR;
4201         return pin_based_exec_ctrl;
4202 }
4203 
4204 static u32 vmx_exec_control(struct vcpu_vmx *vmx)
4205 {
4206         u32 exec_control = vmcs_config.cpu_based_exec_ctrl;
4207         if (!vm_need_tpr_shadow(vmx->vcpu.kvm)) {
4208                 exec_control &= ~CPU_BASED_TPR_SHADOW;
4209 #ifdef CONFIG_X86_64
4210                 exec_control |= CPU_BASED_CR8_STORE_EXITING |
4211                                 CPU_BASED_CR8_LOAD_EXITING;
4212 #endif
4213         }
4214         if (!enable_ept)
4215                 exec_control |= CPU_BASED_CR3_STORE_EXITING |
4216                                 CPU_BASED_CR3_LOAD_EXITING  |
4217                                 CPU_BASED_INVLPG_EXITING;
4218         return exec_control;
4219 }
4220 
4221 static u32 vmx_secondary_exec_control(struct vcpu_vmx *vmx)
4222 {
4223         u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl;
4224         if (!vm_need_virtualize_apic_accesses(vmx->vcpu.kvm))
4225                 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
4226         if (vmx->vpid == 0)
4227                 exec_control &= ~SECONDARY_EXEC_ENABLE_VPID;
4228         if (!enable_ept) {
4229                 exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;
4230                 enable_unrestricted_guest = 0;
4231                 /* Enable INVPCID for non-ept guests may cause performance regression. */
4232                 exec_control &= ~SECONDARY_EXEC_ENABLE_INVPCID;
4233         }
4234         if (!enable_unrestricted_guest)
4235                 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
4236         if (!ple_gap)
4237                 exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
4238         if (!vmx_vm_has_apicv(vmx->vcpu.kvm))
4239                 exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT |
4240                                   SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
4241         exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
4242         /* SECONDARY_EXEC_SHADOW_VMCS is enabled when L1 executes VMPTRLD
4243            (handle_vmptrld).
4244            We can NOT enable shadow_vmcs here because we don't have yet
4245            a current VMCS12
4246         */
4247         exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS;
4248         return exec_control;
4249 }
4250 
4251 static void ept_set_mmio_spte_mask(void)
4252 {
4253         /*
4254          * EPT Misconfigurations can be generated if the value of bits 2:0
4255          * of an EPT paging-structure entry is 110b (write/execute).
4256          * Also, magic bits (0x3ull << 62) is set to quickly identify mmio
4257          * spte.
4258          */
4259         kvm_mmu_set_mmio_spte_mask((0x3ull << 62) | 0x6ull);
4260 }
4261 
4262 /*
4263  * Sets up the vmcs for emulated real mode.
4264  */
4265 static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
4266 {
4267 #ifdef CONFIG_X86_64
4268         unsigned long a;
4269 #endif
4270         int i;
4271 
4272         /* I/O */
4273         vmcs_write64(IO_BITMAP_A, __pa(vmx_io_bitmap_a));
4274         vmcs_write64(IO_BITMAP_B, __pa(vmx_io_bitmap_b));
4275 
4276         if (enable_shadow_vmcs) {
4277                 vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap));
4278                 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap));
4279         }
4280         if (cpu_has_vmx_msr_bitmap())
4281                 vmcs_write64(MSR_BITMAP, __pa(vmx_msr_bitmap_legacy));
4282 
4283         vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */
4284 
4285         /* Control */
4286         vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, vmx_pin_based_exec_ctrl(vmx));
4287 
4288         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, vmx_exec_control(vmx));
4289 
4290         if (cpu_has_secondary_exec_ctrls()) {
4291                 vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
4292                                 vmx_secondary_exec_control(vmx));
4293         }
4294 
4295         if (vmx_vm_has_apicv(vmx->vcpu.kvm)) {
4296                 vmcs_write64(EOI_EXIT_BITMAP0, 0);
4297                 vmcs_write64(EOI_EXIT_BITMAP1, 0);
4298                 vmcs_write64(EOI_EXIT_BITMAP2, 0);
4299                 vmcs_write64(EOI_EXIT_BITMAP3, 0);
4300 
4301                 vmcs_write16(GUEST_INTR_STATUS, 0);
4302 
4303                 vmcs_write64(POSTED_INTR_NV, POSTED_INTR_VECTOR);
4304                 vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->pi_desc)));
4305         }
4306 
4307         if (ple_gap) {
4308                 vmcs_write32(PLE_GAP, ple_gap);
4309                 vmcs_write32(PLE_WINDOW, ple_window);
4310         }
4311 
4312         vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0);
4313         vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0);
4314         vmcs_write32(CR3_TARGET_COUNT, 0);           /* 22.2.1 */
4315 
4316         vmcs_write16(HOST_FS_SELECTOR, 0);            /* 22.2.4 */
4317         vmcs_write16(HOST_GS_SELECTOR, 0);            /* 22.2.4 */
4318         vmx_set_constant_host_state(vmx);
4319 #ifdef CONFIG_X86_64
4320         rdmsrl(MSR_FS_BASE, a);
4321         vmcs_writel(HOST_FS_BASE, a); /* 22.2.4 */
4322         rdmsrl(MSR_GS_BASE, a);
4323         vmcs_writel(HOST_GS_BASE, a); /* 22.2.4 */
4324 #else
4325         vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
4326         vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
4327 #endif
4328 
4329         vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
4330         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
4331         vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host));
4332         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
4333         vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest));
4334 
4335         if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
4336                 u32 msr_low, msr_high;
4337                 u64 host_pat;
4338                 rdmsr(MSR_IA32_CR_PAT, msr_low, msr_high);
4339                 host_pat = msr_low | ((u64) msr_high << 32);
4340                 /* Write the default value follow host pat */
4341                 vmcs_write64(GUEST_IA32_PAT, host_pat);
4342                 /* Keep arch.pat sync with GUEST_IA32_PAT */
4343                 vmx->vcpu.arch.pat = host_pat;
4344         }
4345 
4346         for (i = 0; i < NR_VMX_MSR; ++i) {
4347                 u32 index = vmx_msr_index[i];
4348                 u32 data_low, data_high;
4349                 int j = vmx->nmsrs;
4350 
4351                 if (rdmsr_safe(index, &data_low, &data_high) < 0)
4352                         continue;
4353                 if (wrmsr_safe(index, data_low, data_high) < 0)
4354                         continue;
4355                 vmx->guest_msrs[j].index = i;
4356                 vmx->guest_msrs[j].data = 0;
4357                 vmx->guest_msrs[j].mask = -1ull;
4358                 ++vmx->nmsrs;
4359         }
4360 
4361         vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
4362 
4363         /* 22.2.1, 20.8.1 */
4364         vmcs_write32(VM_ENTRY_CONTROLS, vmcs_config.vmentry_ctrl);
4365 
4366         vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL);
4367         set_cr4_guest_host_mask(vmx);
4368 
4369         return 0;
4370 }
4371 
4372 static void vmx_vcpu_reset(struct kvm_vcpu *vcpu)
4373 {
4374         struct vcpu_vmx *vmx = to_vmx(vcpu);
4375         u64 msr;
4376 
4377         vmx->rmode.vm86_active = 0;
4378 
4379         vmx->soft_vnmi_blocked = 0;
4380 
4381         vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
4382         kvm_set_cr8(&vmx->vcpu, 0);
4383         msr = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
4384         if (kvm_vcpu_is_bsp(&vmx->vcpu))
4385                 msr |= MSR_IA32_APICBASE_BSP;
4386         kvm_set_apic_base(&vmx->vcpu, msr);
4387 
4388         vmx_segment_cache_clear(vmx);
4389 
4390         seg_setup(VCPU_SREG_CS);
4391         vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
4392         vmcs_write32(GUEST_CS_BASE, 0xffff0000);
4393 
4394         seg_setup(VCPU_SREG_DS);
4395         seg_setup(VCPU_SREG_ES);
4396         seg_setup(VCPU_SREG_FS);
4397         seg_setup(VCPU_SREG_GS);
4398         seg_setup(VCPU_SREG_SS);
4399 
4400         vmcs_write16(GUEST_TR_SELECTOR, 0);
4401         vmcs_writel(GUEST_TR_BASE, 0);
4402         vmcs_write32(GUEST_TR_LIMIT, 0xffff);
4403         vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
4404 
4405         vmcs_write16(GUEST_LDTR_SELECTOR, 0);
4406         vmcs_writel(GUEST_LDTR_BASE, 0);
4407         vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
4408         vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);
4409 
4410         vmcs_write32(GUEST_SYSENTER_CS, 0);
4411         vmcs_writel(GUEST_SYSENTER_ESP, 0);
4412         vmcs_writel(GUEST_SYSENTER_EIP, 0);
4413 
4414         vmcs_writel(GUEST_RFLAGS, 0x02);
4415         kvm_rip_write(vcpu, 0xfff0);
4416 
4417         vmcs_writel(GUEST_GDTR_BASE, 0);
4418         vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);
4419 
4420         vmcs_writel(GUEST_IDTR_BASE, 0);
4421         vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);
4422 
4423         vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
4424         vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
4425         vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0);
4426 
4427         /* Special registers */
4428         vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
4429 
4430         setup_msrs(vmx);
4431 
4432         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);  /* 22.2.1 */
4433 
4434         if (cpu_has_vmx_tpr_shadow()) {
4435                 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
4436                 if (vm_need_tpr_shadow(vmx->vcpu.kvm))
4437                         vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
4438                                      __pa(vmx->vcpu.arch.apic->regs));
4439                 vmcs_write32(TPR_THRESHOLD, 0);
4440         }
4441 
4442         if (vm_need_virtualize_apic_accesses(vmx->vcpu.kvm))
4443                 vmcs_write64(APIC_ACCESS_ADDR,
4444                              page_to_phys(vmx->vcpu.kvm->arch.apic_access_page));
4445 
4446         if (vmx_vm_has_apicv(vcpu->kvm))
4447                 memset(&vmx->pi_desc, 0, sizeof(struct pi_desc));
4448 
4449         if (vmx->vpid != 0)
4450                 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
4451 
4452         vmx->vcpu.arch.cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET;
4453         vmx_set_cr0(&vmx->vcpu, kvm_read_cr0(vcpu)); /* enter rmode */
4454         vmx_set_cr4(&vmx->vcpu, 0);
4455         vmx_set_efer(&vmx->vcpu, 0);
4456         vmx_fpu_activate(&vmx->vcpu);
4457         update_exception_bitmap(&vmx->vcpu);
4458 
4459         vpid_sync_context(vmx);
4460 }
4461 
4462 /*
4463  * In nested virtualization, check if L1 asked to exit on external interrupts.
4464  * For most existing hypervisors, this will always return true.
4465  */
4466 static bool nested_exit_on_intr(struct kvm_vcpu *vcpu)
4467 {
4468         return get_vmcs12(vcpu)->pin_based_vm_exec_control &
4469                 PIN_BASED_EXT_INTR_MASK;
4470 }
4471 
4472 static bool nested_exit_on_nmi(struct kvm_vcpu *vcpu)
4473 {
4474         return get_vmcs12(vcpu)->pin_based_vm_exec_control &
4475                 PIN_BASED_NMI_EXITING;
4476 }
4477 
4478 static int enable_irq_window(struct kvm_vcpu *vcpu)
4479 {
4480         u32 cpu_based_vm_exec_control;
4481 
4482         if (is_guest_mode(vcpu) && nested_exit_on_intr(vcpu))
4483                 /*
4484                  * We get here if vmx_interrupt_allowed() said we can't
4485                  * inject to L1 now because L2 must run. The caller will have
4486                  * to make L2 exit right after entry, so we can inject to L1
4487                  * more promptly.
4488                  */
4489                 return -EBUSY;
4490 
4491         cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
4492         cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_INTR_PENDING;
4493         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
4494         return 0;
4495 }
4496 
4497 static int enable_nmi_window(struct kvm_vcpu *vcpu)
4498 {
4499         u32 cpu_based_vm_exec_control;
4500 
4501         if (!cpu_has_virtual_nmis())
4502                 return enable_irq_window(vcpu);
4503 
4504         if (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI)
4505                 return enable_irq_window(vcpu);
4506 
4507         cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
4508         cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_NMI_PENDING;
4509         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
4510         return 0;
4511 }
4512 
4513 static void vmx_inject_irq(struct kvm_vcpu *vcpu)
4514 {
4515         struct vcpu_vmx *vmx = to_vmx(vcpu);
4516         uint32_t intr;
4517         int irq = vcpu->arch.interrupt.nr;
4518 
4519         trace_kvm_inj_virq(irq);
4520 
4521         ++vcpu->stat.irq_injections;
4522         if (vmx->rmode.vm86_active) {
4523                 int inc_eip = 0;
4524                 if (vcpu->arch.interrupt.soft)
4525                         inc_eip = vcpu->arch.event_exit_inst_len;
4526                 if (kvm_inject_realmode_interrupt(vcpu, irq, inc_eip) != EMULATE_DONE)
4527                         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
4528                 return;
4529         }
4530         intr = irq | INTR_INFO_VALID_MASK;
4531         if (vcpu->arch.interrupt.soft) {
4532                 intr |= INTR_TYPE_SOFT_INTR;
4533                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
4534                              vmx->vcpu.arch.event_exit_inst_len);
4535         } else
4536                 intr |= INTR_TYPE_EXT_INTR;
4537         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr);
4538 }
4539 
4540 static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
4541 {
4542         struct vcpu_vmx *vmx = to_vmx(vcpu);
4543 
4544         if (is_guest_mode(vcpu))
4545                 return;
4546 
4547         if (!cpu_has_virtual_nmis()) {
4548                 /*
4549                  * Tracking the NMI-blocked state in software is built upon
4550                  * finding the next open IRQ window. This, in turn, depends on
4551                  * well-behaving guests: They have to keep IRQs disabled at
4552                  * least as long as the NMI handler runs. Otherwise we may
4553                  * cause NMI nesting, maybe breaking the guest. But as this is
4554                  * highly unlikely, we can live with the residual risk.
4555                  */
4556                 vmx->soft_vnmi_blocked = 1;
4557                 vmx->vnmi_blocked_time = 0;
4558         }
4559 
4560         ++vcpu->stat.nmi_injections;
4561         vmx->nmi_known_unmasked = false;
4562         if (vmx->rmode.vm86_active) {
4563                 if (kvm_inject_realmode_interrupt(vcpu, NMI_VECTOR, 0) != EMULATE_DONE)
4564                         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
4565                 return;
4566         }
4567         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
4568                         INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR);
4569 }
4570 
4571 static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu)
4572 {
4573         if (!cpu_has_virtual_nmis())
4574                 return to_vmx(vcpu)->soft_vnmi_blocked;
4575         if (to_vmx(vcpu)->nmi_known_unmasked)
4576                 return false;
4577         return vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_NMI;
4578 }
4579 
4580 static void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
4581 {
4582         struct vcpu_vmx *vmx = to_vmx(vcpu);
4583 
4584         if (!cpu_has_virtual_nmis()) {
4585                 if (vmx->soft_vnmi_blocked != masked) {
4586                         vmx->soft_vnmi_blocked = masked;
4587                         vmx->vnmi_blocked_time = 0;
4588                 }
4589         } else {
4590                 vmx->nmi_known_unmasked = !masked;
4591                 if (masked)
4592                         vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
4593                                       GUEST_INTR_STATE_NMI);
4594                 else
4595                         vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
4596                                         GUEST_INTR_STATE_NMI);
4597         }
4598 }
4599 
4600 static int vmx_nmi_allowed(struct kvm_vcpu *vcpu)
4601 {
4602         if (is_guest_mode(vcpu)) {
4603                 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
4604 
4605                 if (to_vmx(vcpu)->nested.nested_run_pending)
4606                         return 0;
4607                 if (nested_exit_on_nmi(vcpu)) {
4608                         nested_vmx_vmexit(vcpu);
4609                         vmcs12->vm_exit_reason = EXIT_REASON_EXCEPTION_NMI;
4610                         vmcs12->vm_exit_intr_info = NMI_VECTOR |
4611                                 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK;
4612                         /*
4613                          * The NMI-triggered VM exit counts as injection:
4614                          * clear this one and block further NMIs.
4615                          */
4616                         vcpu->arch.nmi_pending = 0;
4617                         vmx_set_nmi_mask(vcpu, true);
4618                         return 0;
4619                 }
4620         }
4621 
4622         if (!cpu_has_virtual_nmis() && to_vmx(vcpu)->soft_vnmi_blocked)
4623                 return 0;
4624 
4625         return  !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
4626                   (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI
4627                    | GUEST_INTR_STATE_NMI));
4628 }
4629 
4630 static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu)
4631 {
4632         if (is_guest_mode(vcpu)) {
4633                 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
4634 
4635                 if (to_vmx(vcpu)->nested.nested_run_pending)
4636                         return 0;
4637                 if (nested_exit_on_intr(vcpu)) {
4638                         nested_vmx_vmexit(vcpu);
4639                         vmcs12->vm_exit_reason =
4640                                 EXIT_REASON_EXTERNAL_INTERRUPT;
4641                         vmcs12->vm_exit_intr_info = 0;
4642                         /*
4643                          * fall through to normal code, but now in L1, not L2
4644                          */
4645                 }
4646         }
4647 
4648         return (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
4649                 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
4650                         (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS));
4651 }
4652 
4653 static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
4654 {
4655         int ret;
4656         struct kvm_userspace_memory_region tss_mem = {
4657                 .slot = TSS_PRIVATE_MEMSLOT,
4658                 .guest_phys_addr = addr,
4659                 .memory_size = PAGE_SIZE * 3,
4660                 .flags = 0,
4661         };
4662 
4663         ret = kvm_set_memory_region(kvm, &tss_mem);
4664         if (ret)
4665                 return ret;
4666         kvm->arch.tss_addr = addr;
4667         if (!init_rmode_tss(kvm))
4668                 return  -ENOMEM;
4669 
4670         return 0;
4671 }
4672 
4673 static bool rmode_exception(struct kvm_vcpu *vcpu, int vec)
4674 {
4675         switch (vec) {
4676         case BP_VECTOR:
4677                 /*
4678                  * Update instruction length as we may reinject the exception
4679                  * from user space while in guest debugging mode.
4680                  */
4681                 to_vmx(vcpu)->vcpu.arch.event_exit_inst_len =
4682                         vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
4683                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
4684                         return false;
4685                 /* fall through */
4686         case DB_VECTOR:
4687                 if (vcpu->guest_debug &
4688                         (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
4689                         return false;
4690                 /* fall through */
4691         case DE_VECTOR:
4692         case OF_VECTOR:
4693         case BR_VECTOR:
4694         case UD_VECTOR:
4695         case DF_VECTOR:
4696         case SS_VECTOR:
4697         case GP_VECTOR:
4698         case MF_VECTOR:
4699                 return true;
4700         break;
4701         }
4702         return false;
4703 }
4704 
4705 static int handle_rmode_exception(struct kvm_vcpu *vcpu,
4706                                   int vec, u32 err_code)
4707 {
4708         /*
4709          * Instruction with address size override prefix opcode 0x67
4710          * Cause the #SS fault with 0 error code in VM86 mode.
4711          */
4712         if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) {
4713                 if (emulate_instruction(vcpu, 0) == EMULATE_DONE) {
4714                         if (vcpu->arch.halt_request) {
4715                                 vcpu->arch.halt_request = 0;
4716                                 return kvm_emulate_halt(vcpu);
4717                         }
4718                         return 1;
4719                 }
4720                 return 0;
4721         }
4722 
4723         /*
4724          * Forward all other exceptions that are valid in real mode.
4725          * FIXME: Breaks guest debugging in real mode, needs to be fixed with
4726          *        the required debugging infrastructure rework.
4727          */
4728         kvm_queue_exception(vcpu, vec);
4729         return 1;
4730 }
4731 
4732 /*
4733  * Trigger machine check on the host. We assume all the MSRs are already set up
4734  * by the CPU and that we still run on the same CPU as the MCE occurred on.
4735  * We pass a fake environment to the machine check handler because we want
4736  * the guest to be always treated like user space, no matter what context
4737  * it used internally.
4738  */
4739 static void kvm_machine_check(void)
4740 {
4741 #if defined(CONFIG_X86_MCE) && defined(CONFIG_X86_64)
4742         struct pt_regs regs = {
4743                 .cs = 3, /* Fake ring 3 no matter what the guest ran on */
4744                 .flags = X86_EFLAGS_IF,
4745         };
4746 
4747         do_machine_check(&regs, 0);
4748 #endif
4749 }
4750 
4751 static int handle_machine_check(struct kvm_vcpu *vcpu)
4752 {
4753         /* already handled by vcpu_run */
4754         return 1;
4755 }
4756 
4757 static int handle_exception(struct kvm_vcpu *vcpu)
4758 {
4759         struct vcpu_vmx *vmx = to_vmx(vcpu);
4760         struct kvm_run *kvm_run = vcpu->run;
4761         u32 intr_info, ex_no, error_code;
4762         unsigned long cr2, rip, dr6;
4763         u32 vect_info;
4764         enum emulation_result er;
4765 
4766         vect_info = vmx->idt_vectoring_info;
4767         intr_info = vmx->exit_intr_info;
4768 
4769         if (is_machine_check(intr_info))
4770                 return handle_machine_check(vcpu);
4771 
4772         if (is_nmi(intr_info))
4773                 return 1;  /* already handled by vmx_vcpu_run() */
4774 
4775         if (is_no_device(intr_info)) {
4776                 vmx_fpu_activate(vcpu);
4777                 return 1;
4778         }
4779 
4780         if (is_invalid_opcode(intr_info)) {
4781                 er = emulate_instruction(vcpu, EMULTYPE_TRAP_UD);
4782                 if (er != EMULATE_DONE)
4783                         kvm_queue_exception(vcpu, UD_VECTOR);
4784                 return 1;
4785         }
4786 
4787         error_code = 0;
4788         if (intr_info & INTR_INFO_DELIVER_CODE_MASK)
4789                 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
4790 
4791         /*
4792          * The #PF with PFEC.RSVD = 1 indicates the guest is accessing
4793          * MMIO, it is better to report an internal error.
4794          * See the comments in vmx_handle_exit.
4795          */
4796         if ((vect_info & VECTORING_INFO_VALID_MASK) &&
4797             !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) {
4798                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
4799                 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX;
4800                 vcpu->run->internal.ndata = 2;
4801                 vcpu->run->internal.data[0] = vect_info;
4802                 vcpu->run->internal.data[1] = intr_info;
4803                 return 0;
4804         }
4805 
4806         if (is_page_fault(intr_info)) {
4807                 /* EPT won't cause page fault directly */
4808                 BUG_ON(enable_ept);
4809                 cr2 = vmcs_readl(EXIT_QUALIFICATION);
4810                 trace_kvm_page_fault(cr2, error_code);
4811 
4812                 if (kvm_event_needs_reinjection(vcpu))
4813                         kvm_mmu_unprotect_page_virt(vcpu, cr2);
4814                 return kvm_mmu_page_fault(vcpu, cr2, error_code, NULL, 0);
4815         }
4816 
4817         ex_no = intr_info & INTR_INFO_VECTOR_MASK;
4818 
4819         if (vmx->rmode.vm86_active && rmode_exception(vcpu, ex_no))
4820                 return handle_rmode_exception(vcpu, ex_no, error_code);
4821 
4822         switch (ex_no) {
4823         case AC_VECTOR:
4824                 kvm_queue_exception_e(vcpu, AC_VECTOR, error_code);
4825                 return 1;
4826         case DB_VECTOR:
4827                 dr6 = vmcs_readl(EXIT_QUALIFICATION);
4828                 if (!(vcpu->guest_debug &
4829                       (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
4830                         vcpu->arch.dr6 = dr6 | DR6_FIXED_1;
4831                         kvm_queue_exception(vcpu, DB_VECTOR);
4832                         return 1;
4833                 }
4834                 kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1;
4835                 kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7);
4836                 /* fall through */
4837         case BP_VECTOR:
4838                 /*
4839                  * Update instruction length as we may reinject #BP from
4840                  * user space while in guest debugging mode. Reading it for
4841                  * #DB as well causes no harm, it is not used in that case.
4842                  */
4843                 vmx->vcpu.arch.event_exit_inst_len =
4844                         vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
4845                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
4846                 rip = kvm_rip_read(vcpu);
4847                 kvm_run->debug.arch.pc = vmcs_readl(GUEST_CS_BASE) + rip;
4848                 kvm_run->debug.arch.exception = ex_no;
4849                 break;
4850         default:
4851                 kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
4852                 kvm_run->ex.exception = ex_no;
4853                 kvm_run->ex.error_code = error_code;
4854                 break;
4855         }
4856         return 0;
4857 }
4858 
4859 static int handle_external_interrupt(struct kvm_vcpu *vcpu)
4860 {
4861         ++vcpu->stat.irq_exits;
4862         return 1;
4863 }
4864 
4865 static int handle_triple_fault(struct kvm_vcpu *vcpu)
4866 {
4867         vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
4868         return 0;
4869 }
4870 
4871 static int handle_io(struct kvm_vcpu *vcpu)
4872 {
4873         unsigned long exit_qualification;
4874         int size, in, string;
4875         unsigned port;
4876 
4877         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
4878         string = (exit_qualification & 16) != 0;
4879         in = (exit_qualification & 8) != 0;
4880 
4881         ++vcpu->stat.io_exits;
4882 
4883         if (string || in)
4884                 return emulate_instruction(vcpu, 0) == EMULATE_DONE;
4885 
4886         port = exit_qualification >> 16;
4887         size = (exit_qualification & 7) + 1;
4888         skip_emulated_instruction(vcpu);
4889 
4890         return kvm_fast_pio_out(vcpu, size, port);
4891 }
4892 
4893 static void
4894 vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
4895 {
4896         /*
4897          * Patch in the VMCALL instruction:
4898          */
4899         hypercall[0] = 0x0f;
4900         hypercall[1] = 0x01;
4901         hypercall[2] = 0xc1;
4902 }
4903 
4904 /* called to set cr0 as appropriate for a mov-to-cr0 exit. */
4905 static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val)
4906 {
4907         if (is_guest_mode(vcpu)) {
4908                 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
4909                 unsigned long orig_val = val;
4910 
4911                 /*
4912                  * We get here when L2 changed cr0 in a way that did not change
4913                  * any of L1's shadowed bits (see nested_vmx_exit_handled_cr),
4914                  * but did change L0 shadowed bits. So we first calculate the
4915                  * effective cr0 value that L1 would like to write into the
4916                  * hardware. It consists of the L2-owned bits from the new
4917                  * value combined with the L1-owned bits from L1's guest_cr0.
4918                  */
4919                 val = (val & ~vmcs12->cr0_guest_host_mask) |
4920                         (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask);
4921 
4922                 /* TODO: will have to take unrestricted guest mode into
4923                  * account */
4924                 if ((val & VMXON_CR0_ALWAYSON) != VMXON_CR0_ALWAYSON)
4925                         return 1;
4926 
4927                 if (kvm_set_cr0(vcpu, val))
4928                         return 1;
4929                 vmcs_writel(CR0_READ_SHADOW, orig_val);
4930                 return 0;
4931         } else {
4932                 if (to_vmx(vcpu)->nested.vmxon &&
4933                     ((val & VMXON_CR0_ALWAYSON) != VMXON_CR0_ALWAYSON))
4934                         return 1;
4935                 return kvm_set_cr0(vcpu, val);
4936         }
4937 }
4938 
4939 static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val)
4940 {
4941         if (is_guest_mode(vcpu)) {
4942                 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
4943                 unsigned long orig_val = val;
4944 
4945                 /* analogously to handle_set_cr0 */
4946                 val = (val & ~vmcs12->cr4_guest_host_mask) |
4947                         (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask);
4948                 if (kvm_set_cr4(vcpu, val))
4949                         return 1;
4950                 vmcs_writel(CR4_READ_SHADOW, orig_val);
4951                 return 0;
4952         } else
4953                 return kvm_set_cr4(vcpu, val);
4954 }
4955 
4956 /* called to set cr0 as approriate for clts instruction exit. */
4957 static void handle_clts(struct kvm_vcpu *vcpu)
4958 {
4959         if (is_guest_mode(vcpu)) {
4960                 /*
4961                  * We get here when L2 did CLTS, and L1 didn't shadow CR0.TS
4962                  * but we did (!fpu_active). We need to keep GUEST_CR0.TS on,
4963                  * just pretend it's off (also in arch.cr0 for fpu_activate).
4964                  */
4965                 vmcs_writel(CR0_READ_SHADOW,
4966                         vmcs_readl(CR0_READ_SHADOW) & ~X86_CR0_TS);
4967                 vcpu->arch.cr0 &= ~X86_CR0_TS;
4968         } else
4969                 vmx_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~X86_CR0_TS));
4970 }
4971 
4972 static int handle_cr(struct kvm_vcpu *vcpu)
4973 {
4974         unsigned long exit_qualification, val;
4975         int cr;
4976         int reg;
4977         int err;
4978 
4979         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
4980         cr = exit_qualification & 15;
4981         reg = (exit_qualification >> 8) & 15;
4982         switch ((exit_qualification >> 4) & 3) {
4983         case 0: /* mov to cr */
4984                 val = kvm_register_read(vcpu, reg);
4985                 trace_kvm_cr_write(cr, val);
4986                 switch (cr) {
4987                 case 0:
4988                         err = handle_set_cr0(vcpu, val);
4989                         kvm_complete_insn_gp(vcpu, err);
4990                         return 1;
4991                 case 3:
4992                         err = kvm_set_cr3(vcpu, val);
4993                         kvm_complete_insn_gp(vcpu, err);
4994                         return 1;
4995                 case 4:
4996                         err = handle_set_cr4(vcpu, val);
4997                         kvm_complete_insn_gp(vcpu, err);
4998                         return 1;
4999                 case 8: {
5000                                 u8 cr8_prev = kvm_get_cr8(vcpu);
5001                                 u8 cr8 = kvm_register_read(vcpu, reg);
5002                                 err = kvm_set_cr8(vcpu, cr8);
5003                                 kvm_complete_insn_gp(vcpu, err);
5004                                 if (irqchip_in_kernel(vcpu->kvm))
5005                                         return 1;
5006                                 if (cr8_prev <= cr8)
5007                                         return 1;
5008                                 vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
5009                                 return 0;
5010                         }
5011                 }
5012                 break;
5013         case 2: /* clts */
5014                 handle_clts(vcpu);
5015                 trace_kvm_cr_write(0, kvm_read_cr0(vcpu));
5016                 skip_emulated_instruction(vcpu);
5017                 vmx_fpu_activate(vcpu);
5018                 return 1;
5019         case 1: /*mov from cr*/
5020                 switch (cr) {
5021                 case 3:
5022                         val = kvm_read_cr3(vcpu);
5023                         kvm_register_write(vcpu, reg, val);
5024                         trace_kvm_cr_read(cr, val);
5025                         skip_emulated_instruction(vcpu);
5026                         return 1;
5027                 case 8:
5028                         val = kvm_get_cr8(vcpu);
5029                         kvm_register_write(vcpu, reg, val);
5030                         trace_kvm_cr_read(cr, val);
5031                         skip_emulated_instruction(vcpu);
5032                         return 1;
5033                 }
5034                 break;
5035         case 3: /* lmsw */
5036                 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
5037                 trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val);
5038                 kvm_lmsw(vcpu, val);
5039 
5040                 skip_emulated_instruction(vcpu);
5041                 return 1;
5042         default:
5043                 break;
5044         }
5045         vcpu->run->exit_reason = 0;
5046         vcpu_unimpl(vcpu, "unhandled control register: op %d cr %d\n",
5047                (int)(exit_qualification >> 4) & 3, cr);
5048         return 0;
5049 }
5050 
5051 static int handle_dr(struct kvm_vcpu *vcpu)
5052 {
5053         unsigned long exit_qualification;
5054         int dr, reg;
5055 
5056         /* Do not handle if the CPL > 0, will trigger GP on re-entry */
5057         if (!kvm_require_cpl(vcpu, 0))
5058                 return 1;
5059         dr = vmcs_readl(GUEST_DR7);
5060         if (dr & DR7_GD) {
5061                 /*
5062                  * As the vm-exit takes precedence over the debug trap, we
5063                  * need to emulate the latter, either for the host or the
5064                  * guest debugging itself.
5065                  */
5066                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
5067                         vcpu->run->debug.arch.dr6 = vcpu->arch.dr6;
5068                         vcpu->run->debug.arch.dr7 = dr;
5069                         vcpu->run->debug.arch.pc =
5070                                 vmcs_readl(GUEST_CS_BASE) +
5071                                 vmcs_readl(GUEST_RIP);
5072                         vcpu->run->debug.arch.exception = DB_VECTOR;
5073                         vcpu->run->exit_reason = KVM_EXIT_DEBUG;
5074                         return 0;
5075                 } else {
5076                         vcpu->arch.dr7 &= ~DR7_GD;
5077                         vcpu->arch.dr6 |= DR6_BD;
5078                         vmcs_writel(GUEST_DR7, vcpu->arch.dr7);
5079                         kvm_queue_exception(vcpu, DB_VECTOR);
5080                         return 1;
5081                 }
5082         }
5083 
5084         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5085         dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
5086         reg = DEBUG_REG_ACCESS_REG(exit_qualification);
5087         if (exit_qualification & TYPE_MOV_FROM_DR) {
5088                 unsigned long val;
5089                 if (!kvm_get_dr(vcpu, dr, &val))
5090                         kvm_register_write(vcpu, reg, val);
5091         } else
5092                 kvm_set_dr(vcpu, dr, vcpu->arch.regs[reg]);
5093         skip_emulated_instruction(vcpu);
5094         return 1;
5095 }
5096 
5097 static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
5098 {
5099         vmcs_writel(GUEST_DR7, val);
5100 }
5101 
5102 static int handle_cpuid(struct kvm_vcpu *vcpu)
5103 {
5104         kvm_emulate_cpuid(vcpu);
5105         return 1;
5106 }
5107 
5108 static int handle_rdmsr(struct kvm_vcpu *vcpu)
5109 {
5110         u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
5111         u64 data;
5112 
5113         if (vmx_get_msr(vcpu, ecx, &data)) {
5114                 trace_kvm_msr_read_ex(ecx);
5115                 kvm_inject_gp(vcpu, 0);
5116                 return 1;
5117         }
5118 
5119         trace_kvm_msr_read(ecx, data);
5120 
5121         /* FIXME: handling of bits 32:63 of rax, rdx */
5122         vcpu->arch.regs[VCPU_REGS_RAX] = data & -1u;
5123         vcpu->arch.regs[VCPU_REGS_RDX] = (data >> 32) & -1u;
5124         skip_emulated_instruction(vcpu);
5125         return 1;
5126 }
5127 
5128 static int handle_wrmsr(struct kvm_vcpu *vcpu)
5129 {
5130         struct msr_data msr;
5131         u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
5132         u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u)
5133                 | ((u64)(vcpu->arch.regs[VCPU_REGS_RDX] & -1u) << 32);
5134 
5135         msr.data = data;
5136         msr.index = ecx;
5137         msr.host_initiated = false;
5138         if (kvm_set_msr(vcpu, &msr) != 0) {
5139                 trace_kvm_msr_write_ex(ecx, data);
5140                 kvm_inject_gp(vcpu, 0);
5141                 return 1;
5142         }
5143 
5144         trace_kvm_msr_write(ecx, data);
5145         skip_emulated_instruction(vcpu);
5146         return 1;
5147 }
5148 
5149 static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)
5150 {
5151         kvm_make_request(KVM_REQ_EVENT, vcpu);
5152         return 1;
5153 }
5154 
5155 static int handle_interrupt_window(struct kvm_vcpu *vcpu)
5156 {
5157         u32 cpu_based_vm_exec_control;
5158 
5159         /* clear pending irq */
5160         cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
5161         cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
5162         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
5163 
5164         kvm_make_request(KVM_REQ_EVENT, vcpu);
5165 
5166         ++vcpu->stat.irq_window_exits;
5167 
5168         /*
5169          * If the user space waits to inject interrupts, exit as soon as
5170          * possible
5171          */
5172         if (!irqchip_in_kernel(vcpu->kvm) &&
5173             vcpu->run->request_interrupt_window &&
5174             !kvm_cpu_has_interrupt(vcpu)) {
5175                 vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
5176                 return 0;
5177         }
5178         return 1;
5179 }
5180 
5181 static int handle_halt(struct kvm_vcpu *vcpu)
5182 {
5183         skip_emulated_instruction(vcpu);
5184         return kvm_emulate_halt(vcpu);
5185 }
5186 
5187 static int handle_vmcall(struct kvm_vcpu *vcpu)
5188 {
5189         skip_emulated_instruction(vcpu);
5190         kvm_emulate_hypercall(vcpu);
5191         return 1;
5192 }
5193 
5194 static int handle_invd(struct kvm_vcpu *vcpu)
5195 {
5196         return emulate_instruction(vcpu, 0) == EMULATE_DONE;
5197 }
5198 
5199 static int handle_invlpg(struct kvm_vcpu *vcpu)
5200 {
5201         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5202 
5203         kvm_mmu_invlpg(vcpu, exit_qualification);
5204         skip_emulated_instruction(vcpu);
5205         return 1;
5206 }
5207 
5208 static int handle_rdpmc(struct kvm_vcpu *vcpu)
5209 {
5210         int err;
5211 
5212         err = kvm_rdpmc(vcpu);
5213         kvm_complete_insn_gp(vcpu, err);
5214 
5215         return 1;
5216 }
5217 
5218 static int handle_wbinvd(struct kvm_vcpu *vcpu)
5219 {
5220         skip_emulated_instruction(vcpu);
5221         kvm_emulate_wbinvd(vcpu);
5222         return 1;
5223 }
5224 
5225 static int handle_xsetbv(struct kvm_vcpu *vcpu)
5226 {
5227         u64 new_bv = kvm_read_edx_eax(vcpu);
5228         u32 index = kvm_register_read(vcpu, VCPU_REGS_RCX);
5229 
5230         if (kvm_set_xcr(vcpu, index, new_bv) == 0)
5231                 skip_emulated_instruction(vcpu);
5232         return 1;
5233 }
5234 
5235 static int handle_apic_access(struct kvm_vcpu *vcpu)
5236 {
5237         if (likely(fasteoi)) {
5238                 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5239                 int access_type, offset;
5240 
5241                 access_type = exit_qualification & APIC_ACCESS_TYPE;
5242                 offset = exit_qualification & APIC_ACCESS_OFFSET;
5243                 /*
5244                  * Sane guest uses MOV to write EOI, with written value
5245                  * not cared. So make a short-circuit here by avoiding
5246                  * heavy instruction emulation.
5247                  */
5248                 if ((access_type == TYPE_LINEAR_APIC_INST_WRITE) &&
5249                     (offset == APIC_EOI)) {
5250                         kvm_lapic_set_eoi(vcpu);
5251                         skip_emulated_instruction(vcpu);
5252                         return 1;
5253                 }
5254         }
5255         return emulate_instruction(vcpu, 0) == EMULATE_DONE;
5256 }
5257 
5258 static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu)
5259 {
5260         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5261         int vector = exit_qualification & 0xff;
5262 
5263         /* EOI-induced VM exit is trap-like and thus no need to adjust IP */
5264         kvm_apic_set_eoi_accelerated(vcpu, vector);
5265         return 1;
5266 }
5267 
5268 static int handle_apic_write(struct kvm_vcpu *vcpu)
5269 {
5270         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5271         u32 offset = exit_qualification & 0xfff;
5272 
5273         /* APIC-write VM exit is trap-like and thus no need to adjust IP */
5274         kvm_apic_write_nodecode(vcpu, offset);
5275         return 1;
5276 }
5277 
5278 static int handle_task_switch(struct kvm_vcpu *vcpu)
5279 {
5280         struct vcpu_vmx *vmx = to_vmx(vcpu);
5281         unsigned long exit_qualification;
5282         bool has_error_code = false;
5283         u32 error_code = 0;
5284         u16 tss_selector;
5285         int reason, type, idt_v, idt_index;
5286 
5287         idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK);
5288         idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK);
5289         type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK);
5290 
5291         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5292 
5293         reason = (u32)exit_qualification >> 30;
5294         if (reason == TASK_SWITCH_GATE && idt_v) {
5295                 switch (type) {
5296                 case INTR_TYPE_NMI_INTR:
5297                         vcpu->arch.nmi_injected = false;
5298                         vmx_set_nmi_mask(vcpu, true);
5299                         break;
5300                 case INTR_TYPE_EXT_INTR:
5301                 case INTR_TYPE_SOFT_INTR:
5302                         kvm_clear_interrupt_queue(vcpu);
5303                         break;
5304                 case INTR_TYPE_HARD_EXCEPTION:
5305                         if (vmx->idt_vectoring_info &
5306                             VECTORING_INFO_DELIVER_CODE_MASK) {
5307                                 has_error_code = true;
5308                                 error_code =
5309                                         vmcs_read32(IDT_VECTORING_ERROR_CODE);
5310                         }
5311                         /* fall through */
5312                 case INTR_TYPE_SOFT_EXCEPTION:
5313                         kvm_clear_exception_queue(vcpu);
5314                         break;
5315                 default:
5316                         break;
5317                 }
5318         }
5319         tss_selector = exit_qualification;
5320 
5321         if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION &&
5322                        type != INTR_TYPE_EXT_INTR &&
5323                        type != INTR_TYPE_NMI_INTR))
5324                 skip_emulated_instruction(vcpu);
5325 
5326         if (kvm_task_switch(vcpu, tss_selector,
5327                             type == INTR_TYPE_SOFT_INTR ? idt_index : -1, reason,
5328                             has_error_code, error_code) == EMULATE_FAIL) {
5329                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
5330                 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
5331                 vcpu->run->internal.ndata = 0;
5332                 return 0;
5333         }
5334 
5335         /* clear all local breakpoint enable flags */
5336         vmcs_writel(GUEST_DR7, vmcs_readl(GUEST_DR7) & ~55);
5337 
5338         /*
5339          * TODO: What about debug traps on tss switch?
5340          *       Are we supposed to inject them and update dr6?
5341          */
5342 
5343         return 1;
5344 }
5345 
5346 static int handle_ept_violation(struct kvm_vcpu *vcpu)
5347 {
5348         unsigned long exit_qualification;
5349         gpa_t gpa;
5350         u32 error_code;
5351         int gla_validity;
5352 
5353         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5354 
5355         gla_validity = (exit_qualification >> 7) & 0x3;
5356         if (gla_validity != 0x3 && gla_validity != 0x1 && gla_validity != 0) {
5357                 printk(KERN_ERR "EPT: Handling EPT violation failed!\n");
5358                 printk(KERN_ERR "EPT: GPA: 0x%lx, GVA: 0x%lx\n",
5359                         (long unsigned int)vmcs_read64(GUEST_PHYSICAL_ADDRESS),
5360                         vmcs_readl(GUEST_LINEAR_ADDRESS));
5361                 printk(KERN_ERR "EPT: Exit qualification is 0x%lx\n",
5362                         (long unsigned int)exit_qualification);
5363                 vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
5364                 vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_VIOLATION;
5365                 return 0;
5366         }
5367 
5368         /*
5369          * EPT violation happened while executing iret from NMI,
5370          * "blocked by NMI" bit has to be set before next VM entry.
5371          * There are errata that may cause this bit to not be set:
5372          * AAK134, BY25.
5373          */
5374         if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
5375                         cpu_has_virtual_nmis() &&
5376                         (exit_qualification & INTR_INFO_UNBLOCK_NMI))
5377                 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI);
5378 
5379         gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
5380         trace_kvm_page_fault(gpa, exit_qualification);
5381 
5382         /* It is a write fault? */
5383         error_code = exit_qualification & (1U << 1);
5384         /* It is a fetch fault? */
5385         error_code |= (exit_qualification & (1U << 2)) << 2;
5386         /* ept page table is present? */
5387         error_code |= (exit_qualification >> 3) & 0x1;
5388 
5389         vcpu->arch.exit_qualification = exit_qualification;
5390 
5391         return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0);
5392 }
5393 
5394 static u64 ept_rsvd_mask(u64 spte, int level)
5395 {
5396         int i;
5397         u64 mask = 0;
5398 
5399         for (i = 51; i > boot_cpu_data.x86_phys_bits; i--)
5400                 mask |= (1ULL << i);
5401 
5402         if (level > 2)
5403                 /* bits 7:3 reserved */
5404                 mask |= 0xf8;
5405         else if (level == 2) {
5406                 if (spte & (1ULL << 7))
5407                         /* 2MB ref, bits 20:12 reserved */
5408                         mask |= 0x1ff000;
5409                 else
5410                         /* bits 6:3 reserved */
5411                         mask |= 0x78;
5412         }
5413 
5414         return mask;
5415 }
5416 
5417 static void ept_misconfig_inspect_spte(struct kvm_vcpu *vcpu, u64 spte,
5418                                        int level)
5419 {
5420         printk(KERN_ERR "%s: spte 0x%llx level %d\n", __func__, spte, level);
5421 
5422         /* 010b (write-only) */
5423         WARN_ON((spte & 0x7) == 0x2);
5424 
5425         /* 110b (write/execute) */
5426         WARN_ON((spte & 0x7) == 0x6);
5427 
5428         /* 100b (execute-only) and value not supported by logical processor */
5429         if (!cpu_has_vmx_ept_execute_only())
5430                 WARN_ON((spte & 0x7) == 0x4);
5431 
5432         /* not 000b */
5433         if ((spte & 0x7)) {
5434                 u64 rsvd_bits = spte & ept_rsvd_mask(spte, level);
5435 
5436                 if (rsvd_bits != 0) {
5437                         printk(KERN_ERR "%s: rsvd_bits = 0x%llx\n",
5438                                          __func__, rsvd_bits);
5439                         WARN_ON(1);
5440                 }
5441 
5442                 if (level == 1 || (level == 2 && (spte & (1ULL << 7)))) {
5443                         u64 ept_mem_type = (spte & 0x38) >> 3;
5444 
5445                         if (ept_mem_type == 2 || ept_mem_type == 3 ||
5446                             ept_mem_type == 7) {
5447                                 printk(KERN_ERR "%s: ept_mem_type=0x%llx\n",
5448                                                 __func__, ept_mem_type);
5449                                 WARN_ON(1);
5450                         }
5451                 }
5452         }
5453 }
5454 
5455 static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
5456 {
5457         u64 sptes[4];
5458         int nr_sptes, i, ret;
5459         gpa_t gpa;
5460 
5461         gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
5462 
5463         ret = handle_mmio_page_fault_common(vcpu, gpa, true);
5464         if (likely(ret == RET_MMIO_PF_EMULATE))
5465                 return x86_emulate_instruction(vcpu, gpa, 0, NULL, 0) ==
5466                                               EMULATE_DONE;
5467 
5468         if (unlikely(ret == RET_MMIO_PF_INVALID))
5469                 return kvm_mmu_page_fault(vcpu, gpa, 0, NULL, 0);
5470 
5471         if (unlikely(ret == RET_MMIO_PF_RETRY))
5472                 return 1;
5473 
5474         /* It is the real ept misconfig */
5475         printk(KERN_ERR "EPT: Misconfiguration.\n");
5476         printk(KERN_ERR "EPT: GPA: 0x%llx\n", gpa);
5477 
5478         nr_sptes = kvm_mmu_get_spte_hierarchy(vcpu, gpa, sptes);
5479 
5480         for (i = PT64_ROOT_LEVEL; i > PT64_ROOT_LEVEL - nr_sptes; --i)
5481                 ept_misconfig_inspect_spte(vcpu, sptes[i-1], i);
5482 
5483         vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
5484         vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_MISCONFIG;
5485 
5486         return 0;
5487 }
5488 
5489 static int handle_nmi_window(struct kvm_vcpu *vcpu)
5490 {
5491         u32 cpu_based_vm_exec_control;
5492 
5493         /* clear pending NMI */
5494         cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
5495         cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING;
5496         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
5497         ++vcpu->stat.nmi_window_exits;
5498         kvm_make_request(KVM_REQ_EVENT, vcpu);
5499 
5500         return 1;
5501 }
5502 
5503 static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
5504 {
5505         struct vcpu_vmx *vmx = to_vmx(vcpu);
5506         enum emulation_result err = EMULATE_DONE;
5507         int ret = 1;
5508         u32 cpu_exec_ctrl;
5509         bool intr_window_requested;
5510         unsigned count = 130;
5511 
5512         cpu_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
5513         intr_window_requested = cpu_exec_ctrl & CPU_BASED_VIRTUAL_INTR_PENDING;
5514 
5515         while (!guest_state_valid(vcpu) && count-- != 0) {
5516                 if (intr_window_requested && vmx_interrupt_allowed(vcpu))
5517                         return handle_interrupt_window(&vmx->vcpu);
5518 
5519                 if (test_bit(KVM_REQ_EVENT, &vcpu->requests))
5520                         return 1;
5521 
5522                 err = emulate_instruction(vcpu, EMULTYPE_NO_REEXECUTE);
5523 
5524                 if (err == EMULATE_USER_EXIT) {
5525                         ++vcpu->stat.mmio_exits;
5526                         ret = 0;
5527                         goto out;
5528                 }
5529 
5530                 if (err != EMULATE_DONE) {
5531                         vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
5532                         vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
5533                         vcpu->run->internal.ndata = 0;
5534                         return 0;
5535                 }
5536 
5537                 if (vcpu->arch.halt_request) {
5538                         vcpu->arch.halt_request = 0;
5539                         ret = kvm_emulate_halt(vcpu);
5540                         goto out;
5541                 }
5542 
5543                 if (signal_pending(current))
5544                         goto out;
5545                 if (need_resched())
5546                         schedule();
5547         }
5548 
5549         vmx->emulation_required = emulation_required(vcpu);
5550 out:
5551         return ret;
5552 }
5553 
5554 /*
5555  * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE
5556  * exiting, so only get here on cpu with PAUSE-Loop-Exiting.
5557  */
5558 static int handle_pause(struct kvm_vcpu *vcpu)
5559 {
5560         skip_emulated_instruction(vcpu);
5561         kvm_vcpu_on_spin(vcpu);
5562 
5563         return 1;
5564 }
5565 
5566 static int handle_invalid_op(struct kvm_vcpu *vcpu)
5567 {
5568         kvm_queue_exception(vcpu, UD_VECTOR);
5569         return 1;
5570 }
5571 
5572 /*
5573  * To run an L2 guest, we need a vmcs02 based on the L1-specified vmcs12.
5574  * We could reuse a single VMCS for all the L2 guests, but we also want the
5575  * option to allocate a separate vmcs02 for each separate loaded vmcs12 - this
5576  * allows keeping them loaded on the processor, and in the future will allow
5577  * optimizations where prepare_vmcs02 doesn't need to set all the fields on
5578  * every entry if they never change.
5579  * So we keep, in vmx->nested.vmcs02_pool, a cache of size VMCS02_POOL_SIZE
5580  * (>=0) with a vmcs02 for each recently loaded vmcs12s, most recent first.
5581  *
5582  * The following functions allocate and free a vmcs02 in this pool.
5583  */
5584 
5585 /* Get a VMCS from the pool to use as vmcs02 for the current vmcs12. */
5586 static struct loaded_vmcs *nested_get_current_vmcs02(struct vcpu_vmx *vmx)
5587 {
5588         struct vmcs02_list *item;
5589         list_for_each_entry(item, &vmx->nested.vmcs02_pool, list)
5590                 if (item->vmptr == vmx->nested.current_vmptr) {
5591                         list_move(&item->list, &vmx->nested.vmcs02_pool);
5592                         return &item->vmcs02;
5593                 }
5594 
5595         if (vmx->nested.vmcs02_num >= max(VMCS02_POOL_SIZE, 1)) {
5596                 /* Recycle the least recently used VMCS. */
5597                 item = list_entry(vmx->nested.vmcs02_pool.prev,
5598                         struct vmcs02_list, list);
5599                 item->vmptr = vmx->nested.current_vmptr;
5600                 list_move(&item->list, &vmx->nested.vmcs02_pool);
5601                 return &item->vmcs02;
5602         }
5603 
5604         /* Create a new VMCS */
5605         item = kmalloc(sizeof(struct vmcs02_list), GFP_KERNEL);
5606         if (!item)
5607                 return NULL;
5608         item->vmcs02.vmcs = alloc_vmcs();
5609         if (!item->vmcs02.vmcs) {
5610                 kfree(item);
5611                 return NULL;
5612         }
5613         loaded_vmcs_init(&item->vmcs02);
5614         item->vmptr = vmx->nested.current_vmptr;
5615         list_add(&(item->list), &(vmx->nested.vmcs02_pool));
5616         vmx->nested.vmcs02_num++;
5617         return &item->vmcs02;
5618 }
5619 
5620 /* Free and remove from pool a vmcs02 saved for a vmcs12 (if there is one) */
5621 static void nested_free_vmcs02(struct vcpu_vmx *vmx, gpa_t vmptr)
5622 {
5623         struct vmcs02_list *item;
5624         list_for_each_entry(item, &vmx->nested.vmcs02_pool, list)
5625                 if (item->vmptr == vmptr) {
5626                         free_loaded_vmcs(&item->vmcs02);
5627                         list_del(&item->list);
5628                         kfree(item);
5629                         vmx->nested.vmcs02_num--;
5630                         return;
5631                 }
5632 }
5633 
5634 /*
5635  * Free all VMCSs saved for this vcpu, except the one pointed by
5636  * vmx->loaded_vmcs. These include the VMCSs in vmcs02_pool (except the one
5637  * currently used, if running L2), and vmcs01 when running L2.
5638  */
5639 static void nested_free_all_saved_vmcss(struct vcpu_vmx *vmx)
5640 {
5641         struct vmcs02_list *item, *n;
5642         list_for_each_entry_safe(item, n, &vmx->nested.vmcs02_pool, list) {
5643                 if (vmx->loaded_vmcs != &item->vmcs02)
5644                         free_loaded_vmcs(&item->vmcs02);
5645                 list_del(&item->list);
5646                 kfree(item);
5647         }
5648         vmx->nested.vmcs02_num = 0;
5649 
5650         if (vmx->loaded_vmcs != &vmx->vmcs01)
5651                 free_loaded_vmcs(&vmx->vmcs01);
5652 }
5653 
5654 /*
5655  * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(),
5656  * set the success or error code of an emulated VMX instruction, as specified
5657  * by Vol 2B, VMX Instruction Reference, "Conventions".
5658  */
5659 static void nested_vmx_succeed(struct kvm_vcpu *vcpu)
5660 {
5661         vmx_set_rflags(vcpu, vmx_get_rflags(vcpu)
5662                         & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
5663                             X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF));
5664 }
5665 
5666 static void nested_vmx_failInvalid(struct kvm_vcpu *vcpu)
5667 {
5668         vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
5669                         & ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
5670                             X86_EFLAGS_SF | X86_EFLAGS_OF))
5671                         | X86_EFLAGS_CF);
5672 }
5673 
5674 static void nested_vmx_failValid(struct kvm_vcpu *vcpu,
5675                                         u32 vm_instruction_error)
5676 {
5677         if (to_vmx(vcpu)->nested.current_vmptr == -1ull) {
5678                 /*
5679                  * failValid writes the error number to the current VMCS, which
5680                  * can't be done there isn't a current VMCS.
5681                  */
5682                 nested_vmx_failInvalid(vcpu);
5683                 return;
5684         }
5685         vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
5686                         & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
5687                             X86_EFLAGS_SF | X86_EFLAGS_OF))
5688                         | X86_EFLAGS_ZF);
5689         get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error;
5690         /*
5691          * We don't need to force a shadow sync because
5692          * VM_INSTRUCTION_ERROR is not shadowed
5693          */
5694 }
5695 
5696 /*
5697  * Emulate the VMXON instruction.
5698  * Currently, we just remember that VMX is active, and do not save or even
5699  * inspect the argument to VMXON (the so-called "VMXON pointer") because we
5700  * do not currently need to store anything in that guest-allocated memory
5701  * region. Consequently, VMCLEAR and VMPTRLD also do not verify that the their
5702  * argument is different from the VMXON pointer (which the spec says they do).
5703  */
5704 static int handle_vmon(struct kvm_vcpu *vcpu)
5705 {
5706         struct kvm_segment cs;
5707         struct vcpu_vmx *vmx = to_vmx(vcpu);
5708         struct vmcs *shadow_vmcs;
5709         const u64 VMXON_NEEDED_FEATURES = FEATURE_CONTROL_LOCKED
5710                 | FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
5711 
5712         /* The Intel VMX Instruction Reference lists a bunch of bits that
5713          * are prerequisite to running VMXON, most notably cr4.VMXE must be
5714          * set to 1 (see vmx_set_cr4() for when we allow the guest to set this).
5715          * Otherwise, we should fail with #UD. We test these now:
5716          */
5717         if (!kvm_read_cr4_bits(vcpu, X86_CR4_VMXE) ||
5718             !kvm_read_cr0_bits(vcpu, X86_CR0_PE) ||
5719             (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) {
5720                 kvm_queue_exception(vcpu, UD_VECTOR);
5721                 return 1;
5722         }
5723 
5724         vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
5725         if (is_long_mode(vcpu) && !cs.l) {
5726                 kvm_queue_exception(vcpu, UD_VECTOR);
5727                 return 1;
5728         }
5729 
5730         if (vmx_get_cpl(vcpu)) {
5731                 kvm_inject_gp(vcpu, 0);
5732                 return 1;
5733         }
5734         if (vmx->nested.vmxon) {
5735                 nested_vmx_failValid(vcpu, VMXERR_VMXON_IN_VMX_ROOT_OPERATION);
5736                 skip_emulated_instruction(vcpu);
5737                 return 1;
5738         }
5739 
5740         if ((vmx->nested.msr_ia32_feature_control & VMXON_NEEDED_FEATURES)
5741                         != VMXON_NEEDED_FEATURES) {
5742                 kvm_inject_gp(vcpu, 0);
5743                 return 1;
5744         }
5745 
5746         if (enable_shadow_vmcs) {
5747                 shadow_vmcs = alloc_vmcs();
5748                 if (!shadow_vmcs)
5749                         return -ENOMEM;
5750                 /* mark vmcs as shadow */
5751                 shadow_vmcs->revision_id |= (1u << 31);
5752                 /* init shadow vmcs */
5753                 vmcs_clear(shadow_vmcs);
5754                 vmx->nested.current_shadow_vmcs = shadow_vmcs;
5755         }
5756 
5757         INIT_LIST_HEAD(&(vmx->nested.vmcs02_pool));
5758         vmx->nested.vmcs02_num = 0;
5759 
5760         vmx->nested.vmxon = true;
5761 
5762         skip_emulated_instruction(vcpu);
5763         nested_vmx_succeed(vcpu);
5764         return 1;
5765 }
5766 
5767 /*
5768  * Intel's VMX Instruction Reference specifies a common set of prerequisites
5769  * for running VMX instructions (except VMXON, whose prerequisites are
5770  * slightly different). It also specifies what exception to inject otherwise.
5771  */
5772 static int nested_vmx_check_permission(struct kvm_vcpu *vcpu)
5773 {
5774         struct kvm_segment cs;
5775         struct vcpu_vmx *vmx = to_vmx(vcpu);
5776 
5777         if (!vmx->nested.vmxon) {
5778                 kvm_queue_exception(vcpu, UD_VECTOR);
5779                 return 0;
5780         }
5781 
5782         vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
5783         if ((vmx_get_rflags(vcpu) & X86_EFLAGS_VM) ||
5784             (is_long_mode(vcpu) && !cs.l)) {
5785                 kvm_queue_exception(vcpu, UD_VECTOR);
5786                 return 0;
5787         }
5788 
5789         if (vmx_get_cpl(vcpu)) {
5790                 kvm_inject_gp(vcpu, 0);
5791                 return 0;
5792         }
5793 
5794         return 1;
5795 }
5796 
5797 static inline void nested_release_vmcs12(struct vcpu_vmx *vmx)
5798 {
5799         u32 exec_control;
5800         if (enable_shadow_vmcs) {
5801                 if (vmx->nested.current_vmcs12 != NULL) {
5802                         /* copy to memory all shadowed fields in case
5803                            they were modified */
5804                         copy_shadow_to_vmcs12(vmx);
5805                         vmx->nested.sync_shadow_vmcs = false;
5806                         exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
5807                         exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS;
5808                         vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);
5809                         vmcs_write64(VMCS_LINK_POINTER, -1ull);
5810                 }
5811         }
5812         kunmap(vmx->nested.current_vmcs12_page);
5813         nested_release_page(vmx->nested.current_vmcs12_page);
5814 }
5815 
5816 /*
5817  * Free whatever needs to be freed from vmx->nested when L1 goes down, or
5818  * just stops using VMX.
5819  */
5820 static void free_nested(struct vcpu_vmx *vmx)
5821 {
5822         if (!vmx->nested.vmxon)
5823                 return;
5824         vmx->nested.vmxon = false;
5825         if (vmx->nested.current_vmptr != -1ull) {
5826                 nested_release_vmcs12(vmx);
5827                 vmx->nested.current_vmptr = -1ull;
5828                 vmx->nested.current_vmcs12 = NULL;
5829         }
5830         if (enable_shadow_vmcs)
5831                 free_vmcs(vmx->nested.current_shadow_vmcs);
5832         /* Unpin physical memory we referred to in current vmcs02 */
5833         if (vmx->nested.apic_access_page) {
5834                 nested_release_page(vmx->nested.apic_access_page);
5835                 vmx->nested.apic_access_page = 0;
5836         }
5837 
5838         nested_free_all_saved_vmcss(vmx);
5839 }
5840 
5841 /* Emulate the VMXOFF instruction */
5842 static int handle_vmoff(struct kvm_vcpu *vcpu)
5843 {
5844         if (!nested_vmx_check_permission(vcpu))
5845                 return 1;
5846         free_nested(to_vmx(vcpu));
5847         skip_emulated_instruction(vcpu);
5848         nested_vmx_succeed(vcpu);
5849         return 1;
5850 }
5851 
5852 /*
5853  * Decode the memory-address operand of a vmx instruction, as recorded on an
5854  * exit caused by such an instruction (run by a guest hypervisor).
5855  * On success, returns 0. When the operand is invalid, returns 1 and throws
5856  * #UD or #GP.
5857  */
5858 static int get_vmx_mem_address(struct kvm_vcpu *vcpu,
5859                                  unsigned long exit_qualification,
5860                                  u32 vmx_instruction_info, gva_t *ret)
5861 {
5862         /*
5863          * According to Vol. 3B, "Information for VM Exits Due to Instruction
5864          * Execution", on an exit, vmx_instruction_info holds most of the
5865          * addressing components of the operand. Only the displacement part
5866          * is put in exit_qualification (see 3B, "Basic VM-Exit Information").
5867          * For how an actual address is calculated from all these components,
5868          * refer to Vol. 1, "Operand Addressing".
5869          */
5870         int  scaling = vmx_instruction_info & 3;
5871         int  addr_size = (vmx_instruction_info >> 7) & 7;
5872         bool is_reg = vmx_instruction_info & (1u << 10);
5873         int  seg_reg = (vmx_instruction_info >> 15) & 7;
5874         int  index_reg = (vmx_instruction_info >> 18) & 0xf;
5875         bool index_is_valid = !(vmx_instruction_info & (1u << 22));
5876         int  base_reg       = (vmx_instruction_info >> 23) & 0xf;
5877         bool base_is_valid  = !(vmx_instruction_info & (1u << 27));
5878 
5879         if (is_reg) {
5880                 kvm_queue_exception(vcpu, UD_VECTOR);
5881                 return 1;
5882         }
5883 
5884         /* Addr = segment_base + offset */
5885         /* offset = base + [index * scale] + displacement */
5886         *ret = vmx_get_segment_base(vcpu, seg_reg);
5887         if (base_is_valid)
5888                 *ret += kvm_register_read(vcpu, base_reg);
5889         if (index_is_valid)
5890                 *ret += kvm_register_read(vcpu, index_reg)<<scaling;
5891         *ret += exit_qualification; /* holds the displacement */
5892 
5893         if (addr_size == 1) /* 32 bit */
5894                 *ret &= 0xffffffff;
5895 
5896         /*
5897          * TODO: throw #GP (and return 1) in various cases that the VM*
5898          * instructions require it - e.g., offset beyond segment limit,
5899          * unusable or unreadable/unwritable segment, non-canonical 64-bit
5900          * address, and so on. Currently these are not checked.
5901          */
5902         return 0;
5903 }
5904 
5905 /* Emulate the VMCLEAR instruction */
5906 static int handle_vmclear(struct kvm_vcpu *vcpu)
5907 {
5908         struct vcpu_vmx *vmx = to_vmx(vcpu);
5909         gva_t gva;
5910         gpa_t vmptr;
5911         struct vmcs12 *vmcs12;
5912         struct page *page;