~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/virt/kvm/kvm_main.c

Version: ~ [ linux-5.8 ] ~ [ linux-5.7.14 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.57 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.138 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.193 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.232 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.232 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-only
  2 /*
  3  * Kernel-based Virtual Machine driver for Linux
  4  *
  5  * This module enables machines with Intel VT-x extensions to run virtual
  6  * machines without emulation or binary translation.
  7  *
  8  * Copyright (C) 2006 Qumranet, Inc.
  9  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
 10  *
 11  * Authors:
 12  *   Avi Kivity   <avi@qumranet.com>
 13  *   Yaniv Kamay  <yaniv@qumranet.com>
 14  */
 15 
 16 #include <kvm/iodev.h>
 17 
 18 #include <linux/kvm_host.h>
 19 #include <linux/kvm.h>
 20 #include <linux/module.h>
 21 #include <linux/errno.h>
 22 #include <linux/percpu.h>
 23 #include <linux/mm.h>
 24 #include <linux/miscdevice.h>
 25 #include <linux/vmalloc.h>
 26 #include <linux/reboot.h>
 27 #include <linux/debugfs.h>
 28 #include <linux/highmem.h>
 29 #include <linux/file.h>
 30 #include <linux/syscore_ops.h>
 31 #include <linux/cpu.h>
 32 #include <linux/sched/signal.h>
 33 #include <linux/sched/mm.h>
 34 #include <linux/sched/stat.h>
 35 #include <linux/cpumask.h>
 36 #include <linux/smp.h>
 37 #include <linux/anon_inodes.h>
 38 #include <linux/profile.h>
 39 #include <linux/kvm_para.h>
 40 #include <linux/pagemap.h>
 41 #include <linux/mman.h>
 42 #include <linux/swap.h>
 43 #include <linux/bitops.h>
 44 #include <linux/spinlock.h>
 45 #include <linux/compat.h>
 46 #include <linux/srcu.h>
 47 #include <linux/hugetlb.h>
 48 #include <linux/slab.h>
 49 #include <linux/sort.h>
 50 #include <linux/bsearch.h>
 51 #include <linux/io.h>
 52 #include <linux/lockdep.h>
 53 
 54 #include <asm/processor.h>
 55 #include <asm/ioctl.h>
 56 #include <linux/uaccess.h>
 57 #include <asm/pgtable.h>
 58 
 59 #include "coalesced_mmio.h"
 60 #include "async_pf.h"
 61 #include "vfio.h"
 62 
 63 #define CREATE_TRACE_POINTS
 64 #include <trace/events/kvm.h>
 65 
 66 /* Worst case buffer size needed for holding an integer. */
 67 #define ITOA_MAX_LEN 12
 68 
 69 MODULE_AUTHOR("Qumranet");
 70 MODULE_LICENSE("GPL");
 71 
 72 /* Architectures should define their poll value according to the halt latency */
 73 unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT;
 74 module_param(halt_poll_ns, uint, 0644);
 75 EXPORT_SYMBOL_GPL(halt_poll_ns);
 76 
 77 /* Default doubles per-vcpu halt_poll_ns. */
 78 unsigned int halt_poll_ns_grow = 2;
 79 module_param(halt_poll_ns_grow, uint, 0644);
 80 EXPORT_SYMBOL_GPL(halt_poll_ns_grow);
 81 
 82 /* The start value to grow halt_poll_ns from */
 83 unsigned int halt_poll_ns_grow_start = 10000; /* 10us */
 84 module_param(halt_poll_ns_grow_start, uint, 0644);
 85 EXPORT_SYMBOL_GPL(halt_poll_ns_grow_start);
 86 
 87 /* Default resets per-vcpu halt_poll_ns . */
 88 unsigned int halt_poll_ns_shrink;
 89 module_param(halt_poll_ns_shrink, uint, 0644);
 90 EXPORT_SYMBOL_GPL(halt_poll_ns_shrink);
 91 
 92 /*
 93  * Ordering of locks:
 94  *
 95  *      kvm->lock --> kvm->slots_lock --> kvm->irq_lock
 96  */
 97 
 98 DEFINE_SPINLOCK(kvm_lock);
 99 static DEFINE_RAW_SPINLOCK(kvm_count_lock);
100 LIST_HEAD(vm_list);
101 
102 static cpumask_var_t cpus_hardware_enabled;
103 static int kvm_usage_count;
104 static atomic_t hardware_enable_failed;
105 
106 struct kmem_cache *kvm_vcpu_cache;
107 EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
108 
109 static __read_mostly struct preempt_ops kvm_preempt_ops;
110 
111 struct dentry *kvm_debugfs_dir;
112 EXPORT_SYMBOL_GPL(kvm_debugfs_dir);
113 
114 static int kvm_debugfs_num_entries;
115 static const struct file_operations *stat_fops_per_vm[];
116 
117 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
118                            unsigned long arg);
119 #ifdef CONFIG_KVM_COMPAT
120 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
121                                   unsigned long arg);
122 #define KVM_COMPAT(c)   .compat_ioctl   = (c)
123 #else
124 static long kvm_no_compat_ioctl(struct file *file, unsigned int ioctl,
125                                 unsigned long arg) { return -EINVAL; }
126 #define KVM_COMPAT(c)   .compat_ioctl   = kvm_no_compat_ioctl
127 #endif
128 static int hardware_enable_all(void);
129 static void hardware_disable_all(void);
130 
131 static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
132 
133 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn);
134 
135 __visible bool kvm_rebooting;
136 EXPORT_SYMBOL_GPL(kvm_rebooting);
137 
138 static bool largepages_enabled = true;
139 
140 #define KVM_EVENT_CREATE_VM 0
141 #define KVM_EVENT_DESTROY_VM 1
142 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm);
143 static unsigned long long kvm_createvm_count;
144 static unsigned long long kvm_active_vms;
145 
146 __weak int kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
147                 unsigned long start, unsigned long end, bool blockable)
148 {
149         return 0;
150 }
151 
152 bool kvm_is_reserved_pfn(kvm_pfn_t pfn)
153 {
154         if (pfn_valid(pfn))
155                 return PageReserved(pfn_to_page(pfn));
156 
157         return true;
158 }
159 
160 /*
161  * Switches to specified vcpu, until a matching vcpu_put()
162  */
163 void vcpu_load(struct kvm_vcpu *vcpu)
164 {
165         int cpu = get_cpu();
166         preempt_notifier_register(&vcpu->preempt_notifier);
167         kvm_arch_vcpu_load(vcpu, cpu);
168         put_cpu();
169 }
170 EXPORT_SYMBOL_GPL(vcpu_load);
171 
172 void vcpu_put(struct kvm_vcpu *vcpu)
173 {
174         preempt_disable();
175         kvm_arch_vcpu_put(vcpu);
176         preempt_notifier_unregister(&vcpu->preempt_notifier);
177         preempt_enable();
178 }
179 EXPORT_SYMBOL_GPL(vcpu_put);
180 
181 /* TODO: merge with kvm_arch_vcpu_should_kick */
182 static bool kvm_request_needs_ipi(struct kvm_vcpu *vcpu, unsigned req)
183 {
184         int mode = kvm_vcpu_exiting_guest_mode(vcpu);
185 
186         /*
187          * We need to wait for the VCPU to reenable interrupts and get out of
188          * READING_SHADOW_PAGE_TABLES mode.
189          */
190         if (req & KVM_REQUEST_WAIT)
191                 return mode != OUTSIDE_GUEST_MODE;
192 
193         /*
194          * Need to kick a running VCPU, but otherwise there is nothing to do.
195          */
196         return mode == IN_GUEST_MODE;
197 }
198 
199 static void ack_flush(void *_completed)
200 {
201 }
202 
203 static inline bool kvm_kick_many_cpus(const struct cpumask *cpus, bool wait)
204 {
205         if (unlikely(!cpus))
206                 cpus = cpu_online_mask;
207 
208         if (cpumask_empty(cpus))
209                 return false;
210 
211         smp_call_function_many(cpus, ack_flush, NULL, wait);
212         return true;
213 }
214 
215 bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req,
216                                  unsigned long *vcpu_bitmap, cpumask_var_t tmp)
217 {
218         int i, cpu, me;
219         struct kvm_vcpu *vcpu;
220         bool called;
221 
222         me = get_cpu();
223 
224         kvm_for_each_vcpu(i, vcpu, kvm) {
225                 if (vcpu_bitmap && !test_bit(i, vcpu_bitmap))
226                         continue;
227 
228                 kvm_make_request(req, vcpu);
229                 cpu = vcpu->cpu;
230 
231                 if (!(req & KVM_REQUEST_NO_WAKEUP) && kvm_vcpu_wake_up(vcpu))
232                         continue;
233 
234                 if (tmp != NULL && cpu != -1 && cpu != me &&
235                     kvm_request_needs_ipi(vcpu, req))
236                         __cpumask_set_cpu(cpu, tmp);
237         }
238 
239         called = kvm_kick_many_cpus(tmp, !!(req & KVM_REQUEST_WAIT));
240         put_cpu();
241 
242         return called;
243 }
244 
245 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
246 {
247         cpumask_var_t cpus;
248         bool called;
249 
250         zalloc_cpumask_var(&cpus, GFP_ATOMIC);
251 
252         called = kvm_make_vcpus_request_mask(kvm, req, NULL, cpus);
253 
254         free_cpumask_var(cpus);
255         return called;
256 }
257 
258 #ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
259 void kvm_flush_remote_tlbs(struct kvm *kvm)
260 {
261         /*
262          * Read tlbs_dirty before setting KVM_REQ_TLB_FLUSH in
263          * kvm_make_all_cpus_request.
264          */
265         long dirty_count = smp_load_acquire(&kvm->tlbs_dirty);
266 
267         /*
268          * We want to publish modifications to the page tables before reading
269          * mode. Pairs with a memory barrier in arch-specific code.
270          * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest
271          * and smp_mb in walk_shadow_page_lockless_begin/end.
272          * - powerpc: smp_mb in kvmppc_prepare_to_enter.
273          *
274          * There is already an smp_mb__after_atomic() before
275          * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that
276          * barrier here.
277          */
278         if (!kvm_arch_flush_remote_tlb(kvm)
279             || kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
280                 ++kvm->stat.remote_tlb_flush;
281         cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
282 }
283 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
284 #endif
285 
286 void kvm_reload_remote_mmus(struct kvm *kvm)
287 {
288         kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
289 }
290 
291 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
292 {
293         struct page *page;
294         int r;
295 
296         mutex_init(&vcpu->mutex);
297         vcpu->cpu = -1;
298         vcpu->kvm = kvm;
299         vcpu->vcpu_id = id;
300         vcpu->pid = NULL;
301         init_swait_queue_head(&vcpu->wq);
302         kvm_async_pf_vcpu_init(vcpu);
303 
304         vcpu->pre_pcpu = -1;
305         INIT_LIST_HEAD(&vcpu->blocked_vcpu_list);
306 
307         page = alloc_page(GFP_KERNEL | __GFP_ZERO);
308         if (!page) {
309                 r = -ENOMEM;
310                 goto fail;
311         }
312         vcpu->run = page_address(page);
313 
314         kvm_vcpu_set_in_spin_loop(vcpu, false);
315         kvm_vcpu_set_dy_eligible(vcpu, false);
316         vcpu->preempted = false;
317 
318         r = kvm_arch_vcpu_init(vcpu);
319         if (r < 0)
320                 goto fail_free_run;
321         return 0;
322 
323 fail_free_run:
324         free_page((unsigned long)vcpu->run);
325 fail:
326         return r;
327 }
328 EXPORT_SYMBOL_GPL(kvm_vcpu_init);
329 
330 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
331 {
332         /*
333          * no need for rcu_read_lock as VCPU_RUN is the only place that
334          * will change the vcpu->pid pointer and on uninit all file
335          * descriptors are already gone.
336          */
337         put_pid(rcu_dereference_protected(vcpu->pid, 1));
338         kvm_arch_vcpu_uninit(vcpu);
339         free_page((unsigned long)vcpu->run);
340 }
341 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
342 
343 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
344 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
345 {
346         return container_of(mn, struct kvm, mmu_notifier);
347 }
348 
349 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
350                                         struct mm_struct *mm,
351                                         unsigned long address,
352                                         pte_t pte)
353 {
354         struct kvm *kvm = mmu_notifier_to_kvm(mn);
355         int idx;
356 
357         idx = srcu_read_lock(&kvm->srcu);
358         spin_lock(&kvm->mmu_lock);
359         kvm->mmu_notifier_seq++;
360 
361         if (kvm_set_spte_hva(kvm, address, pte))
362                 kvm_flush_remote_tlbs(kvm);
363 
364         spin_unlock(&kvm->mmu_lock);
365         srcu_read_unlock(&kvm->srcu, idx);
366 }
367 
368 static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
369                                         const struct mmu_notifier_range *range)
370 {
371         struct kvm *kvm = mmu_notifier_to_kvm(mn);
372         int need_tlb_flush = 0, idx;
373         int ret;
374 
375         idx = srcu_read_lock(&kvm->srcu);
376         spin_lock(&kvm->mmu_lock);
377         /*
378          * The count increase must become visible at unlock time as no
379          * spte can be established without taking the mmu_lock and
380          * count is also read inside the mmu_lock critical section.
381          */
382         kvm->mmu_notifier_count++;
383         need_tlb_flush = kvm_unmap_hva_range(kvm, range->start, range->end);
384         need_tlb_flush |= kvm->tlbs_dirty;
385         /* we've to flush the tlb before the pages can be freed */
386         if (need_tlb_flush)
387                 kvm_flush_remote_tlbs(kvm);
388 
389         spin_unlock(&kvm->mmu_lock);
390 
391         ret = kvm_arch_mmu_notifier_invalidate_range(kvm, range->start,
392                                         range->end,
393                                         mmu_notifier_range_blockable(range));
394 
395         srcu_read_unlock(&kvm->srcu, idx);
396 
397         return ret;
398 }
399 
400 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
401                                         const struct mmu_notifier_range *range)
402 {
403         struct kvm *kvm = mmu_notifier_to_kvm(mn);
404 
405         spin_lock(&kvm->mmu_lock);
406         /*
407          * This sequence increase will notify the kvm page fault that
408          * the page that is going to be mapped in the spte could have
409          * been freed.
410          */
411         kvm->mmu_notifier_seq++;
412         smp_wmb();
413         /*
414          * The above sequence increase must be visible before the
415          * below count decrease, which is ensured by the smp_wmb above
416          * in conjunction with the smp_rmb in mmu_notifier_retry().
417          */
418         kvm->mmu_notifier_count--;
419         spin_unlock(&kvm->mmu_lock);
420 
421         BUG_ON(kvm->mmu_notifier_count < 0);
422 }
423 
424 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
425                                               struct mm_struct *mm,
426                                               unsigned long start,
427                                               unsigned long end)
428 {
429         struct kvm *kvm = mmu_notifier_to_kvm(mn);
430         int young, idx;
431 
432         idx = srcu_read_lock(&kvm->srcu);
433         spin_lock(&kvm->mmu_lock);
434 
435         young = kvm_age_hva(kvm, start, end);
436         if (young)
437                 kvm_flush_remote_tlbs(kvm);
438 
439         spin_unlock(&kvm->mmu_lock);
440         srcu_read_unlock(&kvm->srcu, idx);
441 
442         return young;
443 }
444 
445 static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn,
446                                         struct mm_struct *mm,
447                                         unsigned long start,
448                                         unsigned long end)
449 {
450         struct kvm *kvm = mmu_notifier_to_kvm(mn);
451         int young, idx;
452 
453         idx = srcu_read_lock(&kvm->srcu);
454         spin_lock(&kvm->mmu_lock);
455         /*
456          * Even though we do not flush TLB, this will still adversely
457          * affect performance on pre-Haswell Intel EPT, where there is
458          * no EPT Access Bit to clear so that we have to tear down EPT
459          * tables instead. If we find this unacceptable, we can always
460          * add a parameter to kvm_age_hva so that it effectively doesn't
461          * do anything on clear_young.
462          *
463          * Also note that currently we never issue secondary TLB flushes
464          * from clear_young, leaving this job up to the regular system
465          * cadence. If we find this inaccurate, we might come up with a
466          * more sophisticated heuristic later.
467          */
468         young = kvm_age_hva(kvm, start, end);
469         spin_unlock(&kvm->mmu_lock);
470         srcu_read_unlock(&kvm->srcu, idx);
471 
472         return young;
473 }
474 
475 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
476                                        struct mm_struct *mm,
477                                        unsigned long address)
478 {
479         struct kvm *kvm = mmu_notifier_to_kvm(mn);
480         int young, idx;
481 
482         idx = srcu_read_lock(&kvm->srcu);
483         spin_lock(&kvm->mmu_lock);
484         young = kvm_test_age_hva(kvm, address);
485         spin_unlock(&kvm->mmu_lock);
486         srcu_read_unlock(&kvm->srcu, idx);
487 
488         return young;
489 }
490 
491 static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
492                                      struct mm_struct *mm)
493 {
494         struct kvm *kvm = mmu_notifier_to_kvm(mn);
495         int idx;
496 
497         idx = srcu_read_lock(&kvm->srcu);
498         kvm_arch_flush_shadow_all(kvm);
499         srcu_read_unlock(&kvm->srcu, idx);
500 }
501 
502 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
503         .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
504         .invalidate_range_end   = kvm_mmu_notifier_invalidate_range_end,
505         .clear_flush_young      = kvm_mmu_notifier_clear_flush_young,
506         .clear_young            = kvm_mmu_notifier_clear_young,
507         .test_young             = kvm_mmu_notifier_test_young,
508         .change_pte             = kvm_mmu_notifier_change_pte,
509         .release                = kvm_mmu_notifier_release,
510 };
511 
512 static int kvm_init_mmu_notifier(struct kvm *kvm)
513 {
514         kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
515         return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
516 }
517 
518 #else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
519 
520 static int kvm_init_mmu_notifier(struct kvm *kvm)
521 {
522         return 0;
523 }
524 
525 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
526 
527 static struct kvm_memslots *kvm_alloc_memslots(void)
528 {
529         int i;
530         struct kvm_memslots *slots;
531 
532         slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL_ACCOUNT);
533         if (!slots)
534                 return NULL;
535 
536         for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
537                 slots->id_to_index[i] = slots->memslots[i].id = i;
538 
539         return slots;
540 }
541 
542 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
543 {
544         if (!memslot->dirty_bitmap)
545                 return;
546 
547         kvfree(memslot->dirty_bitmap);
548         memslot->dirty_bitmap = NULL;
549 }
550 
551 /*
552  * Free any memory in @free but not in @dont.
553  */
554 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
555                               struct kvm_memory_slot *dont)
556 {
557         if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
558                 kvm_destroy_dirty_bitmap(free);
559 
560         kvm_arch_free_memslot(kvm, free, dont);
561 
562         free->npages = 0;
563 }
564 
565 static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots)
566 {
567         struct kvm_memory_slot *memslot;
568 
569         if (!slots)
570                 return;
571 
572         kvm_for_each_memslot(memslot, slots)
573                 kvm_free_memslot(kvm, memslot, NULL);
574 
575         kvfree(slots);
576 }
577 
578 static void kvm_destroy_vm_debugfs(struct kvm *kvm)
579 {
580         int i;
581 
582         if (!kvm->debugfs_dentry)
583                 return;
584 
585         debugfs_remove_recursive(kvm->debugfs_dentry);
586 
587         if (kvm->debugfs_stat_data) {
588                 for (i = 0; i < kvm_debugfs_num_entries; i++)
589                         kfree(kvm->debugfs_stat_data[i]);
590                 kfree(kvm->debugfs_stat_data);
591         }
592 }
593 
594 static int kvm_create_vm_debugfs(struct kvm *kvm, int fd)
595 {
596         char dir_name[ITOA_MAX_LEN * 2];
597         struct kvm_stat_data *stat_data;
598         struct kvm_stats_debugfs_item *p;
599 
600         if (!debugfs_initialized())
601                 return 0;
602 
603         snprintf(dir_name, sizeof(dir_name), "%d-%d", task_pid_nr(current), fd);
604         kvm->debugfs_dentry = debugfs_create_dir(dir_name, kvm_debugfs_dir);
605 
606         kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries,
607                                          sizeof(*kvm->debugfs_stat_data),
608                                          GFP_KERNEL_ACCOUNT);
609         if (!kvm->debugfs_stat_data)
610                 return -ENOMEM;
611 
612         for (p = debugfs_entries; p->name; p++) {
613                 stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL_ACCOUNT);
614                 if (!stat_data)
615                         return -ENOMEM;
616 
617                 stat_data->kvm = kvm;
618                 stat_data->offset = p->offset;
619                 kvm->debugfs_stat_data[p - debugfs_entries] = stat_data;
620                 debugfs_create_file(p->name, 0644, kvm->debugfs_dentry,
621                                     stat_data, stat_fops_per_vm[p->kind]);
622         }
623         return 0;
624 }
625 
626 static struct kvm *kvm_create_vm(unsigned long type)
627 {
628         int r, i;
629         struct kvm *kvm = kvm_arch_alloc_vm();
630 
631         if (!kvm)
632                 return ERR_PTR(-ENOMEM);
633 
634         spin_lock_init(&kvm->mmu_lock);
635         mmgrab(current->mm);
636         kvm->mm = current->mm;
637         kvm_eventfd_init(kvm);
638         mutex_init(&kvm->lock);
639         mutex_init(&kvm->irq_lock);
640         mutex_init(&kvm->slots_lock);
641         refcount_set(&kvm->users_count, 1);
642         INIT_LIST_HEAD(&kvm->devices);
643 
644         r = kvm_arch_init_vm(kvm, type);
645         if (r)
646                 goto out_err_no_disable;
647 
648         r = hardware_enable_all();
649         if (r)
650                 goto out_err_no_disable;
651 
652 #ifdef CONFIG_HAVE_KVM_IRQFD
653         INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
654 #endif
655 
656         BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);
657 
658         r = -ENOMEM;
659         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
660                 struct kvm_memslots *slots = kvm_alloc_memslots();
661                 if (!slots)
662                         goto out_err_no_srcu;
663                 /* Generations must be different for each address space. */
664                 slots->generation = i;
665                 rcu_assign_pointer(kvm->memslots[i], slots);
666         }
667 
668         if (init_srcu_struct(&kvm->srcu))
669                 goto out_err_no_srcu;
670         if (init_srcu_struct(&kvm->irq_srcu))
671                 goto out_err_no_irq_srcu;
672         for (i = 0; i < KVM_NR_BUSES; i++) {
673                 rcu_assign_pointer(kvm->buses[i],
674                         kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL_ACCOUNT));
675                 if (!kvm->buses[i])
676                         goto out_err;
677         }
678 
679         r = kvm_init_mmu_notifier(kvm);
680         if (r)
681                 goto out_err;
682 
683         spin_lock(&kvm_lock);
684         list_add(&kvm->vm_list, &vm_list);
685         spin_unlock(&kvm_lock);
686 
687         preempt_notifier_inc();
688 
689         return kvm;
690 
691 out_err:
692         cleanup_srcu_struct(&kvm->irq_srcu);
693 out_err_no_irq_srcu:
694         cleanup_srcu_struct(&kvm->srcu);
695 out_err_no_srcu:
696         hardware_disable_all();
697 out_err_no_disable:
698         refcount_set(&kvm->users_count, 0);
699         for (i = 0; i < KVM_NR_BUSES; i++)
700                 kfree(kvm_get_bus(kvm, i));
701         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
702                 kvm_free_memslots(kvm, __kvm_memslots(kvm, i));
703         kvm_arch_free_vm(kvm);
704         mmdrop(current->mm);
705         return ERR_PTR(r);
706 }
707 
708 static void kvm_destroy_devices(struct kvm *kvm)
709 {
710         struct kvm_device *dev, *tmp;
711 
712         /*
713          * We do not need to take the kvm->lock here, because nobody else
714          * has a reference to the struct kvm at this point and therefore
715          * cannot access the devices list anyhow.
716          */
717         list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) {
718                 list_del(&dev->vm_node);
719                 dev->ops->destroy(dev);
720         }
721 }
722 
723 static void kvm_destroy_vm(struct kvm *kvm)
724 {
725         int i;
726         struct mm_struct *mm = kvm->mm;
727 
728         kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm);
729         kvm_destroy_vm_debugfs(kvm);
730         kvm_arch_sync_events(kvm);
731         spin_lock(&kvm_lock);
732         list_del(&kvm->vm_list);
733         spin_unlock(&kvm_lock);
734         kvm_free_irq_routing(kvm);
735         for (i = 0; i < KVM_NR_BUSES; i++) {
736                 struct kvm_io_bus *bus = kvm_get_bus(kvm, i);
737 
738                 if (bus)
739                         kvm_io_bus_destroy(bus);
740                 kvm->buses[i] = NULL;
741         }
742         kvm_coalesced_mmio_free(kvm);
743 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
744         mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
745 #else
746         kvm_arch_flush_shadow_all(kvm);
747 #endif
748         kvm_arch_destroy_vm(kvm);
749         kvm_destroy_devices(kvm);
750         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
751                 kvm_free_memslots(kvm, __kvm_memslots(kvm, i));
752         cleanup_srcu_struct(&kvm->irq_srcu);
753         cleanup_srcu_struct(&kvm->srcu);
754         kvm_arch_free_vm(kvm);
755         preempt_notifier_dec();
756         hardware_disable_all();
757         mmdrop(mm);
758 }
759 
760 void kvm_get_kvm(struct kvm *kvm)
761 {
762         refcount_inc(&kvm->users_count);
763 }
764 EXPORT_SYMBOL_GPL(kvm_get_kvm);
765 
766 void kvm_put_kvm(struct kvm *kvm)
767 {
768         if (refcount_dec_and_test(&kvm->users_count))
769                 kvm_destroy_vm(kvm);
770 }
771 EXPORT_SYMBOL_GPL(kvm_put_kvm);
772 
773 
774 static int kvm_vm_release(struct inode *inode, struct file *filp)
775 {
776         struct kvm *kvm = filp->private_data;
777 
778         kvm_irqfd_release(kvm);
779 
780         kvm_put_kvm(kvm);
781         return 0;
782 }
783 
784 /*
785  * Allocation size is twice as large as the actual dirty bitmap size.
786  * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
787  */
788 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
789 {
790         unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
791 
792         memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL_ACCOUNT);
793         if (!memslot->dirty_bitmap)
794                 return -ENOMEM;
795 
796         return 0;
797 }
798 
799 /*
800  * Insert memslot and re-sort memslots based on their GFN,
801  * so binary search could be used to lookup GFN.
802  * Sorting algorithm takes advantage of having initially
803  * sorted array and known changed memslot position.
804  */
805 static void update_memslots(struct kvm_memslots *slots,
806                             struct kvm_memory_slot *new,
807                             enum kvm_mr_change change)
808 {
809         int id = new->id;
810         int i = slots->id_to_index[id];
811         struct kvm_memory_slot *mslots = slots->memslots;
812 
813         WARN_ON(mslots[i].id != id);
814         switch (change) {
815         case KVM_MR_CREATE:
816                 slots->used_slots++;
817                 WARN_ON(mslots[i].npages || !new->npages);
818                 break;
819         case KVM_MR_DELETE:
820                 slots->used_slots--;
821                 WARN_ON(new->npages || !mslots[i].npages);
822                 break;
823         default:
824                 break;
825         }
826 
827         while (i < KVM_MEM_SLOTS_NUM - 1 &&
828                new->base_gfn <= mslots[i + 1].base_gfn) {
829                 if (!mslots[i + 1].npages)
830                         break;
831                 mslots[i] = mslots[i + 1];
832                 slots->id_to_index[mslots[i].id] = i;
833                 i++;
834         }
835 
836         /*
837          * The ">=" is needed when creating a slot with base_gfn == 0,
838          * so that it moves before all those with base_gfn == npages == 0.
839          *
840          * On the other hand, if new->npages is zero, the above loop has
841          * already left i pointing to the beginning of the empty part of
842          * mslots, and the ">=" would move the hole backwards in this
843          * case---which is wrong.  So skip the loop when deleting a slot.
844          */
845         if (new->npages) {
846                 while (i > 0 &&
847                        new->base_gfn >= mslots[i - 1].base_gfn) {
848                         mslots[i] = mslots[i - 1];
849                         slots->id_to_index[mslots[i].id] = i;
850                         i--;
851                 }
852         } else
853                 WARN_ON_ONCE(i != slots->used_slots);
854 
855         mslots[i] = *new;
856         slots->id_to_index[mslots[i].id] = i;
857 }
858 
859 static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem)
860 {
861         u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
862 
863 #ifdef __KVM_HAVE_READONLY_MEM
864         valid_flags |= KVM_MEM_READONLY;
865 #endif
866 
867         if (mem->flags & ~valid_flags)
868                 return -EINVAL;
869 
870         return 0;
871 }
872 
873 static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
874                 int as_id, struct kvm_memslots *slots)
875 {
876         struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id);
877         u64 gen = old_memslots->generation;
878 
879         WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);
880         slots->generation = gen | KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;
881 
882         rcu_assign_pointer(kvm->memslots[as_id], slots);
883         synchronize_srcu_expedited(&kvm->srcu);
884 
885         /*
886          * Increment the new memslot generation a second time, dropping the
887          * update in-progress flag and incrementing then generation based on
888          * the number of address spaces.  This provides a unique and easily
889          * identifiable generation number while the memslots are in flux.
890          */
891         gen = slots->generation & ~KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;
892 
893         /*
894          * Generations must be unique even across address spaces.  We do not need
895          * a global counter for that, instead the generation space is evenly split
896          * across address spaces.  For example, with two address spaces, address
897          * space 0 will use generations 0, 2, 4, ... while address space 1 will
898          * use generations 1, 3, 5, ...
899          */
900         gen += KVM_ADDRESS_SPACE_NUM;
901 
902         kvm_arch_memslots_updated(kvm, gen);
903 
904         slots->generation = gen;
905 
906         return old_memslots;
907 }
908 
909 /*
910  * Allocate some memory and give it an address in the guest physical address
911  * space.
912  *
913  * Discontiguous memory is allowed, mostly for framebuffers.
914  *
915  * Must be called holding kvm->slots_lock for write.
916  */
917 int __kvm_set_memory_region(struct kvm *kvm,
918                             const struct kvm_userspace_memory_region *mem)
919 {
920         int r;
921         gfn_t base_gfn;
922         unsigned long npages;
923         struct kvm_memory_slot *slot;
924         struct kvm_memory_slot old, new;
925         struct kvm_memslots *slots = NULL, *old_memslots;
926         int as_id, id;
927         enum kvm_mr_change change;
928 
929         r = check_memory_region_flags(mem);
930         if (r)
931                 goto out;
932 
933         r = -EINVAL;
934         as_id = mem->slot >> 16;
935         id = (u16)mem->slot;
936 
937         /* General sanity checks */
938         if (mem->memory_size & (PAGE_SIZE - 1))
939                 goto out;
940         if (mem->guest_phys_addr & (PAGE_SIZE - 1))
941                 goto out;
942         /* We can read the guest memory with __xxx_user() later on. */
943         if ((id < KVM_USER_MEM_SLOTS) &&
944             ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
945              !access_ok((void __user *)(unsigned long)mem->userspace_addr,
946                         mem->memory_size)))
947                 goto out;
948         if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM)
949                 goto out;
950         if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
951                 goto out;
952 
953         slot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
954         base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
955         npages = mem->memory_size >> PAGE_SHIFT;
956 
957         if (npages > KVM_MEM_MAX_NR_PAGES)
958                 goto out;
959 
960         new = old = *slot;
961 
962         new.id = id;
963         new.base_gfn = base_gfn;
964         new.npages = npages;
965         new.flags = mem->flags;
966 
967         if (npages) {
968                 if (!old.npages)
969                         change = KVM_MR_CREATE;
970                 else { /* Modify an existing slot. */
971                         if ((mem->userspace_addr != old.userspace_addr) ||
972                             (npages != old.npages) ||
973                             ((new.flags ^ old.flags) & KVM_MEM_READONLY))
974                                 goto out;
975 
976                         if (base_gfn != old.base_gfn)
977                                 change = KVM_MR_MOVE;
978                         else if (new.flags != old.flags)
979                                 change = KVM_MR_FLAGS_ONLY;
980                         else { /* Nothing to change. */
981                                 r = 0;
982                                 goto out;
983                         }
984                 }
985         } else {
986                 if (!old.npages)
987                         goto out;
988 
989                 change = KVM_MR_DELETE;
990                 new.base_gfn = 0;
991                 new.flags = 0;
992         }
993 
994         if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
995                 /* Check for overlaps */
996                 r = -EEXIST;
997                 kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) {
998                         if (slot->id == id)
999                                 continue;
1000                         if (!((base_gfn + npages <= slot->base_gfn) ||
1001                               (base_gfn >= slot->base_gfn + slot->npages)))
1002                                 goto out;
1003                 }
1004         }
1005 
1006         /* Free page dirty bitmap if unneeded */
1007         if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
1008                 new.dirty_bitmap = NULL;
1009 
1010         r = -ENOMEM;
1011         if (change == KVM_MR_CREATE) {
1012                 new.userspace_addr = mem->userspace_addr;
1013 
1014                 if (kvm_arch_create_memslot(kvm, &new, npages))
1015                         goto out_free;
1016         }
1017 
1018         /* Allocate page dirty bitmap if needed */
1019         if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
1020                 if (kvm_create_dirty_bitmap(&new) < 0)
1021                         goto out_free;
1022         }
1023 
1024         slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL_ACCOUNT);
1025         if (!slots)
1026                 goto out_free;
1027         memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots));
1028 
1029         if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
1030                 slot = id_to_memslot(slots, id);
1031                 slot->flags |= KVM_MEMSLOT_INVALID;
1032 
1033                 old_memslots = install_new_memslots(kvm, as_id, slots);
1034 
1035                 /* From this point no new shadow pages pointing to a deleted,
1036                  * or moved, memslot will be created.
1037                  *
1038                  * validation of sp->gfn happens in:
1039                  *      - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
1040                  *      - kvm_is_visible_gfn (mmu_check_roots)
1041                  */
1042                 kvm_arch_flush_shadow_memslot(kvm, slot);
1043 
1044                 /*
1045                  * We can re-use the old_memslots from above, the only difference
1046                  * from the currently installed memslots is the invalid flag.  This
1047                  * will get overwritten by update_memslots anyway.
1048                  */
1049                 slots = old_memslots;
1050         }
1051 
1052         r = kvm_arch_prepare_memory_region(kvm, &new, mem, change);
1053         if (r)
1054                 goto out_slots;
1055 
1056         /* actual memory is freed via old in kvm_free_memslot below */
1057         if (change == KVM_MR_DELETE) {
1058                 new.dirty_bitmap = NULL;
1059                 memset(&new.arch, 0, sizeof(new.arch));
1060         }
1061 
1062         update_memslots(slots, &new, change);
1063         old_memslots = install_new_memslots(kvm, as_id, slots);
1064 
1065         kvm_arch_commit_memory_region(kvm, mem, &old, &new, change);
1066 
1067         kvm_free_memslot(kvm, &old, &new);
1068         kvfree(old_memslots);
1069         return 0;
1070 
1071 out_slots:
1072         kvfree(slots);
1073 out_free:
1074         kvm_free_memslot(kvm, &new, &old);
1075 out:
1076         return r;
1077 }
1078 EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
1079 
1080 int kvm_set_memory_region(struct kvm *kvm,
1081                           const struct kvm_userspace_memory_region *mem)
1082 {
1083         int r;
1084 
1085         mutex_lock(&kvm->slots_lock);
1086         r = __kvm_set_memory_region(kvm, mem);
1087         mutex_unlock(&kvm->slots_lock);
1088         return r;
1089 }
1090 EXPORT_SYMBOL_GPL(kvm_set_memory_region);
1091 
1092 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
1093                                           struct kvm_userspace_memory_region *mem)
1094 {
1095         if ((u16)mem->slot >= KVM_USER_MEM_SLOTS)
1096                 return -EINVAL;
1097 
1098         return kvm_set_memory_region(kvm, mem);
1099 }
1100 
1101 int kvm_get_dirty_log(struct kvm *kvm,
1102                         struct kvm_dirty_log *log, int *is_dirty)
1103 {
1104         struct kvm_memslots *slots;
1105         struct kvm_memory_slot *memslot;
1106         int i, as_id, id;
1107         unsigned long n;
1108         unsigned long any = 0;
1109 
1110         as_id = log->slot >> 16;
1111         id = (u16)log->slot;
1112         if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1113                 return -EINVAL;
1114 
1115         slots = __kvm_memslots(kvm, as_id);
1116         memslot = id_to_memslot(slots, id);
1117         if (!memslot->dirty_bitmap)
1118                 return -ENOENT;
1119 
1120         n = kvm_dirty_bitmap_bytes(memslot);
1121 
1122         for (i = 0; !any && i < n/sizeof(long); ++i)
1123                 any = memslot->dirty_bitmap[i];
1124 
1125         if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
1126                 return -EFAULT;
1127 
1128         if (any)
1129                 *is_dirty = 1;
1130         return 0;
1131 }
1132 EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
1133 
1134 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
1135 /**
1136  * kvm_get_dirty_log_protect - get a snapshot of dirty pages
1137  *      and reenable dirty page tracking for the corresponding pages.
1138  * @kvm:        pointer to kvm instance
1139  * @log:        slot id and address to which we copy the log
1140  * @flush:      true if TLB flush is needed by caller
1141  *
1142  * We need to keep it in mind that VCPU threads can write to the bitmap
1143  * concurrently. So, to avoid losing track of dirty pages we keep the
1144  * following order:
1145  *
1146  *    1. Take a snapshot of the bit and clear it if needed.
1147  *    2. Write protect the corresponding page.
1148  *    3. Copy the snapshot to the userspace.
1149  *    4. Upon return caller flushes TLB's if needed.
1150  *
1151  * Between 2 and 4, the guest may write to the page using the remaining TLB
1152  * entry.  This is not a problem because the page is reported dirty using
1153  * the snapshot taken before and step 4 ensures that writes done after
1154  * exiting to userspace will be logged for the next call.
1155  *
1156  */
1157 int kvm_get_dirty_log_protect(struct kvm *kvm,
1158                         struct kvm_dirty_log *log, bool *flush)
1159 {
1160         struct kvm_memslots *slots;
1161         struct kvm_memory_slot *memslot;
1162         int i, as_id, id;
1163         unsigned long n;
1164         unsigned long *dirty_bitmap;
1165         unsigned long *dirty_bitmap_buffer;
1166 
1167         as_id = log->slot >> 16;
1168         id = (u16)log->slot;
1169         if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1170                 return -EINVAL;
1171 
1172         slots = __kvm_memslots(kvm, as_id);
1173         memslot = id_to_memslot(slots, id);
1174 
1175         dirty_bitmap = memslot->dirty_bitmap;
1176         if (!dirty_bitmap)
1177                 return -ENOENT;
1178 
1179         n = kvm_dirty_bitmap_bytes(memslot);
1180         *flush = false;
1181         if (kvm->manual_dirty_log_protect) {
1182                 /*
1183                  * Unlike kvm_get_dirty_log, we always return false in *flush,
1184                  * because no flush is needed until KVM_CLEAR_DIRTY_LOG.  There
1185                  * is some code duplication between this function and
1186                  * kvm_get_dirty_log, but hopefully all architecture
1187                  * transition to kvm_get_dirty_log_protect and kvm_get_dirty_log
1188                  * can be eliminated.
1189                  */
1190                 dirty_bitmap_buffer = dirty_bitmap;
1191         } else {
1192                 dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
1193                 memset(dirty_bitmap_buffer, 0, n);
1194 
1195                 spin_lock(&kvm->mmu_lock);
1196                 for (i = 0; i < n / sizeof(long); i++) {
1197                         unsigned long mask;
1198                         gfn_t offset;
1199 
1200                         if (!dirty_bitmap[i])
1201                                 continue;
1202 
1203                         *flush = true;
1204                         mask = xchg(&dirty_bitmap[i], 0);
1205                         dirty_bitmap_buffer[i] = mask;
1206 
1207                         offset = i * BITS_PER_LONG;
1208                         kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
1209                                                                 offset, mask);
1210                 }
1211                 spin_unlock(&kvm->mmu_lock);
1212         }
1213 
1214         if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
1215                 return -EFAULT;
1216         return 0;
1217 }
1218 EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect);
1219 
1220 /**
1221  * kvm_clear_dirty_log_protect - clear dirty bits in the bitmap
1222  *      and reenable dirty page tracking for the corresponding pages.
1223  * @kvm:        pointer to kvm instance
1224  * @log:        slot id and address from which to fetch the bitmap of dirty pages
1225  * @flush:      true if TLB flush is needed by caller
1226  */
1227 int kvm_clear_dirty_log_protect(struct kvm *kvm,
1228                                 struct kvm_clear_dirty_log *log, bool *flush)
1229 {
1230         struct kvm_memslots *slots;
1231         struct kvm_memory_slot *memslot;
1232         int as_id, id;
1233         gfn_t offset;
1234         unsigned long i, n;
1235         unsigned long *dirty_bitmap;
1236         unsigned long *dirty_bitmap_buffer;
1237 
1238         as_id = log->slot >> 16;
1239         id = (u16)log->slot;
1240         if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1241                 return -EINVAL;
1242 
1243         if (log->first_page & 63)
1244                 return -EINVAL;
1245 
1246         slots = __kvm_memslots(kvm, as_id);
1247         memslot = id_to_memslot(slots, id);
1248 
1249         dirty_bitmap = memslot->dirty_bitmap;
1250         if (!dirty_bitmap)
1251                 return -ENOENT;
1252 
1253         n = ALIGN(log->num_pages, BITS_PER_LONG) / 8;
1254 
1255         if (log->first_page > memslot->npages ||
1256             log->num_pages > memslot->npages - log->first_page ||
1257             (log->num_pages < memslot->npages - log->first_page && (log->num_pages & 63)))
1258             return -EINVAL;
1259 
1260         *flush = false;
1261         dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
1262         if (copy_from_user(dirty_bitmap_buffer, log->dirty_bitmap, n))
1263                 return -EFAULT;
1264 
1265         spin_lock(&kvm->mmu_lock);
1266         for (offset = log->first_page, i = offset / BITS_PER_LONG,
1267                  n = DIV_ROUND_UP(log->num_pages, BITS_PER_LONG); n--;
1268              i++, offset += BITS_PER_LONG) {
1269                 unsigned long mask = *dirty_bitmap_buffer++;
1270                 atomic_long_t *p = (atomic_long_t *) &dirty_bitmap[i];
1271                 if (!mask)
1272                         continue;
1273 
1274                 mask &= atomic_long_fetch_andnot(mask, p);
1275 
1276                 /*
1277                  * mask contains the bits that really have been cleared.  This
1278                  * never includes any bits beyond the length of the memslot (if
1279                  * the length is not aligned to 64 pages), therefore it is not
1280                  * a problem if userspace sets them in log->dirty_bitmap.
1281                 */
1282                 if (mask) {
1283                         *flush = true;
1284                         kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
1285                                                                 offset, mask);
1286                 }
1287         }
1288         spin_unlock(&kvm->mmu_lock);
1289 
1290         return 0;
1291 }
1292 EXPORT_SYMBOL_GPL(kvm_clear_dirty_log_protect);
1293 #endif
1294 
1295 bool kvm_largepages_enabled(void)
1296 {
1297         return largepages_enabled;
1298 }
1299 
1300 void kvm_disable_largepages(void)
1301 {
1302         largepages_enabled = false;
1303 }
1304 EXPORT_SYMBOL_GPL(kvm_disable_largepages);
1305 
1306 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
1307 {
1308         return __gfn_to_memslot(kvm_memslots(kvm), gfn);
1309 }
1310 EXPORT_SYMBOL_GPL(gfn_to_memslot);
1311 
1312 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn)
1313 {
1314         return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn);
1315 }
1316 
1317 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
1318 {
1319         struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
1320 
1321         if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
1322               memslot->flags & KVM_MEMSLOT_INVALID)
1323                 return false;
1324 
1325         return true;
1326 }
1327 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
1328 
1329 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
1330 {
1331         struct vm_area_struct *vma;
1332         unsigned long addr, size;
1333 
1334         size = PAGE_SIZE;
1335 
1336         addr = gfn_to_hva(kvm, gfn);
1337         if (kvm_is_error_hva(addr))
1338                 return PAGE_SIZE;
1339 
1340         down_read(&current->mm->mmap_sem);
1341         vma = find_vma(current->mm, addr);
1342         if (!vma)
1343                 goto out;
1344 
1345         size = vma_kernel_pagesize(vma);
1346 
1347 out:
1348         up_read(&current->mm->mmap_sem);
1349 
1350         return size;
1351 }
1352 
1353 static bool memslot_is_readonly(struct kvm_memory_slot *slot)
1354 {
1355         return slot->flags & KVM_MEM_READONLY;
1356 }
1357 
1358 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1359                                        gfn_t *nr_pages, bool write)
1360 {
1361         if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
1362                 return KVM_HVA_ERR_BAD;
1363 
1364         if (memslot_is_readonly(slot) && write)
1365                 return KVM_HVA_ERR_RO_BAD;
1366 
1367         if (nr_pages)
1368                 *nr_pages = slot->npages - (gfn - slot->base_gfn);
1369 
1370         return __gfn_to_hva_memslot(slot, gfn);
1371 }
1372 
1373 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1374                                      gfn_t *nr_pages)
1375 {
1376         return __gfn_to_hva_many(slot, gfn, nr_pages, true);
1377 }
1378 
1379 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
1380                                         gfn_t gfn)
1381 {
1382         return gfn_to_hva_many(slot, gfn, NULL);
1383 }
1384 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);
1385 
1386 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
1387 {
1388         return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1389 }
1390 EXPORT_SYMBOL_GPL(gfn_to_hva);
1391 
1392 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn)
1393 {
1394         return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL);
1395 }
1396 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva);
1397 
1398 /*
1399  * Return the hva of a @gfn and the R/W attribute if possible.
1400  *
1401  * @slot: the kvm_memory_slot which contains @gfn
1402  * @gfn: the gfn to be translated
1403  * @writable: used to return the read/write attribute of the @slot if the hva
1404  * is valid and @writable is not NULL
1405  */
1406 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot,
1407                                       gfn_t gfn, bool *writable)
1408 {
1409         unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);
1410 
1411         if (!kvm_is_error_hva(hva) && writable)
1412                 *writable = !memslot_is_readonly(slot);
1413 
1414         return hva;
1415 }
1416 
1417 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
1418 {
1419         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1420 
1421         return gfn_to_hva_memslot_prot(slot, gfn, writable);
1422 }
1423 
1424 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable)
1425 {
1426         struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1427 
1428         return gfn_to_hva_memslot_prot(slot, gfn, writable);
1429 }
1430 
1431 static inline int check_user_page_hwpoison(unsigned long addr)
1432 {
1433         int rc, flags = FOLL_HWPOISON | FOLL_WRITE;
1434 
1435         rc = get_user_pages(addr, 1, flags, NULL, NULL);
1436         return rc == -EHWPOISON;
1437 }
1438 
1439 /*
1440  * The fast path to get the writable pfn which will be stored in @pfn,
1441  * true indicates success, otherwise false is returned.  It's also the
1442  * only part that runs if we can are in atomic context.
1443  */
1444 static bool hva_to_pfn_fast(unsigned long addr, bool write_fault,
1445                             bool *writable, kvm_pfn_t *pfn)
1446 {
1447         struct page *page[1];
1448         int npages;
1449 
1450         /*
1451          * Fast pin a writable pfn only if it is a write fault request
1452          * or the caller allows to map a writable pfn for a read fault
1453          * request.
1454          */
1455         if (!(write_fault || writable))
1456                 return false;
1457 
1458         npages = __get_user_pages_fast(addr, 1, 1, page);
1459         if (npages == 1) {
1460                 *pfn = page_to_pfn(page[0]);
1461 
1462                 if (writable)
1463                         *writable = true;
1464                 return true;
1465         }
1466 
1467         return false;
1468 }
1469 
1470 /*
1471  * The slow path to get the pfn of the specified host virtual address,
1472  * 1 indicates success, -errno is returned if error is detected.
1473  */
1474 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
1475                            bool *writable, kvm_pfn_t *pfn)
1476 {
1477         unsigned int flags = FOLL_HWPOISON;
1478         struct page *page;
1479         int npages = 0;
1480 
1481         might_sleep();
1482 
1483         if (writable)
1484                 *writable = write_fault;
1485 
1486         if (write_fault)
1487                 flags |= FOLL_WRITE;
1488         if (async)
1489                 flags |= FOLL_NOWAIT;
1490 
1491         npages = get_user_pages_unlocked(addr, 1, &page, flags);
1492         if (npages != 1)
1493                 return npages;
1494 
1495         /* map read fault as writable if possible */
1496         if (unlikely(!write_fault) && writable) {
1497                 struct page *wpage;
1498 
1499                 if (__get_user_pages_fast(addr, 1, 1, &wpage) == 1) {
1500                         *writable = true;
1501                         put_page(page);
1502                         page = wpage;
1503                 }
1504         }
1505         *pfn = page_to_pfn(page);
1506         return npages;
1507 }
1508 
1509 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
1510 {
1511         if (unlikely(!(vma->vm_flags & VM_READ)))
1512                 return false;
1513 
1514         if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
1515                 return false;
1516 
1517         return true;
1518 }
1519 
1520 static int hva_to_pfn_remapped(struct vm_area_struct *vma,
1521                                unsigned long addr, bool *async,
1522                                bool write_fault, bool *writable,
1523                                kvm_pfn_t *p_pfn)
1524 {
1525         unsigned long pfn;
1526         int r;
1527 
1528         r = follow_pfn(vma, addr, &pfn);
1529         if (r) {
1530                 /*
1531                  * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does
1532                  * not call the fault handler, so do it here.
1533                  */
1534                 bool unlocked = false;
1535                 r = fixup_user_fault(current, current->mm, addr,
1536                                      (write_fault ? FAULT_FLAG_WRITE : 0),
1537                                      &unlocked);
1538                 if (unlocked)
1539                         return -EAGAIN;
1540                 if (r)
1541                         return r;
1542 
1543                 r = follow_pfn(vma, addr, &pfn);
1544                 if (r)
1545                         return r;
1546 
1547         }
1548 
1549         if (writable)
1550                 *writable = true;
1551 
1552         /*
1553          * Get a reference here because callers of *hva_to_pfn* and
1554          * *gfn_to_pfn* ultimately call kvm_release_pfn_clean on the
1555          * returned pfn.  This is only needed if the VMA has VM_MIXEDMAP
1556          * set, but the kvm_get_pfn/kvm_release_pfn_clean pair will
1557          * simply do nothing for reserved pfns.
1558          *
1559          * Whoever called remap_pfn_range is also going to call e.g.
1560          * unmap_mapping_range before the underlying pages are freed,
1561          * causing a call to our MMU notifier.
1562          */ 
1563         kvm_get_pfn(pfn);
1564 
1565         *p_pfn = pfn;
1566         return 0;
1567 }
1568 
1569 /*
1570  * Pin guest page in memory and return its pfn.
1571  * @addr: host virtual address which maps memory to the guest
1572  * @atomic: whether this function can sleep
1573  * @async: whether this function need to wait IO complete if the
1574  *         host page is not in the memory
1575  * @write_fault: whether we should get a writable host page
1576  * @writable: whether it allows to map a writable host page for !@write_fault
1577  *
1578  * The function will map a writable host page for these two cases:
1579  * 1): @write_fault = true
1580  * 2): @write_fault = false && @writable, @writable will tell the caller
1581  *     whether the mapping is writable.
1582  */
1583 static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
1584                         bool write_fault, bool *writable)
1585 {
1586         struct vm_area_struct *vma;
1587         kvm_pfn_t pfn = 0;
1588         int npages, r;
1589 
1590         /* we can do it either atomically or asynchronously, not both */
1591         BUG_ON(atomic && async);
1592 
1593         if (hva_to_pfn_fast(addr, write_fault, writable, &pfn))
1594                 return pfn;
1595 
1596         if (atomic)
1597                 return KVM_PFN_ERR_FAULT;
1598 
1599         npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
1600         if (npages == 1)
1601                 return pfn;
1602 
1603         down_read(&current->mm->mmap_sem);
1604         if (npages == -EHWPOISON ||
1605               (!async && check_user_page_hwpoison(addr))) {
1606                 pfn = KVM_PFN_ERR_HWPOISON;
1607                 goto exit;
1608         }
1609 
1610 retry:
1611         vma = find_vma_intersection(current->mm, addr, addr + 1);
1612 
1613         if (vma == NULL)
1614                 pfn = KVM_PFN_ERR_FAULT;
1615         else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) {
1616                 r = hva_to_pfn_remapped(vma, addr, async, write_fault, writable, &pfn);
1617                 if (r == -EAGAIN)
1618                         goto retry;
1619                 if (r < 0)
1620                         pfn = KVM_PFN_ERR_FAULT;
1621         } else {
1622                 if (async && vma_is_valid(vma, write_fault))
1623                         *async = true;
1624                 pfn = KVM_PFN_ERR_FAULT;
1625         }
1626 exit:
1627         up_read(&current->mm->mmap_sem);
1628         return pfn;
1629 }
1630 
1631 kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn,
1632                                bool atomic, bool *async, bool write_fault,
1633                                bool *writable)
1634 {
1635         unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);
1636 
1637         if (addr == KVM_HVA_ERR_RO_BAD) {
1638                 if (writable)
1639                         *writable = false;
1640                 return KVM_PFN_ERR_RO_FAULT;
1641         }
1642 
1643         if (kvm_is_error_hva(addr)) {
1644                 if (writable)
1645                         *writable = false;
1646                 return KVM_PFN_NOSLOT;
1647         }
1648 
1649         /* Do not map writable pfn in the readonly memslot. */
1650         if (writable && memslot_is_readonly(slot)) {
1651                 *writable = false;
1652                 writable = NULL;
1653         }
1654 
1655         return hva_to_pfn(addr, atomic, async, write_fault,
1656                           writable);
1657 }
1658 EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot);
1659 
1660 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
1661                       bool *writable)
1662 {
1663         return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL,
1664                                     write_fault, writable);
1665 }
1666 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
1667 
1668 kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
1669 {
1670         return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
1671 }
1672 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot);
1673 
1674 kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
1675 {
1676         return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
1677 }
1678 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
1679 
1680 kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
1681 {
1682         return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn);
1683 }
1684 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);
1685 
1686 kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn)
1687 {
1688         return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
1689 }
1690 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic);
1691 
1692 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
1693 {
1694         return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn);
1695 }
1696 EXPORT_SYMBOL_GPL(gfn_to_pfn);
1697 
1698 kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn)
1699 {
1700         return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
1701 }
1702 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn);
1703 
1704 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
1705                             struct page **pages, int nr_pages)
1706 {
1707         unsigned long addr;
1708         gfn_t entry = 0;
1709 
1710         addr = gfn_to_hva_many(slot, gfn, &entry);
1711         if (kvm_is_error_hva(addr))
1712                 return -1;
1713 
1714         if (entry < nr_pages)
1715                 return 0;
1716 
1717         return __get_user_pages_fast(addr, nr_pages, 1, pages);
1718 }
1719 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
1720 
1721 static struct page *kvm_pfn_to_page(kvm_pfn_t pfn)
1722 {
1723         if (is_error_noslot_pfn(pfn))
1724                 return KVM_ERR_PTR_BAD_PAGE;
1725 
1726         if (kvm_is_reserved_pfn(pfn)) {
1727                 WARN_ON(1);
1728                 return KVM_ERR_PTR_BAD_PAGE;
1729         }
1730 
1731         return pfn_to_page(pfn);
1732 }
1733 
1734 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
1735 {
1736         kvm_pfn_t pfn;
1737 
1738         pfn = gfn_to_pfn(kvm, gfn);
1739 
1740         return kvm_pfn_to_page(pfn);
1741 }
1742 EXPORT_SYMBOL_GPL(gfn_to_page);
1743 
1744 static int __kvm_map_gfn(struct kvm_memory_slot *slot, gfn_t gfn,
1745                          struct kvm_host_map *map)
1746 {
1747         kvm_pfn_t pfn;
1748         void *hva = NULL;
1749         struct page *page = KVM_UNMAPPED_PAGE;
1750 
1751         if (!map)
1752                 return -EINVAL;
1753 
1754         pfn = gfn_to_pfn_memslot(slot, gfn);
1755         if (is_error_noslot_pfn(pfn))
1756                 return -EINVAL;
1757 
1758         if (pfn_valid(pfn)) {
1759                 page = pfn_to_page(pfn);
1760                 hva = kmap(page);
1761 #ifdef CONFIG_HAS_IOMEM
1762         } else {
1763                 hva = memremap(pfn_to_hpa(pfn), PAGE_SIZE, MEMREMAP_WB);
1764 #endif
1765         }
1766 
1767         if (!hva)
1768                 return -EFAULT;
1769 
1770         map->page = page;
1771         map->hva = hva;
1772         map->pfn = pfn;
1773         map->gfn = gfn;
1774 
1775         return 0;
1776 }
1777 
1778 int kvm_vcpu_map(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map)
1779 {
1780         return __kvm_map_gfn(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, map);
1781 }
1782 EXPORT_SYMBOL_GPL(kvm_vcpu_map);
1783 
1784 void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map,
1785                     bool dirty)
1786 {
1787         if (!map)
1788                 return;
1789 
1790         if (!map->hva)
1791                 return;
1792 
1793         if (map->page != KVM_UNMAPPED_PAGE)
1794                 kunmap(map->page);
1795 #ifdef CONFIG_HAS_IOMEM
1796         else
1797                 memunmap(map->hva);
1798 #endif
1799 
1800         if (dirty) {
1801                 kvm_vcpu_mark_page_dirty(vcpu, map->gfn);
1802                 kvm_release_pfn_dirty(map->pfn);
1803         } else {
1804                 kvm_release_pfn_clean(map->pfn);
1805         }
1806 
1807         map->hva = NULL;
1808         map->page = NULL;
1809 }
1810 EXPORT_SYMBOL_GPL(kvm_vcpu_unmap);
1811 
1812 struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn)
1813 {
1814         kvm_pfn_t pfn;
1815 
1816         pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn);
1817 
1818         return kvm_pfn_to_page(pfn);
1819 }
1820 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page);
1821 
1822 void kvm_release_page_clean(struct page *page)
1823 {
1824         WARN_ON(is_error_page(page));
1825 
1826         kvm_release_pfn_clean(page_to_pfn(page));
1827 }
1828 EXPORT_SYMBOL_GPL(kvm_release_page_clean);
1829 
1830 void kvm_release_pfn_clean(kvm_pfn_t pfn)
1831 {
1832         if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn))
1833                 put_page(pfn_to_page(pfn));
1834 }
1835 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
1836 
1837 void kvm_release_page_dirty(struct page *page)
1838 {
1839         WARN_ON(is_error_page(page));
1840 
1841         kvm_release_pfn_dirty(page_to_pfn(page));
1842 }
1843 EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
1844 
1845 void kvm_release_pfn_dirty(kvm_pfn_t pfn)
1846 {
1847         kvm_set_pfn_dirty(pfn);
1848         kvm_release_pfn_clean(pfn);
1849 }
1850 EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
1851 
1852 void kvm_set_pfn_dirty(kvm_pfn_t pfn)
1853 {
1854         if (!kvm_is_reserved_pfn(pfn)) {
1855                 struct page *page = pfn_to_page(pfn);
1856 
1857                 if (!PageReserved(page))
1858                         SetPageDirty(page);
1859         }
1860 }
1861 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
1862 
1863 void kvm_set_pfn_accessed(kvm_pfn_t pfn)
1864 {
1865         if (!kvm_is_reserved_pfn(pfn))
1866                 mark_page_accessed(pfn_to_page(pfn));
1867 }
1868 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
1869 
1870 void kvm_get_pfn(kvm_pfn_t pfn)
1871 {
1872         if (!kvm_is_reserved_pfn(pfn))
1873                 get_page(pfn_to_page(pfn));
1874 }
1875 EXPORT_SYMBOL_GPL(kvm_get_pfn);
1876 
1877 static int next_segment(unsigned long len, int offset)
1878 {
1879         if (len > PAGE_SIZE - offset)
1880                 return PAGE_SIZE - offset;
1881         else
1882                 return len;
1883 }
1884 
1885 static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn,
1886                                  void *data, int offset, int len)
1887 {
1888         int r;
1889         unsigned long addr;
1890 
1891         addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
1892         if (kvm_is_error_hva(addr))
1893                 return -EFAULT;
1894         r = __copy_from_user(data, (void __user *)addr + offset, len);
1895         if (r)
1896                 return -EFAULT;
1897         return 0;
1898 }
1899 
1900 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1901                         int len)
1902 {
1903         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1904 
1905         return __kvm_read_guest_page(slot, gfn, data, offset, len);
1906 }
1907 EXPORT_SYMBOL_GPL(kvm_read_guest_page);
1908 
1909 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data,
1910                              int offset, int len)
1911 {
1912         struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1913 
1914         return __kvm_read_guest_page(slot, gfn, data, offset, len);
1915 }
1916 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page);
1917 
1918 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
1919 {
1920         gfn_t gfn = gpa >> PAGE_SHIFT;
1921         int seg;
1922         int offset = offset_in_page(gpa);
1923         int ret;
1924 
1925         while ((seg = next_segment(len, offset)) != 0) {
1926                 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
1927                 if (ret < 0)
1928                         return ret;
1929                 offset = 0;
1930                 len -= seg;
1931                 data += seg;
1932                 ++gfn;
1933         }
1934         return 0;
1935 }
1936 EXPORT_SYMBOL_GPL(kvm_read_guest);
1937 
1938 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len)
1939 {
1940         gfn_t gfn = gpa >> PAGE_SHIFT;
1941         int seg;
1942         int offset = offset_in_page(gpa);
1943         int ret;
1944 
1945         while ((seg = next_segment(len, offset)) != 0) {
1946                 ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg);
1947                 if (ret < 0)
1948                         return ret;
1949                 offset = 0;
1950                 len -= seg;
1951                 data += seg;
1952                 ++gfn;
1953         }
1954         return 0;
1955 }
1956 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest);
1957 
1958 static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
1959                                    void *data, int offset, unsigned long len)
1960 {
1961         int r;
1962         unsigned long addr;
1963 
1964         addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
1965         if (kvm_is_error_hva(addr))
1966                 return -EFAULT;
1967         pagefault_disable();
1968         r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
1969         pagefault_enable();
1970         if (r)
1971                 return -EFAULT;
1972         return 0;
1973 }
1974 
1975 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
1976                           unsigned long len)
1977 {
1978         gfn_t gfn = gpa >> PAGE_SHIFT;
1979         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1980         int offset = offset_in_page(gpa);
1981 
1982         return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
1983 }
1984 EXPORT_SYMBOL_GPL(kvm_read_guest_atomic);
1985 
1986 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa,
1987                                void *data, unsigned long len)
1988 {
1989         gfn_t gfn = gpa >> PAGE_SHIFT;
1990         struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1991         int offset = offset_in_page(gpa);
1992 
1993         return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
1994 }
1995 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic);
1996 
1997 static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn,
1998                                   const void *data, int offset, int len)
1999 {
2000         int r;
2001         unsigned long addr;
2002 
2003         addr = gfn_to_hva_memslot(memslot, gfn);
2004         if (kvm_is_error_hva(addr))
2005                 return -EFAULT;
2006         r = __copy_to_user((void __user *)addr + offset, data, len);
2007         if (r)
2008                 return -EFAULT;
2009         mark_page_dirty_in_slot(memslot, gfn);
2010         return 0;
2011 }
2012 
2013 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn,
2014                          const void *data, int offset, int len)
2015 {
2016         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
2017 
2018         return __kvm_write_guest_page(slot, gfn, data, offset, len);
2019 }
2020 EXPORT_SYMBOL_GPL(kvm_write_guest_page);
2021 
2022 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
2023                               const void *data, int offset, int len)
2024 {
2025         struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
2026 
2027         return __kvm_write_guest_page(slot, gfn, data, offset, len);
2028 }
2029 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page);
2030 
2031 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
2032                     unsigned long len)
2033 {
2034         gfn_t gfn = gpa >> PAGE_SHIFT;
2035         int seg;
2036         int offset = offset_in_page(gpa);
2037         int ret;
2038 
2039         while ((seg = next_segment(len, offset)) != 0) {
2040                 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
2041                 if (ret < 0)
2042                         return ret;
2043                 offset = 0;
2044                 len -= seg;
2045                 data += seg;
2046                 ++gfn;
2047         }
2048         return 0;
2049 }
2050 EXPORT_SYMBOL_GPL(kvm_write_guest);
2051 
2052 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data,
2053                          unsigned long len)
2054 {
2055         gfn_t gfn = gpa >> PAGE_SHIFT;
2056         int seg;
2057         int offset = offset_in_page(gpa);
2058         int ret;
2059 
2060         while ((seg = next_segment(len, offset)) != 0) {
2061                 ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg);
2062                 if (ret < 0)
2063                         return ret;
2064                 offset = 0;
2065                 len -= seg;
2066                 data += seg;
2067                 ++gfn;
2068         }
2069         return 0;
2070 }
2071 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest);
2072 
2073 static int __kvm_gfn_to_hva_cache_init(struct kvm_memslots *slots,
2074                                        struct gfn_to_hva_cache *ghc,
2075                                        gpa_t gpa, unsigned long len)
2076 {
2077         int offset = offset_in_page(gpa);
2078         gfn_t start_gfn = gpa >> PAGE_SHIFT;
2079         gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
2080         gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
2081         gfn_t nr_pages_avail;
2082         int r = start_gfn <= end_gfn ? 0 : -EINVAL;
2083 
2084         ghc->gpa = gpa;
2085         ghc->generation = slots->generation;
2086         ghc->len = len;
2087         ghc->hva = KVM_HVA_ERR_BAD;
2088 
2089         /*
2090          * If the requested region crosses two memslots, we still
2091          * verify that the entire region is valid here.
2092          */
2093         while (!r && start_gfn <= end_gfn) {
2094                 ghc->memslot = __gfn_to_memslot(slots, start_gfn);
2095                 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
2096                                            &nr_pages_avail);
2097                 if (kvm_is_error_hva(ghc->hva))
2098                         r = -EFAULT;
2099                 start_gfn += nr_pages_avail;
2100         }
2101 
2102         /* Use the slow path for cross page reads and writes. */
2103         if (!r && nr_pages_needed == 1)
2104                 ghc->hva += offset;
2105         else
2106                 ghc->memslot = NULL;
2107 
2108         return r;
2109 }
2110 
2111 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
2112                               gpa_t gpa, unsigned long len)
2113 {
2114         struct kvm_memslots *slots = kvm_memslots(kvm);
2115         return __kvm_gfn_to_hva_cache_init(slots, ghc, gpa, len);
2116 }
2117 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
2118 
2119 int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
2120                                   void *data, unsigned int offset,
2121                                   unsigned long len)
2122 {
2123         struct kvm_memslots *slots = kvm_memslots(kvm);
2124         int r;
2125         gpa_t gpa = ghc->gpa + offset;
2126 
2127         BUG_ON(len + offset > ghc->len);
2128 
2129         if (slots->generation != ghc->generation)
2130                 __kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len);
2131 
2132         if (unlikely(!ghc->memslot))
2133                 return kvm_write_guest(kvm, gpa, data, len);
2134 
2135         if (kvm_is_error_hva(ghc->hva))
2136                 return -EFAULT;
2137 
2138         r = __copy_to_user((void __user *)ghc->hva + offset, data, len);
2139         if (r)
2140                 return -EFAULT;
2141         mark_page_dirty_in_slot(ghc->memslot, gpa >> PAGE_SHIFT);
2142 
2143         return 0;
2144 }
2145 EXPORT_SYMBOL_GPL(kvm_write_guest_offset_cached);
2146 
2147 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
2148                            void *data, unsigned long len)
2149 {
2150         return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len);
2151 }
2152 EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
2153 
2154 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
2155                            void *data, unsigned long len)
2156 {
2157         struct kvm_memslots *slots = kvm_memslots(kvm);
2158         int r;
2159 
2160         BUG_ON(len > ghc->len);
2161 
2162         if (slots->generation != ghc->generation)
2163                 __kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len);
2164 
2165         if (unlikely(!ghc->memslot))
2166                 return kvm_read_guest(kvm, ghc->gpa, data, len);
2167 
2168         if (kvm_is_error_hva(ghc->hva))
2169                 return -EFAULT;
2170 
2171         r = __copy_from_user(data, (void __user *)ghc->hva, len);
2172         if (r)
2173                 return -EFAULT;
2174 
2175         return 0;
2176 }
2177 EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
2178 
2179 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
2180 {
2181         const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));
2182 
2183         return kvm_write_guest_page(kvm, gfn, zero_page, offset, len);
2184 }
2185 EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
2186 
2187 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
2188 {
2189         gfn_t gfn = gpa >> PAGE_SHIFT;
2190         int seg;
2191         int offset = offset_in_page(gpa);
2192         int ret;
2193 
2194         while ((seg = next_segment(len, offset)) != 0) {
2195                 ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
2196                 if (ret < 0)
2197                         return ret;
2198                 offset = 0;
2199                 len -= seg;
2200                 ++gfn;
2201         }
2202         return 0;
2203 }
2204 EXPORT_SYMBOL_GPL(kvm_clear_guest);
2205 
2206 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot,
2207                                     gfn_t gfn)
2208 {
2209         if (memslot && memslot->dirty_bitmap) {
2210                 unsigned long rel_gfn = gfn - memslot->base_gfn;
2211 
2212                 set_bit_le(rel_gfn, memslot->dirty_bitmap);
2213         }
2214 }
2215 
2216 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
2217 {
2218         struct kvm_memory_slot *memslot;
2219 
2220         memslot = gfn_to_memslot(kvm, gfn);
2221         mark_page_dirty_in_slot(memslot, gfn);
2222 }
2223 EXPORT_SYMBOL_GPL(mark_page_dirty);
2224 
2225 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn)
2226 {
2227         struct kvm_memory_slot *memslot;
2228 
2229         memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
2230         mark_page_dirty_in_slot(memslot, gfn);
2231 }
2232 EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty);
2233 
2234 void kvm_sigset_activate(struct kvm_vcpu *vcpu)
2235 {
2236         if (!vcpu->sigset_active)
2237                 return;
2238 
2239         /*
2240          * This does a lockless modification of ->real_blocked, which is fine
2241          * because, only current can change ->real_blocked and all readers of
2242          * ->real_blocked don't care as long ->real_blocked is always a subset
2243          * of ->blocked.
2244          */
2245         sigprocmask(SIG_SETMASK, &vcpu->sigset, &current->real_blocked);
2246 }
2247 
2248 void kvm_sigset_deactivate(struct kvm_vcpu *vcpu)
2249 {
2250         if (!vcpu->sigset_active)
2251                 return;
2252 
2253         sigprocmask(SIG_SETMASK, &current->real_blocked, NULL);
2254         sigemptyset(&current->real_blocked);
2255 }
2256 
2257 static void grow_halt_poll_ns(struct kvm_vcpu *vcpu)
2258 {
2259         unsigned int old, val, grow, grow_start;
2260 
2261         old = val = vcpu->halt_poll_ns;
2262         grow_start = READ_ONCE(halt_poll_ns_grow_start);
2263         grow = READ_ONCE(halt_poll_ns_grow);
2264         if (!grow)
2265                 goto out;
2266 
2267         val *= grow;
2268         if (val < grow_start)
2269                 val = grow_start;
2270 
2271         if (val > halt_poll_ns)
2272                 val = halt_poll_ns;
2273 
2274         vcpu->halt_poll_ns = val;
2275 out:
2276         trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old);
2277 }
2278 
2279 static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu)
2280 {
2281         unsigned int old, val, shrink;
2282 
2283         old = val = vcpu->halt_poll_ns;
2284         shrink = READ_ONCE(halt_poll_ns_shrink);
2285         if (shrink == 0)
2286                 val = 0;
2287         else
2288                 val /= shrink;
2289 
2290         vcpu->halt_poll_ns = val;
2291         trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old);
2292 }
2293 
2294 static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
2295 {
2296         int ret = -EINTR;
2297         int idx = srcu_read_lock(&vcpu->kvm->srcu);
2298 
2299         if (kvm_arch_vcpu_runnable(vcpu)) {
2300                 kvm_make_request(KVM_REQ_UNHALT, vcpu);
2301                 goto out;
2302         }
2303         if (kvm_cpu_has_pending_timer(vcpu))
2304                 goto out;
2305         if (signal_pending(current))
2306                 goto out;
2307 
2308         ret = 0;
2309 out:
2310         srcu_read_unlock(&vcpu->kvm->srcu, idx);
2311         return ret;
2312 }
2313 
2314 /*
2315  * The vCPU has executed a HLT instruction with in-kernel mode enabled.
2316  */
2317 void kvm_vcpu_block(struct kvm_vcpu *vcpu)
2318 {
2319         ktime_t start, cur;
2320         DECLARE_SWAITQUEUE(wait);
2321         bool waited = false;
2322         u64 block_ns;
2323 
2324         start = cur = ktime_get();
2325         if (vcpu->halt_poll_ns && !kvm_arch_no_poll(vcpu)) {
2326                 ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns);
2327 
2328                 ++vcpu->stat.halt_attempted_poll;
2329                 do {
2330                         /*
2331                          * This sets KVM_REQ_UNHALT if an interrupt
2332                          * arrives.
2333                          */
2334                         if (kvm_vcpu_check_block(vcpu) < 0) {
2335                                 ++vcpu->stat.halt_successful_poll;
2336                                 if (!vcpu_valid_wakeup(vcpu))
2337                                         ++vcpu->stat.halt_poll_invalid;
2338                                 goto out;
2339                         }
2340                         cur = ktime_get();
2341                 } while (single_task_running() && ktime_before(cur, stop));
2342         }
2343 
2344         kvm_arch_vcpu_blocking(vcpu);
2345 
2346         for (;;) {
2347                 prepare_to_swait_exclusive(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
2348 
2349                 if (kvm_vcpu_check_block(vcpu) < 0)
2350                         break;
2351 
2352                 waited = true;
2353                 schedule();
2354         }
2355 
2356         finish_swait(&vcpu->wq, &wait);
2357         cur = ktime_get();
2358 
2359         kvm_arch_vcpu_unblocking(vcpu);
2360 out:
2361         block_ns = ktime_to_ns(cur) - ktime_to_ns(start);
2362 
2363         if (!vcpu_valid_wakeup(vcpu))
2364                 shrink_halt_poll_ns(vcpu);
2365         else if (halt_poll_ns) {
2366                 if (block_ns <= vcpu->halt_poll_ns)
2367                         ;
2368                 /* we had a long block, shrink polling */
2369                 else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns)
2370                         shrink_halt_poll_ns(vcpu);
2371                 /* we had a short halt and our poll time is too small */
2372                 else if (vcpu->halt_poll_ns < halt_poll_ns &&
2373                         block_ns < halt_poll_ns)
2374                         grow_halt_poll_ns(vcpu);
2375         } else
2376                 vcpu->halt_poll_ns = 0;
2377 
2378         trace_kvm_vcpu_wakeup(block_ns, waited, vcpu_valid_wakeup(vcpu));
2379         kvm_arch_vcpu_block_finish(vcpu);
2380 }
2381 EXPORT_SYMBOL_GPL(kvm_vcpu_block);
2382 
2383 bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu)
2384 {
2385         struct swait_queue_head *wqp;
2386 
2387         wqp = kvm_arch_vcpu_wq(vcpu);
2388         if (swq_has_sleeper(wqp)) {
2389                 swake_up_one(wqp);
2390                 ++vcpu->stat.halt_wakeup;
2391                 return true;
2392         }
2393 
2394         return false;
2395 }
2396 EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up);
2397 
2398 #ifndef CONFIG_S390
2399 /*
2400  * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
2401  */
2402 void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
2403 {
2404         int me;
2405         int cpu = vcpu->cpu;
2406 
2407         if (kvm_vcpu_wake_up(vcpu))
2408                 return;
2409 
2410         me = get_cpu();
2411         if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
2412                 if (kvm_arch_vcpu_should_kick(vcpu))
2413                         smp_send_reschedule(cpu);
2414         put_cpu();
2415 }
2416 EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
2417 #endif /* !CONFIG_S390 */
2418 
2419 int kvm_vcpu_yield_to(struct kvm_vcpu *target)
2420 {
2421         struct pid *pid;
2422         struct task_struct *task = NULL;
2423         int ret = 0;
2424 
2425         rcu_read_lock();
2426         pid = rcu_dereference(target->pid);
2427         if (pid)
2428                 task = get_pid_task(pid, PIDTYPE_PID);
2429         rcu_read_unlock();
2430         if (!task)
2431                 return ret;
2432         ret = yield_to(task, 1);
2433         put_task_struct(task);
2434 
2435         return ret;
2436 }
2437 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
2438 
2439 /*
2440  * Helper that checks whether a VCPU is eligible for directed yield.
2441  * Most eligible candidate to yield is decided by following heuristics:
2442  *
2443  *  (a) VCPU which has not done pl-exit or cpu relax intercepted recently
2444  *  (preempted lock holder), indicated by @in_spin_loop.
2445  *  Set at the beiginning and cleared at the end of interception/PLE handler.
2446  *
2447  *  (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
2448  *  chance last time (mostly it has become eligible now since we have probably
2449  *  yielded to lockholder in last iteration. This is done by toggling
2450  *  @dy_eligible each time a VCPU checked for eligibility.)
2451  *
2452  *  Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
2453  *  to preempted lock-holder could result in wrong VCPU selection and CPU
2454  *  burning. Giving priority for a potential lock-holder increases lock
2455  *  progress.
2456  *
2457  *  Since algorithm is based on heuristics, accessing another VCPU data without
2458  *  locking does not harm. It may result in trying to yield to  same VCPU, fail
2459  *  and continue with next VCPU and so on.
2460  */
2461 static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
2462 {
2463 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
2464         bool eligible;
2465 
2466         eligible = !vcpu->spin_loop.in_spin_loop ||
2467                     vcpu->spin_loop.dy_eligible;
2468 
2469         if (vcpu->spin_loop.in_spin_loop)
2470                 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
2471 
2472         return eligible;
2473 #else
2474         return true;
2475 #endif
2476 }
2477 
2478 /*
2479  * Unlike kvm_arch_vcpu_runnable, this function is called outside
2480  * a vcpu_load/vcpu_put pair.  However, for most architectures
2481  * kvm_arch_vcpu_runnable does not require vcpu_load.
2482  */
2483 bool __weak kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
2484 {
2485         return kvm_arch_vcpu_runnable(vcpu);
2486 }
2487 
2488 static bool vcpu_dy_runnable(struct kvm_vcpu *vcpu)
2489 {
2490         if (kvm_arch_dy_runnable(vcpu))
2491                 return true;
2492 
2493 #ifdef CONFIG_KVM_ASYNC_PF
2494         if (!list_empty_careful(&vcpu->async_pf.done))
2495                 return true;
2496 #endif
2497 
2498         return false;
2499 }
2500 
2501 void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode)
2502 {
2503         struct kvm *kvm = me->kvm;
2504         struct kvm_vcpu *vcpu;
2505         int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
2506         int yielded = 0;
2507         int try = 3;
2508         int pass;
2509         int i;
2510 
2511         kvm_vcpu_set_in_spin_loop(me, true);
2512         /*
2513          * We boost the priority of a VCPU that is runnable but not
2514          * currently running, because it got preempted by something
2515          * else and called schedule in __vcpu_run.  Hopefully that
2516          * VCPU is holding the lock that we need and will release it.
2517          * We approximate round-robin by starting at the last boosted VCPU.
2518          */
2519         for (pass = 0; pass < 2 && !yielded && try; pass++) {
2520                 kvm_for_each_vcpu(i, vcpu, kvm) {
2521                         if (!pass && i <= last_boosted_vcpu) {
2522                                 i = last_boosted_vcpu;
2523                                 continue;
2524                         } else if (pass && i > last_boosted_vcpu)
2525                                 break;
2526                         if (!READ_ONCE(vcpu->preempted))
2527                                 continue;
2528                         if (vcpu == me)
2529                                 continue;
2530                         if (swait_active(&vcpu->wq) && !vcpu_dy_runnable(vcpu))
2531                                 continue;
2532                         if (yield_to_kernel_mode && !kvm_arch_vcpu_in_kernel(vcpu))
2533                                 continue;
2534                         if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
2535                                 continue;
2536 
2537                         yielded = kvm_vcpu_yield_to(vcpu);
2538                         if (yielded > 0) {
2539                                 kvm->last_boosted_vcpu = i;
2540                                 break;
2541                         } else if (yielded < 0) {
2542                                 try--;
2543                                 if (!try)
2544                                         break;
2545                         }
2546                 }
2547         }
2548         kvm_vcpu_set_in_spin_loop(me, false);
2549 
2550         /* Ensure vcpu is not eligible during next spinloop */
2551         kvm_vcpu_set_dy_eligible(me, false);
2552 }
2553 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
2554 
2555 static vm_fault_t kvm_vcpu_fault(struct vm_fault *vmf)
2556 {
2557         struct kvm_vcpu *vcpu = vmf->vma->vm_file->private_data;
2558         struct page *page;
2559 
2560         if (vmf->pgoff == 0)
2561                 page = virt_to_page(vcpu->run);
2562 #ifdef CONFIG_X86
2563         else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
2564                 page = virt_to_page(vcpu->arch.pio_data);
2565 #endif
2566 #ifdef CONFIG_KVM_MMIO
2567         else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
2568                 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
2569 #endif
2570         else
2571                 return kvm_arch_vcpu_fault(vcpu, vmf);
2572         get_page(page);
2573         vmf->page = page;
2574         return 0;
2575 }
2576 
2577 static const struct vm_operations_struct kvm_vcpu_vm_ops = {
2578         .fault = kvm_vcpu_fault,
2579 };
2580 
2581 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
2582 {
2583         vma->vm_ops = &kvm_vcpu_vm_ops;
2584         return 0;
2585 }
2586 
2587 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
2588 {
2589         struct kvm_vcpu *vcpu = filp->private_data;
2590 
2591         debugfs_remove_recursive(vcpu->debugfs_dentry);
2592         kvm_put_kvm(vcpu->kvm);
2593         return 0;
2594 }
2595 
2596 static struct file_operations kvm_vcpu_fops = {
2597         .release        = kvm_vcpu_release,
2598         .unlocked_ioctl = kvm_vcpu_ioctl,
2599         .mmap           = kvm_vcpu_mmap,
2600         .llseek         = noop_llseek,
2601         KVM_COMPAT(kvm_vcpu_compat_ioctl),
2602 };
2603 
2604 /*
2605  * Allocates an inode for the vcpu.
2606  */
2607 static int create_vcpu_fd(struct kvm_vcpu *vcpu)
2608 {
2609         char name[8 + 1 + ITOA_MAX_LEN + 1];
2610 
2611         snprintf(name, sizeof(name), "kvm-vcpu:%d", vcpu->vcpu_id);
2612         return anon_inode_getfd(name, &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
2613 }
2614 
2615 static int kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
2616 {
2617         char dir_name[ITOA_MAX_LEN * 2];
2618         int ret;
2619 
2620         if (!kvm_arch_has_vcpu_debugfs())
2621                 return 0;
2622 
2623         if (!debugfs_initialized())
2624                 return 0;
2625 
2626         snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id);
2627         vcpu->debugfs_dentry = debugfs_create_dir(dir_name,
2628                                                                 vcpu->kvm->debugfs_dentry);
2629         if (!vcpu->debugfs_dentry)
2630                 return -ENOMEM;
2631 
2632         ret = kvm_arch_create_vcpu_debugfs(vcpu);
2633         if (ret < 0) {
2634                 debugfs_remove_recursive(vcpu->debugfs_dentry);
2635                 return ret;
2636         }
2637 
2638         return 0;
2639 }
2640 
2641 /*
2642  * Creates some virtual cpus.  Good luck creating more than one.
2643  */
2644 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
2645 {
2646         int r;
2647         struct kvm_vcpu *vcpu;
2648 
2649         if (id >= KVM_MAX_VCPU_ID)
2650                 return -EINVAL;
2651 
2652         mutex_lock(&kvm->lock);
2653         if (kvm->created_vcpus == KVM_MAX_VCPUS) {
2654                 mutex_unlock(&kvm->lock);
2655                 return -EINVAL;
2656         }
2657 
2658         kvm->created_vcpus++;
2659         mutex_unlock(&kvm->lock);
2660 
2661         vcpu = kvm_arch_vcpu_create(kvm, id);
2662         if (IS_ERR(vcpu)) {
2663                 r = PTR_ERR(vcpu);
2664                 goto vcpu_decrement;
2665         }
2666 
2667         preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
2668 
2669         r = kvm_arch_vcpu_setup(vcpu);
2670         if (r)
2671                 goto vcpu_destroy;
2672 
2673         r = kvm_create_vcpu_debugfs(vcpu);
2674         if (r)
2675                 goto vcpu_destroy;
2676 
2677         mutex_lock(&kvm->lock);
2678         if (kvm_get_vcpu_by_id(kvm, id)) {
2679                 r = -EEXIST;
2680                 goto unlock_vcpu_destroy;
2681         }
2682 
2683         BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
2684 
2685         /* Now it's all set up, let userspace reach it */
2686         kvm_get_kvm(kvm);
2687         r = create_vcpu_fd(vcpu);
2688         if (r < 0) {
2689                 kvm_put_kvm(kvm);
2690                 goto unlock_vcpu_destroy;
2691         }
2692 
2693         kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
2694 
2695         /*
2696          * Pairs with smp_rmb() in kvm_get_vcpu.  Write kvm->vcpus
2697          * before kvm->online_vcpu's incremented value.
2698          */
2699         smp_wmb();
2700         atomic_inc(&kvm->online_vcpus);
2701 
2702         mutex_unlock(&kvm->lock);
2703         kvm_arch_vcpu_postcreate(vcpu);
2704         return r;
2705 
2706 unlock_vcpu_destroy:
2707         mutex_unlock(&kvm->lock);
2708         debugfs_remove_recursive(vcpu->debugfs_dentry);
2709 vcpu_destroy:
2710         kvm_arch_vcpu_destroy(vcpu);
2711 vcpu_decrement:
2712         mutex_lock(&kvm->lock);
2713         kvm->created_vcpus--;
2714         mutex_unlock(&kvm->lock);
2715         return r;
2716 }
2717 
2718 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
2719 {
2720         if (sigset) {
2721                 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
2722                 vcpu->sigset_active = 1;
2723                 vcpu->sigset = *sigset;
2724         } else
2725                 vcpu->sigset_active = 0;
2726         return 0;
2727 }
2728 
2729 static long kvm_vcpu_ioctl(struct file *filp,
2730                            unsigned int ioctl, unsigned long arg)
2731 {
2732         struct kvm_vcpu *vcpu = filp->private_data;
2733         void __user *argp = (void __user *)arg;
2734         int r;
2735         struct kvm_fpu *fpu = NULL;
2736         struct kvm_sregs *kvm_sregs = NULL;
2737 
2738         if (vcpu->kvm->mm != current->mm)
2739                 return -EIO;
2740 
2741         if (unlikely(_IOC_TYPE(ioctl) != KVMIO))
2742                 return -EINVAL;
2743 
2744         /*
2745          * Some architectures have vcpu ioctls that are asynchronous to vcpu
2746          * execution; mutex_lock() would break them.
2747          */
2748         r = kvm_arch_vcpu_async_ioctl(filp, ioctl, arg);
2749         if (r != -ENOIOCTLCMD)
2750                 return r;
2751 
2752         if (mutex_lock_killable(&vcpu->mutex))
2753                 return -EINTR;
2754         switch (ioctl) {
2755         case KVM_RUN: {
2756                 struct pid *oldpid;
2757                 r = -EINVAL;
2758                 if (arg)
2759                         goto out;
2760                 oldpid = rcu_access_pointer(vcpu->pid);
2761                 if (unlikely(oldpid != task_pid(current))) {
2762                         /* The thread running this VCPU changed. */
2763                         struct pid *newpid;
2764 
2765                         r = kvm_arch_vcpu_run_pid_change(vcpu);
2766                         if (r)
2767                                 break;
2768 
2769                         newpid = get_task_pid(current, PIDTYPE_PID);
2770                         rcu_assign_pointer(vcpu->pid, newpid);
2771                         if (oldpid)
2772                                 synchronize_rcu();
2773                         put_pid(oldpid);
2774                 }
2775                 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
2776                 trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
2777                 break;
2778         }
2779         case KVM_GET_REGS: {
2780                 struct kvm_regs *kvm_regs;
2781 
2782                 r = -ENOMEM;
2783                 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL_ACCOUNT);
2784                 if (!kvm_regs)
2785                         goto out;
2786                 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
2787                 if (r)
2788                         goto out_free1;
2789                 r = -EFAULT;
2790                 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
2791                         goto out_free1;
2792                 r = 0;
2793 out_free1:
2794                 kfree(kvm_regs);
2795                 break;
2796         }
2797         case KVM_SET_REGS: {
2798                 struct kvm_regs *kvm_regs;
2799 
2800                 r = -ENOMEM;
2801                 kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
2802                 if (IS_ERR(kvm_regs)) {
2803                         r = PTR_ERR(kvm_regs);
2804                         goto out;
2805                 }
2806                 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
2807                 kfree(kvm_regs);
2808                 break;
2809         }
2810         case KVM_GET_SREGS: {
2811                 kvm_sregs = kzalloc(sizeof(struct kvm_sregs),
2812                                     GFP_KERNEL_ACCOUNT);
2813                 r = -ENOMEM;
2814                 if (!kvm_sregs)
2815                         goto out;
2816                 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
2817                 if (r)
2818                         goto out;
2819                 r = -EFAULT;
2820                 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
2821                         goto out;
2822                 r = 0;
2823                 break;
2824         }
2825         case KVM_SET_SREGS: {
2826                 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
2827                 if (IS_ERR(kvm_sregs)) {
2828                         r = PTR_ERR(kvm_sregs);
2829                         kvm_sregs = NULL;
2830                         goto out;
2831                 }
2832                 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
2833                 break;
2834         }
2835         case KVM_GET_MP_STATE: {
2836                 struct kvm_mp_state mp_state;
2837 
2838                 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
2839                 if (r)
2840                         goto out;
2841                 r = -EFAULT;
2842                 if (copy_to_user(argp, &mp_state, sizeof(mp_state)))
2843                         goto out;
2844                 r = 0;
2845                 break;
2846         }
2847         case KVM_SET_MP_STATE: {
2848                 struct kvm_mp_state mp_state;
2849 
2850                 r = -EFAULT;
2851                 if (copy_from_user(&mp_state, argp, sizeof(mp_state)))
2852                         goto out;
2853                 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
2854                 break;
2855         }
2856         case KVM_TRANSLATE: {
2857                 struct kvm_translation tr;
2858 
2859                 r = -EFAULT;
2860                 if (copy_from_user(&tr, argp, sizeof(tr)))
2861                         goto out;
2862                 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
2863                 if (r)
2864                         goto out;
2865                 r = -EFAULT;
2866                 if (copy_to_user(argp, &tr, sizeof(tr)))
2867                         goto out;
2868                 r = 0;
2869                 break;
2870         }
2871         case KVM_SET_GUEST_DEBUG: {
2872                 struct kvm_guest_debug dbg;
2873 
2874                 r = -EFAULT;
2875                 if (copy_from_user(&dbg, argp, sizeof(dbg)))
2876                         goto out;
2877                 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
2878                 break;
2879         }
2880         case KVM_SET_SIGNAL_MASK: {
2881                 struct kvm_signal_mask __user *sigmask_arg = argp;
2882                 struct kvm_signal_mask kvm_sigmask;
2883                 sigset_t sigset, *p;
2884 
2885                 p = NULL;
2886                 if (argp) {
2887                         r = -EFAULT;
2888                         if (copy_from_user(&kvm_sigmask, argp,
2889                                            sizeof(kvm_sigmask)))
2890                                 goto out;
2891                         r = -EINVAL;
2892                         if (kvm_sigmask.len != sizeof(sigset))
2893                                 goto out;
2894                         r = -EFAULT;
2895                         if (copy_from_user(&sigset, sigmask_arg->sigset,
2896                                            sizeof(sigset)))
2897                                 goto out;
2898                         p = &sigset;
2899                 }
2900                 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
2901                 break;
2902         }
2903         case KVM_GET_FPU: {
2904                 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL_ACCOUNT);
2905                 r = -ENOMEM;
2906                 if (!fpu)
2907                         goto out;
2908                 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
2909                 if (r)
2910                         goto out;
2911                 r = -EFAULT;
2912                 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
2913                         goto out;
2914                 r = 0;
2915                 break;
2916         }
2917         case KVM_SET_FPU: {
2918                 fpu = memdup_user(argp, sizeof(*fpu));
2919                 if (IS_ERR(fpu)) {
2920                         r = PTR_ERR(fpu);
2921                         fpu = NULL;
2922                         goto out;
2923                 }
2924                 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
2925                 break;
2926         }
2927         default:
2928                 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
2929         }
2930 out:
2931         mutex_unlock(&vcpu->mutex);
2932         kfree(fpu);
2933         kfree(kvm_sregs);
2934         return r;
2935 }
2936 
2937 #ifdef CONFIG_KVM_COMPAT
2938 static long kvm_vcpu_compat_ioctl(struct file *filp,
2939                                   unsigned int ioctl, unsigned long arg)
2940 {
2941         struct kvm_vcpu *vcpu = filp->private_data;
2942         void __user *argp = compat_ptr(arg);
2943         int r;
2944 
2945         if (vcpu->kvm->mm != current->mm)
2946                 return -EIO;
2947 
2948         switch (ioctl) {
2949         case KVM_SET_SIGNAL_MASK: {
2950                 struct kvm_signal_mask __user *sigmask_arg = argp;
2951                 struct kvm_signal_mask kvm_sigmask;
2952                 sigset_t sigset;
2953 
2954                 if (argp) {
2955                         r = -EFAULT;
2956                         if (copy_from_user(&kvm_sigmask, argp,
2957                                            sizeof(kvm_sigmask)))
2958                                 goto out;
2959                         r = -EINVAL;
2960                         if (kvm_sigmask.len != sizeof(compat_sigset_t))
2961                                 goto out;
2962                         r = -EFAULT;
2963                         if (get_compat_sigset(&sigset, (void *)sigmask_arg->sigset))
2964                                 goto out;
2965                         r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
2966                 } else
2967                         r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
2968                 break;
2969         }
2970         default:
2971                 r = kvm_vcpu_ioctl(filp, ioctl, arg);
2972         }
2973 
2974 out:
2975         return r;
2976 }
2977 #endif
2978 
2979 static int kvm_device_mmap(struct file *filp, struct vm_area_struct *vma)
2980 {
2981         struct kvm_device *dev = filp->private_data;
2982 
2983         if (dev->ops->mmap)
2984                 return dev->ops->mmap(dev, vma);
2985 
2986         return -ENODEV;
2987 }
2988 
2989 static int kvm_device_ioctl_attr(struct kvm_device *dev,
2990                                  int (*accessor)(struct kvm_device *dev,
2991                                                  struct kvm_device_attr *attr),
2992                                  unsigned long arg)
2993 {
2994         struct kvm_device_attr attr;
2995 
2996         if (!accessor)
2997                 return -EPERM;
2998 
2999         if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
3000                 return -EFAULT;
3001 
3002         return accessor(dev, &attr);
3003 }
3004 
3005 static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
3006                              unsigned long arg)
3007 {
3008         struct kvm_device *dev = filp->private_data;
3009 
3010         if (dev->kvm->mm != current->mm)
3011                 return -EIO;
3012 
3013         switch (ioctl) {
3014         case KVM_SET_DEVICE_ATTR:
3015                 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
3016         case KVM_GET_DEVICE_ATTR:
3017                 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
3018         case KVM_HAS_DEVICE_ATTR:
3019                 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
3020         default:
3021                 if (dev->ops->ioctl)
3022                         return dev->ops->ioctl(dev, ioctl, arg);
3023 
3024                 return -ENOTTY;
3025         }
3026 }
3027 
3028 static int kvm_device_release(struct inode *inode, struct file *filp)
3029 {
3030         struct kvm_device *dev = filp->private_data;
3031         struct kvm *kvm = dev->kvm;
3032 
3033         if (dev->ops->release) {
3034                 mutex_lock(&kvm->lock);
3035                 list_del(&dev->vm_node);
3036                 dev->ops->release(dev);
3037                 mutex_unlock(&kvm->lock);
3038         }
3039 
3040         kvm_put_kvm(kvm);
3041         return 0;
3042 }
3043 
3044 static const struct file_operations kvm_device_fops = {
3045         .unlocked_ioctl = kvm_device_ioctl,
3046         .release = kvm_device_release,
3047         KVM_COMPAT(kvm_device_ioctl),
3048         .mmap = kvm_device_mmap,
3049 };
3050 
3051 struct kvm_device *kvm_device_from_filp(struct file *filp)
3052 {
3053         if (filp->f_op != &kvm_device_fops)
3054                 return NULL;
3055 
3056         return filp->private_data;
3057 }
3058 
3059 static struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = {
3060 #ifdef CONFIG_KVM_MPIC
3061         [KVM_DEV_TYPE_FSL_MPIC_20]      = &kvm_mpic_ops,
3062         [KVM_DEV_TYPE_FSL_MPIC_42]      = &kvm_mpic_ops,
3063 #endif
3064 };
3065 
3066 int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type)
3067 {
3068         if (type >= ARRAY_SIZE(kvm_device_ops_table))
3069                 return -ENOSPC;
3070 
3071         if (kvm_device_ops_table[type] != NULL)
3072                 return -EEXIST;
3073 
3074         kvm_device_ops_table[type] = ops;
3075         return 0;
3076 }
3077 
3078 void kvm_unregister_device_ops(u32 type)
3079 {
3080         if (kvm_device_ops_table[type] != NULL)
3081                 kvm_device_ops_table[type] = NULL;
3082 }
3083 
3084 static int kvm_ioctl_create_device(struct kvm *kvm,
3085                                    struct kvm_create_device *cd)
3086 {
3087         struct kvm_device_ops *ops = NULL;
3088         struct kvm_device *dev;
3089         bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
3090         int type;
3091         int ret;
3092 
3093         if (cd->type >= ARRAY_SIZE(kvm_device_ops_table))
3094                 return -ENODEV;
3095 
3096         type = array_index_nospec(cd->type, ARRAY_SIZE(kvm_device_ops_table));
3097         ops = kvm_device_ops_table[type];
3098         if (ops == NULL)
3099                 return -ENODEV;
3100 
3101         if (test)
3102                 return 0;
3103 
3104         dev = kzalloc(sizeof(*dev), GFP_KERNEL_ACCOUNT);
3105         if (!dev)
3106                 return -ENOMEM;
3107 
3108         dev->ops = ops;
3109         dev->kvm = kvm;
3110 
3111         mutex_lock(&kvm->lock);
3112         ret = ops->create(dev, type);
3113         if (ret < 0) {
3114                 mutex_unlock(&kvm->lock);
3115                 kfree(dev);
3116                 return ret;
3117         }
3118         list_add(&dev->vm_node, &kvm->devices);
3119         mutex_unlock(&kvm->lock);
3120 
3121         if (ops->init)
3122                 ops->init(dev);
3123 
3124         kvm_get_kvm(kvm);
3125         ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
3126         if (ret < 0) {
3127                 kvm_put_kvm(kvm);
3128                 mutex_lock(&kvm->lock);
3129                 list_del(&dev->vm_node);
3130                 mutex_unlock(&kvm->lock);
3131                 ops->destroy(dev);
3132                 return ret;
3133         }
3134 
3135         cd->fd = ret;
3136         return 0;
3137 }
3138 
3139 static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
3140 {
3141         switch (arg) {
3142         case KVM_CAP_USER_MEMORY:
3143         case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
3144         case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
3145         case KVM_CAP_INTERNAL_ERROR_DATA:
3146 #ifdef CONFIG_HAVE_KVM_MSI
3147         case KVM_CAP_SIGNAL_MSI:
3148 #endif
3149 #ifdef CONFIG_HAVE_KVM_IRQFD
3150         case KVM_CAP_IRQFD:
3151         case KVM_CAP_IRQFD_RESAMPLE:
3152 #endif
3153         case KVM_CAP_IOEVENTFD_ANY_LENGTH:
3154         case KVM_CAP_CHECK_EXTENSION_VM:
3155         case KVM_CAP_ENABLE_CAP_VM:
3156 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
3157         case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2:
3158 #endif
3159                 return 1;
3160 #ifdef CONFIG_KVM_MMIO
3161         case KVM_CAP_COALESCED_MMIO:
3162                 return KVM_COALESCED_MMIO_PAGE_OFFSET;
3163         case KVM_CAP_COALESCED_PIO:
3164                 return 1;
3165 #endif
3166 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
3167         case KVM_CAP_IRQ_ROUTING:
3168                 return KVM_MAX_IRQ_ROUTES;
3169 #endif
3170 #if KVM_ADDRESS_SPACE_NUM > 1
3171         case KVM_CAP_MULTI_ADDRESS_SPACE:
3172                 return KVM_ADDRESS_SPACE_NUM;
3173 #endif
3174         case KVM_CAP_NR_MEMSLOTS:
3175                 return KVM_USER_MEM_SLOTS;
3176         default:
3177                 break;
3178         }
3179         return kvm_vm_ioctl_check_extension(kvm, arg);
3180 }
3181 
3182 int __attribute__((weak)) kvm_vm_ioctl_enable_cap(struct kvm *kvm,
3183                                                   struct kvm_enable_cap *cap)
3184 {
3185         return -EINVAL;
3186 }
3187 
3188 static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm,
3189                                            struct kvm_enable_cap *cap)
3190 {
3191         switch (cap->cap) {
3192 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
3193         case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2:
3194                 if (cap->flags || (cap->args[0] & ~1))
3195                         return -EINVAL;
3196                 kvm->manual_dirty_log_protect = cap->args[0];
3197                 return 0;
3198 #endif
3199         default:
3200                 return kvm_vm_ioctl_enable_cap(kvm, cap);
3201         }
3202 }
3203 
3204 static long kvm_vm_ioctl(struct file *filp,
3205                            unsigned int ioctl, unsigned long arg)
3206 {
3207         struct kvm *kvm = filp->private_data;
3208         void __user *argp = (void __user *)arg;
3209         int r;
3210 
3211         if (kvm->mm != current->mm)
3212                 return -EIO;
3213         switch (ioctl) {
3214         case KVM_CREATE_VCPU:
3215                 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
3216                 break;
3217         case KVM_ENABLE_CAP: {
3218                 struct kvm_enable_cap cap;
3219 
3220                 r = -EFAULT;
3221                 if (copy_from_user(&cap, argp, sizeof(cap)))
3222                         goto out;
3223                 r = kvm_vm_ioctl_enable_cap_generic(kvm, &cap);
3224                 break;
3225         }
3226         case KVM_SET_USER_MEMORY_REGION: {
3227                 struct kvm_userspace_memory_region kvm_userspace_mem;
3228 
3229                 r = -EFAULT;
3230                 if (copy_from_user(&kvm_userspace_mem, argp,
3231                                                 sizeof(kvm_userspace_mem)))
3232                         goto out;
3233 
3234                 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
3235                 break;
3236         }
3237         case KVM_GET_DIRTY_LOG: {
3238                 struct kvm_dirty_log log;
3239 
3240                 r = -EFAULT;
3241                 if (copy_from_user(&log, argp, sizeof(log)))
3242                         goto out;
3243                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
3244                 break;
3245         }
3246 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
3247         case KVM_CLEAR_DIRTY_LOG: {
3248                 struct kvm_clear_dirty_log log;
3249 
3250                 r = -EFAULT;
3251                 if (copy_from_user(&log, argp, sizeof(log)))
3252                         goto out;
3253                 r = kvm_vm_ioctl_clear_dirty_log(kvm, &log);
3254                 break;
3255         }
3256 #endif
3257 #ifdef CONFIG_KVM_MMIO
3258         case KVM_REGISTER_COALESCED_MMIO: {
3259                 struct kvm_coalesced_mmio_zone zone;
3260 
3261                 r = -EFAULT;
3262                 if (copy_from_user(&zone, argp, sizeof(zone)))
3263                         goto out;
3264                 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
3265                 break;
3266         }
3267         case KVM_UNREGISTER_COALESCED_MMIO: {
3268                 struct kvm_coalesced_mmio_zone zone;
3269 
3270                 r = -EFAULT;
3271                 if (copy_from_user(&zone, argp, sizeof(zone)))
3272                         goto out;
3273                 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
3274                 break;
3275         }
3276 #endif
3277         case KVM_IRQFD: {
3278                 struct kvm_irqfd data;
3279 
3280                 r = -EFAULT;
3281                 if (copy_from_user(&data, argp, sizeof(data)))
3282                         goto out;
3283                 r = kvm_irqfd(kvm, &data);
3284                 break;
3285         }
3286         case KVM_IOEVENTFD: {
3287                 struct kvm_ioeventfd data;
3288 
3289                 r = -EFAULT;
3290                 if (copy_from_user(&data, argp, sizeof(data)))
3291                         goto out;
3292                 r = kvm_ioeventfd(kvm, &data);
3293                 break;
3294         }
3295 #ifdef CONFIG_HAVE_KVM_MSI
3296         case KVM_SIGNAL_MSI: {
3297                 struct kvm_msi msi;
3298 
3299                 r = -EFAULT;
3300                 if (copy_from_user(&msi, argp, sizeof(msi)))
3301                         goto out;
3302                 r = kvm_send_userspace_msi(kvm, &msi);
3303                 break;
3304         }
3305 #endif
3306 #ifdef __KVM_HAVE_IRQ_LINE
3307         case KVM_IRQ_LINE_STATUS:
3308         case KVM_IRQ_LINE: {
3309                 struct kvm_irq_level irq_event;
3310 
3311                 r = -EFAULT;
3312                 if (copy_from_user(&irq_event, argp, sizeof(irq_event)))
3313                         goto out;
3314 
3315                 r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
3316                                         ioctl == KVM_IRQ_LINE_STATUS);
3317                 if (r)
3318                         goto out;
3319 
3320                 r = -EFAULT;
3321                 if (ioctl == KVM_IRQ_LINE_STATUS) {
3322                         if (copy_to_user(argp, &irq_event, sizeof(irq_event)))
3323                                 goto out;
3324                 }
3325 
3326                 r = 0;
3327                 break;
3328         }
3329 #endif
3330 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
3331         case KVM_SET_GSI_ROUTING: {
3332                 struct kvm_irq_routing routing;
3333                 struct kvm_irq_routing __user *urouting;
3334                 struct kvm_irq_routing_entry *entries = NULL;
3335 
3336                 r = -EFAULT;
3337                 if (copy_from_user(&routing, argp, sizeof(routing)))
3338                         goto out;
3339                 r = -EINVAL;
3340                 if (!kvm_arch_can_set_irq_routing(kvm))
3341                         goto out;
3342                 if (routing.nr > KVM_MAX_IRQ_ROUTES)
3343                         goto out;
3344                 if (routing.flags)
3345                         goto out;
3346                 if (routing.nr) {
3347                         r = -ENOMEM;
3348                         entries = vmalloc(array_size(sizeof(*entries),
3349                                                      routing.nr));
3350                         if (!entries)
3351                                 goto out;
3352                         r = -EFAULT;
3353                         urouting = argp;
3354                         if (copy_from_user(entries, urouting->entries,
3355                                            routing.nr * sizeof(*entries)))
3356                                 goto out_free_irq_routing;
3357                 }
3358                 r = kvm_set_irq_routing(kvm, entries, routing.nr,
3359                                         routing.flags);
3360 out_free_irq_routing:
3361                 vfree(entries);
3362                 break;
3363         }
3364 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
3365         case KVM_CREATE_DEVICE: {
3366                 struct kvm_create_device cd;
3367 
3368                 r = -EFAULT;
3369                 if (copy_from_user(&cd, argp, sizeof(cd)))
3370                         goto out;
3371 
3372                 r = kvm_ioctl_create_device(kvm, &cd);
3373                 if (r)
3374                         goto out;
3375 
3376                 r = -EFAULT;
3377                 if (copy_to_user(argp, &cd, sizeof(cd)))
3378                         goto out;
3379 
3380                 r = 0;
3381                 break;
3382         }
3383         case KVM_CHECK_EXTENSION:
3384                 r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
3385                 break;
3386         default:
3387                 r = kvm_arch_vm_ioctl(filp, ioctl, arg);
3388         }
3389 out:
3390         return r;
3391 }
3392 
3393 #ifdef CONFIG_KVM_COMPAT
3394 struct compat_kvm_dirty_log {
3395         __u32 slot;
3396         __u32 padding1;
3397         union {
3398                 compat_uptr_t dirty_bitmap; /* one bit per page */
3399                 __u64 padding2;
3400         };
3401 };
3402 
3403 static long kvm_vm_compat_ioctl(struct file *filp,
3404                            unsigned int ioctl, unsigned long arg)
3405 {
3406         struct kvm *kvm = filp->private_data;
3407         int r;
3408 
3409         if (kvm->mm != current->mm)
3410                 return -EIO;
3411         switch (ioctl) {
3412         case KVM_GET_DIRTY_LOG: {
3413                 struct compat_kvm_dirty_log compat_log;
3414                 struct kvm_dirty_log log;
3415 
3416                 if (copy_from_user(&compat_log, (void __user *)arg,
3417                                    sizeof(compat_log)))
3418                         return -EFAULT;
3419                 log.slot         = compat_log.slot;
3420                 log.padding1     = compat_log.padding1;
3421                 log.padding2     = compat_log.padding2;
3422                 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
3423 
3424                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
3425                 break;
3426         }
3427         default:
3428                 r = kvm_vm_ioctl(filp, ioctl, arg);
3429         }
3430         return r;
3431 }
3432 #endif
3433 
3434 static struct file_operations kvm_vm_fops = {
3435         .release        = kvm_vm_release,
3436         .unlocked_ioctl = kvm_vm_ioctl,
3437         .llseek         = noop_llseek,
3438         KVM_COMPAT(kvm_vm_compat_ioctl),
3439 };
3440 
3441 static int kvm_dev_ioctl_create_vm(unsigned long type)
3442 {
3443         int r;
3444         struct kvm *kvm;
3445         struct file *file;
3446 
3447         kvm = kvm_create_vm(type);
3448         if (IS_ERR(kvm))
3449                 return PTR_ERR(kvm);
3450 #ifdef CONFIG_KVM_MMIO
3451         r = kvm_coalesced_mmio_init(kvm);
3452         if (r < 0)
3453                 goto put_kvm;
3454 #endif
3455         r = get_unused_fd_flags(O_CLOEXEC);
3456         if (r < 0)
3457                 goto put_kvm;
3458 
3459         file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
3460         if (IS_ERR(file)) {
3461                 put_unused_fd(r);
3462                 r = PTR_ERR(file);
3463                 goto put_kvm;
3464         }
3465 
3466         /*
3467          * Don't call kvm_put_kvm anymore at this point; file->f_op is
3468          * already set, with ->release() being kvm_vm_release().  In error
3469          * cases it will be called by the final fput(file) and will take
3470          * care of doing kvm_put_kvm(kvm).
3471          */
3472         if (kvm_create_vm_debugfs(kvm, r) < 0) {
3473                 put_unused_fd(r);
3474                 fput(file);
3475                 return -ENOMEM;
3476         }
3477         kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm);
3478 
3479         fd_install(r, file);
3480         return r;
3481 
3482 put_kvm:
3483         kvm_put_kvm(kvm);
3484         return r;
3485 }
3486 
3487 static long kvm_dev_ioctl(struct file *filp,
3488                           unsigned int ioctl, unsigned long arg)
3489 {
3490         long r = -EINVAL;
3491 
3492         switch (ioctl) {
3493         case KVM_GET_API_VERSION:
3494                 if (arg)
3495                         goto out;
3496                 r = KVM_API_VERSION;
3497                 break;
3498         case KVM_CREATE_VM:
3499                 r = kvm_dev_ioctl_create_vm(arg);
3500                 break;
3501         case KVM_CHECK_EXTENSION:
3502                 r = kvm_vm_ioctl_check_extension_generic(NULL, arg);
3503                 break;
3504         case KVM_GET_VCPU_MMAP_SIZE:
3505                 if (arg)
3506                         goto out;
3507                 r = PAGE_SIZE;     /* struct kvm_run */
3508 #ifdef CONFIG_X86
3509                 r += PAGE_SIZE;    /* pio data page */
3510 #endif
3511 #ifdef CONFIG_KVM_MMIO
3512                 r += PAGE_SIZE;    /* coalesced mmio ring page */
3513 #endif
3514                 break;
3515         case KVM_TRACE_ENABLE:
3516         case KVM_TRACE_PAUSE:
3517         case KVM_TRACE_DISABLE:
3518                 r = -EOPNOTSUPP;
3519                 break;
3520         default:
3521                 return kvm_arch_dev_ioctl(filp, ioctl, arg);
3522         }
3523 out:
3524         return r;
3525 }
3526 
3527 static struct file_operations kvm_chardev_ops = {
3528         .unlocked_ioctl = kvm_dev_ioctl,
3529         .llseek         = noop_llseek,
3530         KVM_COMPAT(kvm_dev_ioctl),
3531 };
3532 
3533 static struct miscdevice kvm_dev = {
3534         KVM_MINOR,
3535         "kvm",
3536         &kvm_chardev_ops,
3537 };
3538 
3539 static void hardware_enable_nolock(void *junk)
3540 {
3541         int cpu = raw_smp_processor_id();
3542         int r;
3543 
3544         if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
3545                 return;
3546 
3547         cpumask_set_cpu(cpu, cpus_hardware_enabled);
3548 
3549         r = kvm_arch_hardware_enable();
3550 
3551         if (r) {
3552                 cpumask_clear_cpu(cpu, cpus_hardware_enabled);
3553                 atomic_inc(&hardware_enable_failed);
3554                 pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu);
3555         }
3556 }
3557 
3558 static int kvm_starting_cpu(unsigned int cpu)
3559 {
3560         raw_spin_lock(&kvm_count_lock);
3561         if (kvm_usage_count)
3562                 hardware_enable_nolock(NULL);
3563         raw_spin_unlock(&kvm_count_lock);
3564         return 0;
3565 }
3566 
3567 static void hardware_disable_nolock(void *junk)
3568 {
3569         int cpu = raw_smp_processor_id();
3570 
3571         if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
3572                 return;
3573         cpumask_clear_cpu(cpu, cpus_hardware_enabled);
3574         kvm_arch_hardware_disable();
3575 }
3576 
3577 static int kvm_dying_cpu(unsigned int cpu)
3578 {
3579         raw_spin_lock(&kvm_count_lock);
3580         if (kvm_usage_count)
3581                 hardware_disable_nolock(NULL);
3582         raw_spin_unlock(&kvm_count_lock);
3583         return 0;
3584 }
3585 
3586 static void hardware_disable_all_nolock(void)
3587 {
3588         BUG_ON(!kvm_usage_count);
3589 
3590         kvm_usage_count--;
3591         if (!kvm_usage_count)
3592                 on_each_cpu(hardware_disable_nolock, NULL, 1);
3593 }
3594 
3595 static void hardware_disable_all(void)
3596 {
3597         raw_spin_lock(&kvm_count_lock);
3598         hardware_disable_all_nolock();
3599         raw_spin_unlock(&kvm_count_lock);
3600 }
3601 
3602 static int hardware_enable_all(void)
3603 {
3604         int r = 0;
3605 
3606         raw_spin_lock(&kvm_count_lock);
3607 
3608         kvm_usage_count++;
3609         if (kvm_usage_count == 1) {
3610                 atomic_set(&hardware_enable_failed, 0);
3611                 on_each_cpu(hardware_enable_nolock, NULL, 1);
3612 
3613                 if (atomic_read(&hardware_enable_failed)) {
3614                         hardware_disable_all_nolock();
3615                         r = -EBUSY;
3616                 }
3617         }
3618 
3619         raw_spin_unlock(&kvm_count_lock);
3620 
3621         return r;
3622 }
3623 
3624 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
3625                       void *v)
3626 {
3627         /*
3628          * Some (well, at least mine) BIOSes hang on reboot if
3629          * in vmx root mode.
3630          *
3631          * And Intel TXT required VMX off for all cpu when system shutdown.
3632          */
3633         pr_info("kvm: exiting hardware virtualization\n");
3634         kvm_rebooting = true;
3635         on_each_cpu(hardware_disable_nolock, NULL, 1);
3636         return NOTIFY_OK;
3637 }
3638 
3639 static struct notifier_block kvm_reboot_notifier = {
3640         .notifier_call = kvm_reboot,
3641         .priority = 0,
3642 };
3643 
3644 static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
3645 {
3646         int i;
3647 
3648         for (i = 0; i < bus->dev_count; i++) {
3649                 struct kvm_io_device *pos = bus->range[i].dev;
3650 
3651                 kvm_iodevice_destructor(pos);
3652         }
3653         kfree(bus);
3654 }
3655 
3656 static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
3657                                  const struct kvm_io_range *r2)
3658 {
3659         gpa_t addr1 = r1->addr;
3660         gpa_t addr2 = r2->addr;
3661 
3662         if (addr1 < addr2)
3663                 return -1;
3664 
3665         /* If r2->len == 0, match the exact address.  If r2->len != 0,
3666          * accept any overlapping write.  Any order is acceptable for
3667          * overlapping ranges, because kvm_io_bus_get_first_dev ensures
3668          * we process all of them.
3669          */
3670         if (r2->len) {
3671                 addr1 += r1->len;
3672                 addr2 += r2->len;
3673         }
3674 
3675         if (addr1 > addr2)
3676                 return 1;
3677 
3678         return 0;
3679 }
3680 
3681 static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
3682 {
3683         return kvm_io_bus_cmp(p1, p2);
3684 }
3685 
3686 static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
3687                              gpa_t addr, int len)
3688 {
3689         struct kvm_io_range *range, key;
3690         int off;
3691 
3692         key = (struct kvm_io_range) {
3693                 .addr = addr,
3694                 .len = len,
3695         };
3696 
3697         range = bsearch(&key, bus->range, bus->dev_count,
3698                         sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
3699         if (range == NULL)
3700                 return -ENOENT;
3701 
3702         off = range - bus->range;
3703 
3704         while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0)
3705                 off--;
3706 
3707         return off;
3708 }
3709 
3710 static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
3711                               struct kvm_io_range *range, const void *val)
3712 {
3713         int idx;
3714 
3715         idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
3716         if (idx < 0)
3717                 return -EOPNOTSUPP;
3718 
3719         while (idx < bus->dev_count &&
3720                 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
3721                 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr,
3722                                         range->len, val))
3723                         return idx;
3724                 idx++;
3725         }
3726 
3727         return -EOPNOTSUPP;
3728 }
3729 
3730 /* kvm_io_bus_write - called under kvm->slots_lock */
3731 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
3732                      int len, const void *val)
3733 {
3734         struct kvm_io_bus *bus;
3735         struct kvm_io_range range;
3736         int r;
3737 
3738         range = (struct kvm_io_range) {
3739                 .addr = addr,
3740                 .len = len,
3741         };
3742 
3743         bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3744         if (!bus)
3745                 return -ENOMEM;
3746         r = __kvm_io_bus_write(vcpu, bus, &range, val);
3747         return r < 0 ? r : 0;
3748 }
3749 EXPORT_SYMBOL_GPL(kvm_io_bus_write);
3750 
3751 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */
3752 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
3753                             gpa_t addr, int len, const void *val, long cookie)
3754 {
3755         struct kvm_io_bus *bus;
3756         struct kvm_io_range range;
3757 
3758         range = (struct kvm_io_range) {
3759                 .addr = addr,
3760                 .len = len,
3761         };
3762 
3763         bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3764         if (!bus)
3765                 return -ENOMEM;
3766 
3767         /* First try the device referenced by cookie. */
3768         if ((cookie >= 0) && (cookie < bus->dev_count) &&
3769             (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
3770                 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len,
3771                                         val))
3772                         return cookie;
3773 
3774         /*
3775          * cookie contained garbage; fall back to search and return the
3776          * correct cookie value.
3777          */
3778         return __kvm_io_bus_write(vcpu, bus, &range, val);
3779 }
3780 
3781 static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
3782                              struct kvm_io_range *range, void *val)
3783 {
3784         int idx;
3785 
3786         idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
3787         if (idx < 0)
3788                 return -EOPNOTSUPP;
3789 
3790         while (idx < bus->dev_count &&
3791                 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
3792                 if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr,
3793                                        range->len, val))
3794                         return idx;
3795                 idx++;
3796         }
3797 
3798         return -EOPNOTSUPP;
3799 }
3800 
3801 /* kvm_io_bus_read - called under kvm->slots_lock */
3802 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
3803                     int len, void *val)
3804 {
3805         struct kvm_io_bus *bus;
3806         struct kvm_io_range range;
3807         int r;
3808 
3809         range = (struct kvm_io_range) {
3810                 .addr = addr,
3811                 .len = len,
3812         };
3813 
3814         bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3815         if (!bus)
3816                 return -ENOMEM;
3817         r = __kvm_io_bus_read(vcpu, bus, &range, val);
3818         return r < 0 ? r : 0;
3819 }
3820 
3821 /* Caller must hold slots_lock. */
3822 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
3823                             int len, struct kvm_io_device *dev)
3824 {
3825         int i;
3826         struct kvm_io_bus *new_bus, *bus;
3827         struct kvm_io_range range;
3828 
3829         bus = kvm_get_bus(kvm, bus_idx);
3830         if (!bus)
3831                 return -ENOMEM;
3832 
3833         /* exclude ioeventfd which is limited by maximum fd */
3834         if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1)
3835                 return -ENOSPC;
3836 
3837         new_bus = kmalloc(struct_size(bus, range, bus->dev_count + 1),
3838                           GFP_KERNEL_ACCOUNT);
3839         if (!new_bus)
3840                 return -ENOMEM;
3841 
3842         range = (struct kvm_io_range) {
3843                 .addr = addr,
3844                 .len = len,
3845                 .dev = dev,
3846         };
3847 
3848         for (i = 0; i < bus->dev_count; i++)
3849                 if (kvm_io_bus_cmp(&bus->range[i], &range) > 0)
3850                         break;
3851 
3852         memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
3853         new_bus->dev_count++;
3854         new_bus->range[i] = range;
3855         memcpy(new_bus->range + i + 1, bus->range + i,
3856                 (bus->dev_count - i) * sizeof(struct kvm_io_range));
3857         rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
3858         synchronize_srcu_expedited(&kvm->srcu);
3859         kfree(bus);
3860 
3861         return 0;
3862 }
3863 
3864 /* Caller must hold slots_lock. */
3865 void kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
3866                                struct kvm_io_device *dev)
3867 {
3868         int i;
3869         struct kvm_io_bus *new_bus, *bus;
3870 
3871         bus = kvm_get_bus(kvm, bus_idx);
3872         if (!bus)
3873                 return;
3874 
3875         for (i = 0; i < bus->dev_count; i++)
3876                 if (bus->range[i].dev == dev) {
3877                         break;
3878                 }
3879 
3880         if (i == bus->dev_count)
3881                 return;
3882 
3883         new_bus = kmalloc(struct_size(bus, range, bus->dev_count - 1),
3884                           GFP_KERNEL_ACCOUNT);
3885         if (!new_bus)  {
3886                 pr_err("kvm: failed to shrink bus, removing it completely\n");
3887                 goto broken;
3888         }
3889 
3890         memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
3891         new_bus->dev_count--;
3892         memcpy(new_bus->range + i, bus->range + i + 1,
3893                (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
3894 
3895 broken:
3896         rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
3897         synchronize_srcu_expedited(&kvm->srcu);
3898         kfree(bus);
3899         return;
3900 }
3901 
3902 struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
3903                                          gpa_t addr)
3904 {
3905         struct kvm_io_bus *bus;
3906         int dev_idx, srcu_idx;
3907         struct kvm_io_device *iodev = NULL;
3908 
3909         srcu_idx = srcu_read_lock(&kvm->srcu);
3910 
3911         bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
3912         if (!bus)
3913                 goto out_unlock;
3914 
3915         dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1);
3916         if (dev_idx < 0)
3917                 goto out_unlock;
3918 
3919         iodev = bus->range[dev_idx].dev;
3920 
3921 out_unlock:
3922         srcu_read_unlock(&kvm->srcu, srcu_idx);
3923 
3924         return iodev;
3925 }
3926 EXPORT_SYMBOL_GPL(kvm_io_bus_get_dev);
3927 
3928 static int kvm_debugfs_open(struct inode *inode, struct file *file,
3929                            int (*get)(void *, u64 *), int (*set)(void *, u64),
3930                            const char *fmt)
3931 {
3932         struct kvm_stat_data *stat_data = (struct kvm_stat_data *)
3933                                           inode->i_private;
3934 
3935         /* The debugfs files are a reference to the kvm struct which
3936          * is still valid when kvm_destroy_vm is called.
3937          * To avoid the race between open and the removal of the debugfs
3938          * directory we test against the users count.
3939          */
3940         if (!refcount_inc_not_zero(&stat_data->kvm->users_count))
3941                 return -ENOENT;
3942 
3943         if (simple_attr_open(inode, file, get, set, fmt)) {
3944                 kvm_put_kvm(stat_data->kvm);
3945                 return -ENOMEM;
3946         }
3947 
3948         return 0;
3949 }
3950 
3951 static int kvm_debugfs_release(struct inode *inode, struct file *file)
3952 {
3953         struct kvm_stat_data *stat_data = (struct kvm_stat_data *)
3954                                           inode->i_private;
3955 
3956         simple_attr_release(inode, file);
3957         kvm_put_kvm(stat_data->kvm);
3958 
3959         return 0;
3960 }
3961 
3962 static int vm_stat_get_per_vm(void *data, u64 *val)
3963 {
3964         struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
3965 
3966         *val = *(ulong *)((void *)stat_data->kvm + stat_data->offset);
3967 
3968         return 0;
3969 }
3970 
3971 static int vm_stat_clear_per_vm(void *data, u64 val)
3972 {
3973         struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
3974 
3975         if (val)
3976                 return -EINVAL;
3977 
3978         *(ulong *)((void *)stat_data->kvm + stat_data->offset) = 0;
3979 
3980         return 0;
3981 }
3982 
3983 static int vm_stat_get_per_vm_open(struct inode *inode, struct file *file)
3984 {
3985         __simple_attr_check_format("%llu\n", 0ull);
3986         return kvm_debugfs_open(inode, file, vm_stat_get_per_vm,
3987                                 vm_stat_clear_per_vm, "%llu\n");
3988 }
3989 
3990 static const struct file_operations vm_stat_get_per_vm_fops = {
3991         .owner   = THIS_MODULE,
3992         .open    = vm_stat_get_per_vm_open,
3993         .release = kvm_debugfs_release,
3994         .read    = simple_attr_read,
3995         .write   = simple_attr_write,
3996         .llseek  = no_llseek,
3997 };
3998 
3999 static int vcpu_stat_get_per_vm(void *data, u64 *val)
4000 {
4001         int i;
4002         struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
4003         struct kvm_vcpu *vcpu;
4004 
4005         *val = 0;
4006 
4007         kvm_for_each_vcpu(i, vcpu, stat_data->kvm)
4008                 *val += *(u64 *)((void *)vcpu + stat_data->offset);
4009 
4010         return 0;
4011 }
4012 
4013 static int vcpu_stat_clear_per_vm(void *data, u64 val)
4014 {
4015         int i;
4016         struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
4017         struct kvm_vcpu *vcpu;
4018 
4019         if (val)
4020                 return -EINVAL;
4021 
4022         kvm_for_each_vcpu(i, vcpu, stat_data->kvm)
4023                 *(u64 *)((void *)vcpu + stat_data->offset) = 0;
4024 
4025         return 0;
4026 }
4027 
4028 static int vcpu_stat_get_per_vm_open(struct inode *inode, struct file *file)
4029 {
4030         __simple_attr_check_format("%llu\n", 0ull);
4031         return kvm_debugfs_open(inode, file, vcpu_stat_get_per_vm,
4032                                  vcpu_stat_clear_per_vm, "%llu\n");
4033 }
4034 
4035 static const struct file_operations vcpu_stat_get_per_vm_fops = {
4036         .owner   = THIS_MODULE,
4037         .open    = vcpu_stat_get_per_vm_open,
4038         .release = kvm_debugfs_release,
4039         .read    = simple_attr_read,
4040         .write   = simple_attr_write,
4041         .llseek  = no_llseek,
4042 };
4043 
4044 static const struct file_operations *stat_fops_per_vm[] = {
4045         [KVM_STAT_VCPU] = &vcpu_stat_get_per_vm_fops,
4046         [KVM_STAT_VM]   = &vm_stat_get_per_vm_fops,
4047 };
4048 
4049 static int vm_stat_get(void *_offset, u64 *val)
4050 {
4051         unsigned offset = (long)_offset;
4052         struct kvm *kvm;
4053         struct kvm_stat_data stat_tmp = {.offset = offset};
4054         u64 tmp_val;
4055 
4056         *val = 0;
4057         spin_lock(&kvm_lock);
4058         list_for_each_entry(kvm, &vm_list, vm_list) {
4059                 stat_tmp.kvm = kvm;
4060                 vm_stat_get_per_vm((void *)&stat_tmp, &tmp_val);
4061                 *val += tmp_val;
4062         }
4063         spin_unlock(&kvm_lock);
4064         return 0;
4065 }
4066 
4067 static int vm_stat_clear(void *_offset, u64 val)
4068 {
4069         unsigned offset = (long)_offset;
4070         struct kvm *kvm;
4071         struct kvm_stat_data stat_tmp = {.offset = offset};
4072 
4073         if (val)
4074                 return -EINVAL;
4075 
4076         spin_lock(&kvm_lock);
4077         list_for_each_entry(kvm, &vm_list, vm_list) {
4078                 stat_tmp.kvm = kvm;
4079                 vm_stat_clear_per_vm((void *)&stat_tmp, 0);
4080         }
4081         spin_unlock(&kvm_lock);
4082 
4083         return 0;
4084 }
4085 
4086 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, vm_stat_clear, "%llu\n");
4087 
4088 static int vcpu_stat_get(void *_offset, u64 *val)
4089 {
4090         unsigned offset = (long)_offset;
4091         struct kvm *kvm;
4092         struct kvm_stat_data stat_tmp = {.offset = offset};
4093         u64 tmp_val;
4094 
4095         *val = 0;
4096         spin_lock(&kvm_lock);
4097         list_for_each_entry(kvm, &vm_list, vm_list) {
4098                 stat_tmp.kvm = kvm;
4099                 vcpu_stat_get_per_vm((void *)&stat_tmp, &tmp_val);
4100                 *val += tmp_val;
4101         }
4102         spin_unlock(&kvm_lock);
4103         return 0;
4104 }
4105 
4106 static int vcpu_stat_clear(void *_offset, u64 val)
4107 {
4108         unsigned offset = (long)_offset;
4109         struct kvm *kvm;
4110         struct kvm_stat_data stat_tmp = {.offset = offset};
4111 
4112         if (val)
4113                 return -EINVAL;
4114 
4115         spin_lock(&kvm_lock);
4116         list_for_each_entry(kvm, &vm_list, vm_list) {
4117                 stat_tmp.kvm = kvm;
4118                 vcpu_stat_clear_per_vm((void *)&stat_tmp, 0);
4119         }
4120         spin_unlock(&kvm_lock);
4121 
4122         return 0;
4123 }
4124 
4125 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear,
4126                         "%llu\n");
4127 
4128 static const struct file_operations *stat_fops[] = {
4129         [KVM_STAT_VCPU] = &vcpu_stat_fops,
4130         [KVM_STAT_VM]   = &vm_stat_fops,
4131 };
4132 
4133 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm)
4134 {
4135         struct kobj_uevent_env *env;
4136         unsigned long long created, active;
4137 
4138         if (!kvm_dev.this_device || !kvm)
4139                 return;
4140 
4141         spin_lock(&kvm_lock);
4142         if (type == KVM_EVENT_CREATE_VM) {
4143                 kvm_createvm_count++;
4144                 kvm_active_vms++;
4145         } else if (type == KVM_EVENT_DESTROY_VM) {
4146                 kvm_active_vms--;
4147         }
4148         created = kvm_createvm_count;
4149         active = kvm_active_vms;
4150         spin_unlock(&kvm_lock);
4151 
4152         env = kzalloc(sizeof(*env), GFP_KERNEL_ACCOUNT);
4153         if (!env)
4154                 return;
4155 
4156         add_uevent_var(env, "CREATED=%llu", created);
4157         add_uevent_var(env, "COUNT=%llu", active);
4158 
4159         if (type == KVM_EVENT_CREATE_VM) {
4160                 add_uevent_var(env, "EVENT=create");
4161                 kvm->userspace_pid = task_pid_nr(current);
4162         } else if (type == KVM_EVENT_DESTROY_VM) {
4163                 add_uevent_var(env, "EVENT=destroy");
4164         }
4165         add_uevent_var(env, "PID=%d", kvm->userspace_pid);
4166 
4167         if (!IS_ERR_OR_NULL(kvm->debugfs_dentry)) {
4168                 char *tmp, *p = kmalloc(PATH_MAX, GFP_KERNEL_ACCOUNT);
4169 
4170                 if (p) {
4171                         tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX);
4172                         if (!IS_ERR(tmp))
4173                                 add_uevent_var(env, "STATS_PATH=%s", tmp);
4174                         kfree(p);
4175                 }
4176         }
4177         /* no need for checks, since we are adding at most only 5 keys */
4178         env->envp[env->envp_idx++] = NULL;
4179         kobject_uevent_env(&kvm_dev.this_device->kobj, KOBJ_CHANGE, env->envp);
4180         kfree(env);
4181 }
4182 
4183 static void kvm_init_debug(void)
4184 {
4185         struct kvm_stats_debugfs_item *p;
4186 
4187         kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
4188 
4189         kvm_debugfs_num_entries = 0;
4190         for (p = debugfs_entries; p->name; ++p, kvm_debugfs_num_entries++) {
4191                 debugfs_create_file(p->name, 0644, kvm_debugfs_dir,
4192                                     (void *)(long)p->offset,
4193                                     stat_fops[p->kind]);
4194         }
4195 }
4196 
4197 static int kvm_suspend(void)
4198 {
4199         if (kvm_usage_count)
4200                 hardware_disable_nolock(NULL);
4201         return 0;
4202 }
4203 
4204 static void kvm_resume(void)
4205 {
4206         if (kvm_usage_count) {
4207 #ifdef CONFIG_LOCKDEP
4208                 WARN_ON(lockdep_is_held(&kvm_count_lock));
4209 #endif
4210                 hardware_enable_nolock(NULL);
4211         }
4212 }
4213 
4214 static struct syscore_ops kvm_syscore_ops = {
4215         .suspend = kvm_suspend,
4216         .resume = kvm_resume,
4217 };
4218 
4219 static inline
4220 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
4221 {
4222         return container_of(pn, struct kvm_vcpu, preempt_notifier);
4223 }
4224 
4225 static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
4226 {
4227         struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
4228 
4229         if (vcpu->preempted)
4230                 vcpu->preempted = false;
4231 
4232         kvm_arch_sched_in(vcpu, cpu);
4233 
4234         kvm_arch_vcpu_load(vcpu, cpu);
4235 }
4236 
4237 static void kvm_sched_out(struct preempt_notifier *pn,
4238                           struct task_struct *next)
4239 {
4240         struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
4241 
4242         if (current->state == TASK_RUNNING)
4243                 vcpu->preempted = true;
4244         kvm_arch_vcpu_put(vcpu);
4245 }
4246 
4247 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
4248                   struct module *module)
4249 {
4250         int r;
4251         int cpu;
4252 
4253         r = kvm_arch_init(opaque);
4254         if (r)
4255                 goto out_fail;
4256 
4257         /*
4258          * kvm_arch_init makes sure there's at most one caller
4259          * for architectures that support multiple implementations,
4260          * like intel and amd on x86.
4261          * kvm_arch_init must be called before kvm_irqfd_init to avoid creating
4262          * conflicts in case kvm is already setup for another implementation.
4263          */
4264         r = kvm_irqfd_init();
4265         if (r)
4266                 goto out_irqfd;
4267 
4268         if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
4269                 r = -ENOMEM;
4270                 goto out_free_0;
4271         }
4272 
4273         r = kvm_arch_hardware_setup();
4274         if (r < 0)
4275                 goto out_free_0a;
4276 
4277         for_each_online_cpu(cpu) {
4278                 smp_call_function_single(cpu,
4279                                 kvm_arch_check_processor_compat,
4280                                 &r, 1);
4281                 if (r < 0)
4282                         goto out_free_1;
4283         }
4284 
4285         r = cpuhp_setup_state_nocalls(CPUHP_AP_KVM_STARTING, "kvm/cpu:starting",
4286                                       kvm_starting_cpu, kvm_dying_cpu);
4287         if (r)
4288                 goto out_free_2;
4289         register_reboot_notifier(&kvm_reboot_notifier);
4290 
4291         /* A kmem cache lets us meet the alignment requirements of fx_save. */
4292         if (!vcpu_align)
4293                 vcpu_align = __alignof__(struct kvm_vcpu);
4294         kvm_vcpu_cache =
4295                 kmem_cache_create_usercopy("kvm_vcpu", vcpu_size, vcpu_align,
4296                                            SLAB_ACCOUNT,
4297                                            offsetof(struct kvm_vcpu, arch),
4298                                            sizeof_field(struct kvm_vcpu, arch),
4299                                            NULL);
4300         if (!kvm_vcpu_cache) {
4301                 r = -ENOMEM;
4302                 goto out_free_3;
4303         }
4304 
4305         r = kvm_async_pf_init();
4306         if (r)
4307                 goto out_free;
4308 
4309         kvm_chardev_ops.owner = module;
4310         kvm_vm_fops.owner = module;
4311         kvm_vcpu_fops.owner = module;
4312 
4313         r = misc_register(&kvm_dev);
4314         if (r) {
4315                 pr_err("kvm: misc device register failed\n");
4316                 goto out_unreg;
4317         }
4318 
4319         register_syscore_ops(&kvm_syscore_ops);
4320 
4321         kvm_preempt_ops.sched_in = kvm_sched_in;
4322         kvm_preempt_ops.sched_out = kvm_sched_out;
4323 
4324         kvm_init_debug();
4325 
4326         r = kvm_vfio_ops_init();
4327         WARN_ON(r);
4328 
4329         return 0;
4330 
4331 out_unreg:
4332         kvm_async_pf_deinit();
4333 out_free:
4334         kmem_cache_destroy(kvm_vcpu_cache);
4335 out_free_3:
4336         unregister_reboot_notifier(&kvm_reboot_notifier);
4337         cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
4338 out_free_2:
4339 out_free_1:
4340         kvm_arch_hardware_unsetup();
4341 out_free_0a:
4342         free_cpumask_var(cpus_hardware_enabled);
4343 out_free_0:
4344         kvm_irqfd_exit();
4345 out_irqfd:
4346         kvm_arch_exit();
4347 out_fail:
4348         return r;
4349 }
4350 EXPORT_SYMBOL_GPL(kvm_init);
4351 
4352 void kvm_exit(void)
4353 {
4354         debugfs_remove_recursive(kvm_debugfs_dir);
4355         misc_deregister(&kvm_dev);
4356         kmem_cache_destroy(kvm_vcpu_cache);
4357         kvm_async_pf_deinit();
4358         unregister_syscore_ops(&kvm_syscore_ops);
4359         unregister_reboot_notifier(&kvm_reboot_notifier);
4360         cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
4361         on_each_cpu(hardware_disable_nolock, NULL, 1);
4362         kvm_arch_hardware_unsetup();
4363         kvm_arch_exit();
4364         kvm_irqfd_exit();
4365         free_cpumask_var(cpus_hardware_enabled);
4366         kvm_vfio_ops_exit();
4367 }
4368 EXPORT_SYMBOL_GPL(kvm_exit);
4369 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp