~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm/kvm/arm.c

Version: ~ [ linux-5.3 ] ~ [ linux-5.2.15 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.73 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.144 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.193 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.193 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.73 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
  3  * Author: Christoffer Dall <c.dall@virtualopensystems.com>
  4  *
  5  * This program is free software; you can redistribute it and/or modify
  6  * it under the terms of the GNU General Public License, version 2, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This program is distributed in the hope that it will be useful,
 10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 12  * GNU General Public License for more details.
 13  *
 14  * You should have received a copy of the GNU General Public License
 15  * along with this program; if not, write to the Free Software
 16  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 17  */
 18 
 19 #include <linux/cpu.h>
 20 #include <linux/cpu_pm.h>
 21 #include <linux/errno.h>
 22 #include <linux/err.h>
 23 #include <linux/kvm_host.h>
 24 #include <linux/module.h>
 25 #include <linux/vmalloc.h>
 26 #include <linux/fs.h>
 27 #include <linux/mman.h>
 28 #include <linux/sched.h>
 29 #include <linux/kvm.h>
 30 #include <trace/events/kvm.h>
 31 
 32 #define CREATE_TRACE_POINTS
 33 #include "trace.h"
 34 
 35 #include <asm/uaccess.h>
 36 #include <asm/ptrace.h>
 37 #include <asm/mman.h>
 38 #include <asm/tlbflush.h>
 39 #include <asm/cacheflush.h>
 40 #include <asm/virt.h>
 41 #include <asm/kvm_arm.h>
 42 #include <asm/kvm_asm.h>
 43 #include <asm/kvm_mmu.h>
 44 #include <asm/kvm_emulate.h>
 45 #include <asm/kvm_coproc.h>
 46 #include <asm/kvm_psci.h>
 47 
 48 #ifdef REQUIRES_VIRT
 49 __asm__(".arch_extension        virt");
 50 #endif
 51 
 52 static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
 53 static kvm_cpu_context_t __percpu *kvm_host_cpu_state;
 54 static unsigned long hyp_default_vectors;
 55 
 56 /* Per-CPU variable containing the currently running vcpu. */
 57 static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_arm_running_vcpu);
 58 
 59 /* The VMID used in the VTTBR */
 60 static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1);
 61 static u8 kvm_next_vmid;
 62 static DEFINE_SPINLOCK(kvm_vmid_lock);
 63 
 64 static void kvm_arm_set_running_vcpu(struct kvm_vcpu *vcpu)
 65 {
 66         BUG_ON(preemptible());
 67         __this_cpu_write(kvm_arm_running_vcpu, vcpu);
 68 }
 69 
 70 /**
 71  * kvm_arm_get_running_vcpu - get the vcpu running on the current CPU.
 72  * Must be called from non-preemptible context
 73  */
 74 struct kvm_vcpu *kvm_arm_get_running_vcpu(void)
 75 {
 76         BUG_ON(preemptible());
 77         return __this_cpu_read(kvm_arm_running_vcpu);
 78 }
 79 
 80 /**
 81  * kvm_arm_get_running_vcpus - get the per-CPU array of currently running vcpus.
 82  */
 83 struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void)
 84 {
 85         return &kvm_arm_running_vcpu;
 86 }
 87 
 88 int kvm_arch_hardware_enable(void)
 89 {
 90         return 0;
 91 }
 92 
 93 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
 94 {
 95         return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
 96 }
 97 
 98 int kvm_arch_hardware_setup(void)
 99 {
100         return 0;
101 }
102 
103 void kvm_arch_check_processor_compat(void *rtn)
104 {
105         *(int *)rtn = 0;
106 }
107 
108 
109 /**
110  * kvm_arch_init_vm - initializes a VM data structure
111  * @kvm:        pointer to the KVM struct
112  */
113 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
114 {
115         int ret = 0;
116 
117         if (type)
118                 return -EINVAL;
119 
120         ret = kvm_alloc_stage2_pgd(kvm);
121         if (ret)
122                 goto out_fail_alloc;
123 
124         ret = create_hyp_mappings(kvm, kvm + 1);
125         if (ret)
126                 goto out_free_stage2_pgd;
127 
128         kvm_vgic_early_init(kvm);
129         kvm_timer_init(kvm);
130 
131         /* Mark the initial VMID generation invalid */
132         kvm->arch.vmid_gen = 0;
133 
134         /* The maximum number of VCPUs is limited by the host's GIC model */
135         kvm->arch.max_vcpus = kvm_vgic_get_max_vcpus();
136 
137         return ret;
138 out_free_stage2_pgd:
139         kvm_free_stage2_pgd(kvm);
140 out_fail_alloc:
141         return ret;
142 }
143 
144 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
145 {
146         return VM_FAULT_SIGBUS;
147 }
148 
149 
150 /**
151  * kvm_arch_destroy_vm - destroy the VM data structure
152  * @kvm:        pointer to the KVM struct
153  */
154 void kvm_arch_destroy_vm(struct kvm *kvm)
155 {
156         int i;
157 
158         kvm_free_stage2_pgd(kvm);
159 
160         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
161                 if (kvm->vcpus[i]) {
162                         kvm_arch_vcpu_free(kvm->vcpus[i]);
163                         kvm->vcpus[i] = NULL;
164                 }
165         }
166 
167         kvm_vgic_destroy(kvm);
168 }
169 
170 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
171 {
172         int r;
173         switch (ext) {
174         case KVM_CAP_IRQCHIP:
175         case KVM_CAP_IOEVENTFD:
176         case KVM_CAP_DEVICE_CTRL:
177         case KVM_CAP_USER_MEMORY:
178         case KVM_CAP_SYNC_MMU:
179         case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
180         case KVM_CAP_ONE_REG:
181         case KVM_CAP_ARM_PSCI:
182         case KVM_CAP_ARM_PSCI_0_2:
183         case KVM_CAP_READONLY_MEM:
184         case KVM_CAP_MP_STATE:
185                 r = 1;
186                 break;
187         case KVM_CAP_COALESCED_MMIO:
188                 r = KVM_COALESCED_MMIO_PAGE_OFFSET;
189                 break;
190         case KVM_CAP_ARM_SET_DEVICE_ADDR:
191                 r = 1;
192                 break;
193         case KVM_CAP_NR_VCPUS:
194                 r = num_online_cpus();
195                 break;
196         case KVM_CAP_MAX_VCPUS:
197                 r = KVM_MAX_VCPUS;
198                 break;
199         default:
200                 r = kvm_arch_dev_ioctl_check_extension(ext);
201                 break;
202         }
203         return r;
204 }
205 
206 long kvm_arch_dev_ioctl(struct file *filp,
207                         unsigned int ioctl, unsigned long arg)
208 {
209         return -EINVAL;
210 }
211 
212 
213 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id)
214 {
215         int err;
216         struct kvm_vcpu *vcpu;
217 
218         if (irqchip_in_kernel(kvm) && vgic_initialized(kvm)) {
219                 err = -EBUSY;
220                 goto out;
221         }
222 
223         if (id >= kvm->arch.max_vcpus) {
224                 err = -EINVAL;
225                 goto out;
226         }
227 
228         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
229         if (!vcpu) {
230                 err = -ENOMEM;
231                 goto out;
232         }
233 
234         err = kvm_vcpu_init(vcpu, kvm, id);
235         if (err)
236                 goto free_vcpu;
237 
238         err = create_hyp_mappings(vcpu, vcpu + 1);
239         if (err)
240                 goto vcpu_uninit;
241 
242         return vcpu;
243 vcpu_uninit:
244         kvm_vcpu_uninit(vcpu);
245 free_vcpu:
246         kmem_cache_free(kvm_vcpu_cache, vcpu);
247 out:
248         return ERR_PTR(err);
249 }
250 
251 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
252 {
253         kvm_vgic_vcpu_early_init(vcpu);
254 }
255 
256 void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
257 {
258         kvm_mmu_free_memory_caches(vcpu);
259         kvm_timer_vcpu_terminate(vcpu);
260         kvm_vgic_vcpu_destroy(vcpu);
261         kmem_cache_free(kvm_vcpu_cache, vcpu);
262 }
263 
264 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
265 {
266         kvm_arch_vcpu_free(vcpu);
267 }
268 
269 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
270 {
271         return kvm_timer_should_fire(vcpu);
272 }
273 
274 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
275 {
276         /* Force users to call KVM_ARM_VCPU_INIT */
277         vcpu->arch.target = -1;
278         bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
279 
280         /* Set up the timer */
281         kvm_timer_vcpu_init(vcpu);
282 
283         kvm_arm_reset_debug_ptr(vcpu);
284 
285         return 0;
286 }
287 
288 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
289 {
290         vcpu->cpu = cpu;
291         vcpu->arch.host_cpu_context = this_cpu_ptr(kvm_host_cpu_state);
292 
293         kvm_arm_set_running_vcpu(vcpu);
294 }
295 
296 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
297 {
298         /*
299          * The arch-generic KVM code expects the cpu field of a vcpu to be -1
300          * if the vcpu is no longer assigned to a cpu.  This is used for the
301          * optimized make_all_cpus_request path.
302          */
303         vcpu->cpu = -1;
304 
305         kvm_arm_set_running_vcpu(NULL);
306 }
307 
308 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
309                                     struct kvm_mp_state *mp_state)
310 {
311         if (vcpu->arch.pause)
312                 mp_state->mp_state = KVM_MP_STATE_STOPPED;
313         else
314                 mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
315 
316         return 0;
317 }
318 
319 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
320                                     struct kvm_mp_state *mp_state)
321 {
322         switch (mp_state->mp_state) {
323         case KVM_MP_STATE_RUNNABLE:
324                 vcpu->arch.pause = false;
325                 break;
326         case KVM_MP_STATE_STOPPED:
327                 vcpu->arch.pause = true;
328                 break;
329         default:
330                 return -EINVAL;
331         }
332 
333         return 0;
334 }
335 
336 /**
337  * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled
338  * @v:          The VCPU pointer
339  *
340  * If the guest CPU is not waiting for interrupts or an interrupt line is
341  * asserted, the CPU is by definition runnable.
342  */
343 int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
344 {
345         return !!v->arch.irq_lines || kvm_vgic_vcpu_pending_irq(v);
346 }
347 
348 /* Just ensure a guest exit from a particular CPU */
349 static void exit_vm_noop(void *info)
350 {
351 }
352 
353 void force_vm_exit(const cpumask_t *mask)
354 {
355         smp_call_function_many(mask, exit_vm_noop, NULL, true);
356 }
357 
358 /**
359  * need_new_vmid_gen - check that the VMID is still valid
360  * @kvm: The VM's VMID to checkt
361  *
362  * return true if there is a new generation of VMIDs being used
363  *
364  * The hardware supports only 256 values with the value zero reserved for the
365  * host, so we check if an assigned value belongs to a previous generation,
366  * which which requires us to assign a new value. If we're the first to use a
367  * VMID for the new generation, we must flush necessary caches and TLBs on all
368  * CPUs.
369  */
370 static bool need_new_vmid_gen(struct kvm *kvm)
371 {
372         return unlikely(kvm->arch.vmid_gen != atomic64_read(&kvm_vmid_gen));
373 }
374 
375 /**
376  * update_vttbr - Update the VTTBR with a valid VMID before the guest runs
377  * @kvm The guest that we are about to run
378  *
379  * Called from kvm_arch_vcpu_ioctl_run before entering the guest to ensure the
380  * VM has a valid VMID, otherwise assigns a new one and flushes corresponding
381  * caches and TLBs.
382  */
383 static void update_vttbr(struct kvm *kvm)
384 {
385         phys_addr_t pgd_phys;
386         u64 vmid;
387 
388         if (!need_new_vmid_gen(kvm))
389                 return;
390 
391         spin_lock(&kvm_vmid_lock);
392 
393         /*
394          * We need to re-check the vmid_gen here to ensure that if another vcpu
395          * already allocated a valid vmid for this vm, then this vcpu should
396          * use the same vmid.
397          */
398         if (!need_new_vmid_gen(kvm)) {
399                 spin_unlock(&kvm_vmid_lock);
400                 return;
401         }
402 
403         /* First user of a new VMID generation? */
404         if (unlikely(kvm_next_vmid == 0)) {
405                 atomic64_inc(&kvm_vmid_gen);
406                 kvm_next_vmid = 1;
407 
408                 /*
409                  * On SMP we know no other CPUs can use this CPU's or each
410                  * other's VMID after force_vm_exit returns since the
411                  * kvm_vmid_lock blocks them from reentry to the guest.
412                  */
413                 force_vm_exit(cpu_all_mask);
414                 /*
415                  * Now broadcast TLB + ICACHE invalidation over the inner
416                  * shareable domain to make sure all data structures are
417                  * clean.
418                  */
419                 kvm_call_hyp(__kvm_flush_vm_context);
420         }
421 
422         kvm->arch.vmid_gen = atomic64_read(&kvm_vmid_gen);
423         kvm->arch.vmid = kvm_next_vmid;
424         kvm_next_vmid++;
425 
426         /* update vttbr to be used with the new vmid */
427         pgd_phys = virt_to_phys(kvm_get_hwpgd(kvm));
428         BUG_ON(pgd_phys & ~VTTBR_BADDR_MASK);
429         vmid = ((u64)(kvm->arch.vmid) << VTTBR_VMID_SHIFT) & VTTBR_VMID_MASK;
430         kvm->arch.vttbr = pgd_phys | vmid;
431 
432         spin_unlock(&kvm_vmid_lock);
433 }
434 
435 static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu)
436 {
437         struct kvm *kvm = vcpu->kvm;
438         int ret;
439 
440         if (likely(vcpu->arch.has_run_once))
441                 return 0;
442 
443         vcpu->arch.has_run_once = true;
444 
445         /*
446          * Map the VGIC hardware resources before running a vcpu the first
447          * time on this VM.
448          */
449         if (unlikely(irqchip_in_kernel(kvm) && !vgic_ready(kvm))) {
450                 ret = kvm_vgic_map_resources(kvm);
451                 if (ret)
452                         return ret;
453         }
454 
455         /*
456          * Enable the arch timers only if we have an in-kernel VGIC
457          * and it has been properly initialized, since we cannot handle
458          * interrupts from the virtual timer with a userspace gic.
459          */
460         if (irqchip_in_kernel(kvm) && vgic_initialized(kvm))
461                 kvm_timer_enable(kvm);
462 
463         return 0;
464 }
465 
466 bool kvm_arch_intc_initialized(struct kvm *kvm)
467 {
468         return vgic_initialized(kvm);
469 }
470 
471 static void vcpu_pause(struct kvm_vcpu *vcpu)
472 {
473         wait_queue_head_t *wq = kvm_arch_vcpu_wq(vcpu);
474 
475         wait_event_interruptible(*wq, !vcpu->arch.pause);
476 }
477 
478 static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu)
479 {
480         return vcpu->arch.target >= 0;
481 }
482 
483 /**
484  * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code
485  * @vcpu:       The VCPU pointer
486  * @run:        The kvm_run structure pointer used for userspace state exchange
487  *
488  * This function is called through the VCPU_RUN ioctl called from user space. It
489  * will execute VM code in a loop until the time slice for the process is used
490  * or some emulation is needed from user space in which case the function will
491  * return with return value 0 and with the kvm_run structure filled in with the
492  * required data for the requested emulation.
493  */
494 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
495 {
496         int ret;
497         sigset_t sigsaved;
498 
499         if (unlikely(!kvm_vcpu_initialized(vcpu)))
500                 return -ENOEXEC;
501 
502         ret = kvm_vcpu_first_run_init(vcpu);
503         if (ret)
504                 return ret;
505 
506         if (run->exit_reason == KVM_EXIT_MMIO) {
507                 ret = kvm_handle_mmio_return(vcpu, vcpu->run);
508                 if (ret)
509                         return ret;
510         }
511 
512         if (vcpu->sigset_active)
513                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
514 
515         ret = 1;
516         run->exit_reason = KVM_EXIT_UNKNOWN;
517         while (ret > 0) {
518                 /*
519                  * Check conditions before entering the guest
520                  */
521                 cond_resched();
522 
523                 update_vttbr(vcpu->kvm);
524 
525                 if (vcpu->arch.pause)
526                         vcpu_pause(vcpu);
527 
528                 /*
529                  * Disarming the background timer must be done in a
530                  * preemptible context, as this call may sleep.
531                  */
532                 kvm_timer_flush_hwstate(vcpu);
533 
534                 /*
535                  * Preparing the interrupts to be injected also
536                  * involves poking the GIC, which must be done in a
537                  * non-preemptible context.
538                  */
539                 preempt_disable();
540                 kvm_vgic_flush_hwstate(vcpu);
541 
542                 local_irq_disable();
543 
544                 /*
545                  * Re-check atomic conditions
546                  */
547                 if (signal_pending(current)) {
548                         ret = -EINTR;
549                         run->exit_reason = KVM_EXIT_INTR;
550                 }
551 
552                 if (ret <= 0 || need_new_vmid_gen(vcpu->kvm)) {
553                         local_irq_enable();
554                         kvm_vgic_sync_hwstate(vcpu);
555                         preempt_enable();
556                         kvm_timer_sync_hwstate(vcpu);
557                         continue;
558                 }
559 
560                 kvm_arm_setup_debug(vcpu);
561 
562                 /**************************************************************
563                  * Enter the guest
564                  */
565                 trace_kvm_entry(*vcpu_pc(vcpu));
566                 __kvm_guest_enter();
567                 vcpu->mode = IN_GUEST_MODE;
568 
569                 ret = kvm_call_hyp(__kvm_vcpu_run, vcpu);
570 
571                 vcpu->mode = OUTSIDE_GUEST_MODE;
572                 /*
573                  * Back from guest
574                  *************************************************************/
575 
576                 kvm_arm_clear_debug(vcpu);
577 
578                 /*
579                  * We may have taken a host interrupt in HYP mode (ie
580                  * while executing the guest). This interrupt is still
581                  * pending, as we haven't serviced it yet!
582                  *
583                  * We're now back in SVC mode, with interrupts
584                  * disabled.  Enabling the interrupts now will have
585                  * the effect of taking the interrupt again, in SVC
586                  * mode this time.
587                  */
588                 local_irq_enable();
589 
590                 /*
591                  * We do local_irq_enable() before calling kvm_guest_exit() so
592                  * that if a timer interrupt hits while running the guest we
593                  * account that tick as being spent in the guest.  We enable
594                  * preemption after calling kvm_guest_exit() so that if we get
595                  * preempted we make sure ticks after that is not counted as
596                  * guest time.
597                  */
598                 kvm_guest_exit();
599                 trace_kvm_exit(kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu));
600 
601                 kvm_vgic_sync_hwstate(vcpu);
602 
603                 preempt_enable();
604 
605                 kvm_timer_sync_hwstate(vcpu);
606 
607                 ret = handle_exit(vcpu, run, ret);
608         }
609 
610         if (vcpu->sigset_active)
611                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
612         return ret;
613 }
614 
615 static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
616 {
617         int bit_index;
618         bool set;
619         unsigned long *ptr;
620 
621         if (number == KVM_ARM_IRQ_CPU_IRQ)
622                 bit_index = __ffs(HCR_VI);
623         else /* KVM_ARM_IRQ_CPU_FIQ */
624                 bit_index = __ffs(HCR_VF);
625 
626         ptr = (unsigned long *)&vcpu->arch.irq_lines;
627         if (level)
628                 set = test_and_set_bit(bit_index, ptr);
629         else
630                 set = test_and_clear_bit(bit_index, ptr);
631 
632         /*
633          * If we didn't change anything, no need to wake up or kick other CPUs
634          */
635         if (set == level)
636                 return 0;
637 
638         /*
639          * The vcpu irq_lines field was updated, wake up sleeping VCPUs and
640          * trigger a world-switch round on the running physical CPU to set the
641          * virtual IRQ/FIQ fields in the HCR appropriately.
642          */
643         kvm_vcpu_kick(vcpu);
644 
645         return 0;
646 }
647 
648 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
649                           bool line_status)
650 {
651         u32 irq = irq_level->irq;
652         unsigned int irq_type, vcpu_idx, irq_num;
653         int nrcpus = atomic_read(&kvm->online_vcpus);
654         struct kvm_vcpu *vcpu = NULL;
655         bool level = irq_level->level;
656 
657         irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
658         vcpu_idx = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
659         irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
660 
661         trace_kvm_irq_line(irq_type, vcpu_idx, irq_num, irq_level->level);
662 
663         switch (irq_type) {
664         case KVM_ARM_IRQ_TYPE_CPU:
665                 if (irqchip_in_kernel(kvm))
666                         return -ENXIO;
667 
668                 if (vcpu_idx >= nrcpus)
669                         return -EINVAL;
670 
671                 vcpu = kvm_get_vcpu(kvm, vcpu_idx);
672                 if (!vcpu)
673                         return -EINVAL;
674 
675                 if (irq_num > KVM_ARM_IRQ_CPU_FIQ)
676                         return -EINVAL;
677 
678                 return vcpu_interrupt_line(vcpu, irq_num, level);
679         case KVM_ARM_IRQ_TYPE_PPI:
680                 if (!irqchip_in_kernel(kvm))
681                         return -ENXIO;
682 
683                 if (vcpu_idx >= nrcpus)
684                         return -EINVAL;
685 
686                 vcpu = kvm_get_vcpu(kvm, vcpu_idx);
687                 if (!vcpu)
688                         return -EINVAL;
689 
690                 if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS)
691                         return -EINVAL;
692 
693                 return kvm_vgic_inject_irq(kvm, vcpu->vcpu_id, irq_num, level);
694         case KVM_ARM_IRQ_TYPE_SPI:
695                 if (!irqchip_in_kernel(kvm))
696                         return -ENXIO;
697 
698                 if (irq_num < VGIC_NR_PRIVATE_IRQS)
699                         return -EINVAL;
700 
701                 return kvm_vgic_inject_irq(kvm, 0, irq_num, level);
702         }
703 
704         return -EINVAL;
705 }
706 
707 static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
708                                const struct kvm_vcpu_init *init)
709 {
710         unsigned int i;
711         int phys_target = kvm_target_cpu();
712 
713         if (init->target != phys_target)
714                 return -EINVAL;
715 
716         /*
717          * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must
718          * use the same target.
719          */
720         if (vcpu->arch.target != -1 && vcpu->arch.target != init->target)
721                 return -EINVAL;
722 
723         /* -ENOENT for unknown features, -EINVAL for invalid combinations. */
724         for (i = 0; i < sizeof(init->features) * 8; i++) {
725                 bool set = (init->features[i / 32] & (1 << (i % 32)));
726 
727                 if (set && i >= KVM_VCPU_MAX_FEATURES)
728                         return -ENOENT;
729 
730                 /*
731                  * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must
732                  * use the same feature set.
733                  */
734                 if (vcpu->arch.target != -1 && i < KVM_VCPU_MAX_FEATURES &&
735                     test_bit(i, vcpu->arch.features) != set)
736                         return -EINVAL;
737 
738                 if (set)
739                         set_bit(i, vcpu->arch.features);
740         }
741 
742         vcpu->arch.target = phys_target;
743 
744         /* Now we know what it is, we can reset it. */
745         return kvm_reset_vcpu(vcpu);
746 }
747 
748 
749 static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
750                                          struct kvm_vcpu_init *init)
751 {
752         int ret;
753 
754         ret = kvm_vcpu_set_target(vcpu, init);
755         if (ret)
756                 return ret;
757 
758         /*
759          * Ensure a rebooted VM will fault in RAM pages and detect if the
760          * guest MMU is turned off and flush the caches as needed.
761          */
762         if (vcpu->arch.has_run_once)
763                 stage2_unmap_vm(vcpu->kvm);
764 
765         vcpu_reset_hcr(vcpu);
766 
767         /*
768          * Handle the "start in power-off" case by marking the VCPU as paused.
769          */
770         if (test_bit(KVM_ARM_VCPU_POWER_OFF, vcpu->arch.features))
771                 vcpu->arch.pause = true;
772         else
773                 vcpu->arch.pause = false;
774 
775         return 0;
776 }
777 
778 long kvm_arch_vcpu_ioctl(struct file *filp,
779                          unsigned int ioctl, unsigned long arg)
780 {
781         struct kvm_vcpu *vcpu = filp->private_data;
782         void __user *argp = (void __user *)arg;
783 
784         switch (ioctl) {
785         case KVM_ARM_VCPU_INIT: {
786                 struct kvm_vcpu_init init;
787 
788                 if (copy_from_user(&init, argp, sizeof(init)))
789                         return -EFAULT;
790 
791                 return kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init);
792         }
793         case KVM_SET_ONE_REG:
794         case KVM_GET_ONE_REG: {
795                 struct kvm_one_reg reg;
796 
797                 if (unlikely(!kvm_vcpu_initialized(vcpu)))
798                         return -ENOEXEC;
799 
800                 if (copy_from_user(&reg, argp, sizeof(reg)))
801                         return -EFAULT;
802                 if (ioctl == KVM_SET_ONE_REG)
803                         return kvm_arm_set_reg(vcpu, &reg);
804                 else
805                         return kvm_arm_get_reg(vcpu, &reg);
806         }
807         case KVM_GET_REG_LIST: {
808                 struct kvm_reg_list __user *user_list = argp;
809                 struct kvm_reg_list reg_list;
810                 unsigned n;
811 
812                 if (unlikely(!kvm_vcpu_initialized(vcpu)))
813                         return -ENOEXEC;
814 
815                 if (copy_from_user(&reg_list, user_list, sizeof(reg_list)))
816                         return -EFAULT;
817                 n = reg_list.n;
818                 reg_list.n = kvm_arm_num_regs(vcpu);
819                 if (copy_to_user(user_list, &reg_list, sizeof(reg_list)))
820                         return -EFAULT;
821                 if (n < reg_list.n)
822                         return -E2BIG;
823                 return kvm_arm_copy_reg_indices(vcpu, user_list->reg);
824         }
825         default:
826                 return -EINVAL;
827         }
828 }
829 
830 /**
831  * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot
832  * @kvm: kvm instance
833  * @log: slot id and address to which we copy the log
834  *
835  * Steps 1-4 below provide general overview of dirty page logging. See
836  * kvm_get_dirty_log_protect() function description for additional details.
837  *
838  * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we
839  * always flush the TLB (step 4) even if previous step failed  and the dirty
840  * bitmap may be corrupt. Regardless of previous outcome the KVM logging API
841  * does not preclude user space subsequent dirty log read. Flushing TLB ensures
842  * writes will be marked dirty for next log read.
843  *
844  *   1. Take a snapshot of the bit and clear it if needed.
845  *   2. Write protect the corresponding page.
846  *   3. Copy the snapshot to the userspace.
847  *   4. Flush TLB's if needed.
848  */
849 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
850 {
851         bool is_dirty = false;
852         int r;
853 
854         mutex_lock(&kvm->slots_lock);
855 
856         r = kvm_get_dirty_log_protect(kvm, log, &is_dirty);
857 
858         if (is_dirty)
859                 kvm_flush_remote_tlbs(kvm);
860 
861         mutex_unlock(&kvm->slots_lock);
862         return r;
863 }
864 
865 static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
866                                         struct kvm_arm_device_addr *dev_addr)
867 {
868         unsigned long dev_id, type;
869 
870         dev_id = (dev_addr->id & KVM_ARM_DEVICE_ID_MASK) >>
871                 KVM_ARM_DEVICE_ID_SHIFT;
872         type = (dev_addr->id & KVM_ARM_DEVICE_TYPE_MASK) >>
873                 KVM_ARM_DEVICE_TYPE_SHIFT;
874 
875         switch (dev_id) {
876         case KVM_ARM_DEVICE_VGIC_V2:
877                 return kvm_vgic_addr(kvm, type, &dev_addr->addr, true);
878         default:
879                 return -ENODEV;
880         }
881 }
882 
883 long kvm_arch_vm_ioctl(struct file *filp,
884                        unsigned int ioctl, unsigned long arg)
885 {
886         struct kvm *kvm = filp->private_data;
887         void __user *argp = (void __user *)arg;
888 
889         switch (ioctl) {
890         case KVM_CREATE_IRQCHIP: {
891                 return kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
892         }
893         case KVM_ARM_SET_DEVICE_ADDR: {
894                 struct kvm_arm_device_addr dev_addr;
895 
896                 if (copy_from_user(&dev_addr, argp, sizeof(dev_addr)))
897                         return -EFAULT;
898                 return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr);
899         }
900         case KVM_ARM_PREFERRED_TARGET: {
901                 int err;
902                 struct kvm_vcpu_init init;
903 
904                 err = kvm_vcpu_preferred_target(&init);
905                 if (err)
906                         return err;
907 
908                 if (copy_to_user(argp, &init, sizeof(init)))
909                         return -EFAULT;
910 
911                 return 0;
912         }
913         default:
914                 return -EINVAL;
915         }
916 }
917 
918 static void cpu_init_hyp_mode(void *dummy)
919 {
920         phys_addr_t boot_pgd_ptr;
921         phys_addr_t pgd_ptr;
922         unsigned long hyp_stack_ptr;
923         unsigned long stack_page;
924         unsigned long vector_ptr;
925 
926         /* Switch from the HYP stub to our own HYP init vector */
927         __hyp_set_vectors(kvm_get_idmap_vector());
928 
929         boot_pgd_ptr = kvm_mmu_get_boot_httbr();
930         pgd_ptr = kvm_mmu_get_httbr();
931         stack_page = __this_cpu_read(kvm_arm_hyp_stack_page);
932         hyp_stack_ptr = stack_page + PAGE_SIZE;
933         vector_ptr = (unsigned long)__kvm_hyp_vector;
934 
935         __cpu_init_hyp_mode(boot_pgd_ptr, pgd_ptr, hyp_stack_ptr, vector_ptr);
936 
937         kvm_arm_init_debug();
938 }
939 
940 static int hyp_init_cpu_notify(struct notifier_block *self,
941                                unsigned long action, void *cpu)
942 {
943         switch (action) {
944         case CPU_STARTING:
945         case CPU_STARTING_FROZEN:
946                 if (__hyp_get_vectors() == hyp_default_vectors)
947                         cpu_init_hyp_mode(NULL);
948                 break;
949         }
950 
951         return NOTIFY_OK;
952 }
953 
954 static struct notifier_block hyp_init_cpu_nb = {
955         .notifier_call = hyp_init_cpu_notify,
956 };
957 
958 #ifdef CONFIG_CPU_PM
959 static int hyp_init_cpu_pm_notifier(struct notifier_block *self,
960                                     unsigned long cmd,
961                                     void *v)
962 {
963         if (cmd == CPU_PM_EXIT &&
964             __hyp_get_vectors() == hyp_default_vectors) {
965                 cpu_init_hyp_mode(NULL);
966                 return NOTIFY_OK;
967         }
968 
969         return NOTIFY_DONE;
970 }
971 
972 static struct notifier_block hyp_init_cpu_pm_nb = {
973         .notifier_call = hyp_init_cpu_pm_notifier,
974 };
975 
976 static void __init hyp_cpu_pm_init(void)
977 {
978         cpu_pm_register_notifier(&hyp_init_cpu_pm_nb);
979 }
980 #else
981 static inline void hyp_cpu_pm_init(void)
982 {
983 }
984 #endif
985 
986 /**
987  * Inits Hyp-mode on all online CPUs
988  */
989 static int init_hyp_mode(void)
990 {
991         int cpu;
992         int err = 0;
993 
994         /*
995          * Allocate Hyp PGD and setup Hyp identity mapping
996          */
997         err = kvm_mmu_init();
998         if (err)
999                 goto out_err;
1000 
1001         /*
1002          * It is probably enough to obtain the default on one
1003          * CPU. It's unlikely to be different on the others.
1004          */
1005         hyp_default_vectors = __hyp_get_vectors();
1006 
1007         /*
1008          * Allocate stack pages for Hypervisor-mode
1009          */
1010         for_each_possible_cpu(cpu) {
1011                 unsigned long stack_page;
1012 
1013                 stack_page = __get_free_page(GFP_KERNEL);
1014                 if (!stack_page) {
1015                         err = -ENOMEM;
1016                         goto out_free_stack_pages;
1017                 }
1018 
1019                 per_cpu(kvm_arm_hyp_stack_page, cpu) = stack_page;
1020         }
1021 
1022         /*
1023          * Map the Hyp-code called directly from the host
1024          */
1025         err = create_hyp_mappings(__kvm_hyp_code_start, __kvm_hyp_code_end);
1026         if (err) {
1027                 kvm_err("Cannot map world-switch code\n");
1028                 goto out_free_mappings;
1029         }
1030 
1031         /*
1032          * Map the Hyp stack pages
1033          */
1034         for_each_possible_cpu(cpu) {
1035                 char *stack_page = (char *)per_cpu(kvm_arm_hyp_stack_page, cpu);
1036                 err = create_hyp_mappings(stack_page, stack_page + PAGE_SIZE);
1037 
1038                 if (err) {
1039                         kvm_err("Cannot map hyp stack\n");
1040                         goto out_free_mappings;
1041                 }
1042         }
1043 
1044         /*
1045          * Map the host CPU structures
1046          */
1047         kvm_host_cpu_state = alloc_percpu(kvm_cpu_context_t);
1048         if (!kvm_host_cpu_state) {
1049                 err = -ENOMEM;
1050                 kvm_err("Cannot allocate host CPU state\n");
1051                 goto out_free_mappings;
1052         }
1053 
1054         for_each_possible_cpu(cpu) {
1055                 kvm_cpu_context_t *cpu_ctxt;
1056 
1057                 cpu_ctxt = per_cpu_ptr(kvm_host_cpu_state, cpu);
1058                 err = create_hyp_mappings(cpu_ctxt, cpu_ctxt + 1);
1059 
1060                 if (err) {
1061                         kvm_err("Cannot map host CPU state: %d\n", err);
1062                         goto out_free_context;
1063                 }
1064         }
1065 
1066         /*
1067          * Execute the init code on each CPU.
1068          */
1069         on_each_cpu(cpu_init_hyp_mode, NULL, 1);
1070 
1071         /*
1072          * Init HYP view of VGIC
1073          */
1074         err = kvm_vgic_hyp_init();
1075         if (err)
1076                 goto out_free_context;
1077 
1078         /*
1079          * Init HYP architected timer support
1080          */
1081         err = kvm_timer_hyp_init();
1082         if (err)
1083                 goto out_free_context;
1084 
1085 #ifndef CONFIG_HOTPLUG_CPU
1086         free_boot_hyp_pgd();
1087 #endif
1088 
1089         kvm_perf_init();
1090 
1091         kvm_info("Hyp mode initialized successfully\n");
1092 
1093         return 0;
1094 out_free_context:
1095         free_percpu(kvm_host_cpu_state);
1096 out_free_mappings:
1097         free_hyp_pgds();
1098 out_free_stack_pages:
1099         for_each_possible_cpu(cpu)
1100                 free_page(per_cpu(kvm_arm_hyp_stack_page, cpu));
1101 out_err:
1102         kvm_err("error initializing Hyp mode: %d\n", err);
1103         return err;
1104 }
1105 
1106 static void check_kvm_target_cpu(void *ret)
1107 {
1108         *(int *)ret = kvm_target_cpu();
1109 }
1110 
1111 struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr)
1112 {
1113         struct kvm_vcpu *vcpu;
1114         int i;
1115 
1116         mpidr &= MPIDR_HWID_BITMASK;
1117         kvm_for_each_vcpu(i, vcpu, kvm) {
1118                 if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu))
1119                         return vcpu;
1120         }
1121         return NULL;
1122 }
1123 
1124 /**
1125  * Initialize Hyp-mode and memory mappings on all CPUs.
1126  */
1127 int kvm_arch_init(void *opaque)
1128 {
1129         int err;
1130         int ret, cpu;
1131 
1132         if (!is_hyp_mode_available()) {
1133                 kvm_err("HYP mode not available\n");
1134                 return -ENODEV;
1135         }
1136 
1137         for_each_online_cpu(cpu) {
1138                 smp_call_function_single(cpu, check_kvm_target_cpu, &ret, 1);
1139                 if (ret < 0) {
1140                         kvm_err("Error, CPU %d not supported!\n", cpu);
1141                         return -ENODEV;
1142                 }
1143         }
1144 
1145         cpu_notifier_register_begin();
1146 
1147         err = init_hyp_mode();
1148         if (err)
1149                 goto out_err;
1150 
1151         err = __register_cpu_notifier(&hyp_init_cpu_nb);
1152         if (err) {
1153                 kvm_err("Cannot register HYP init CPU notifier (%d)\n", err);
1154                 goto out_err;
1155         }
1156 
1157         cpu_notifier_register_done();
1158 
1159         hyp_cpu_pm_init();
1160 
1161         kvm_coproc_table_init();
1162         return 0;
1163 out_err:
1164         cpu_notifier_register_done();
1165         return err;
1166 }
1167 
1168 /* NOP: Compiling as a module not supported */
1169 void kvm_arch_exit(void)
1170 {
1171         kvm_perf_teardown();
1172 }
1173 
1174 static int arm_init(void)
1175 {
1176         int rc = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
1177         return rc;
1178 }
1179 
1180 module_init(arm_init);
1181 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp