~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/s390/kvm/kvm-s390.c

Version: ~ [ linux-5.15-rc7 ] ~ [ linux-5.14.14 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.75 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.155 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.213 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.252 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.287 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.289 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * hosting zSeries kernel virtual machines
  3  *
  4  * Copyright IBM Corp. 2008, 2009
  5  *
  6  * This program is free software; you can redistribute it and/or modify
  7  * it under the terms of the GNU General Public License (version 2 only)
  8  * as published by the Free Software Foundation.
  9  *
 10  *    Author(s): Carsten Otte <cotte@de.ibm.com>
 11  *               Christian Borntraeger <borntraeger@de.ibm.com>
 12  *               Heiko Carstens <heiko.carstens@de.ibm.com>
 13  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
 14  *               Jason J. Herne <jjherne@us.ibm.com>
 15  */
 16 
 17 #include <linux/compiler.h>
 18 #include <linux/err.h>
 19 #include <linux/fs.h>
 20 #include <linux/hrtimer.h>
 21 #include <linux/init.h>
 22 #include <linux/kvm.h>
 23 #include <linux/kvm_host.h>
 24 #include <linux/module.h>
 25 #include <linux/slab.h>
 26 #include <linux/timer.h>
 27 #include <asm/asm-offsets.h>
 28 #include <asm/lowcore.h>
 29 #include <asm/pgtable.h>
 30 #include <asm/nmi.h>
 31 #include <asm/switch_to.h>
 32 #include <asm/facility.h>
 33 #include <asm/sclp.h>
 34 #include "kvm-s390.h"
 35 #include "gaccess.h"
 36 
 37 #define CREATE_TRACE_POINTS
 38 #include "trace.h"
 39 #include "trace-s390.h"
 40 
 41 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
 42 
 43 struct kvm_stats_debugfs_item debugfs_entries[] = {
 44         { "userspace_handled", VCPU_STAT(exit_userspace) },
 45         { "exit_null", VCPU_STAT(exit_null) },
 46         { "exit_validity", VCPU_STAT(exit_validity) },
 47         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
 48         { "exit_external_request", VCPU_STAT(exit_external_request) },
 49         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
 50         { "exit_instruction", VCPU_STAT(exit_instruction) },
 51         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
 52         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
 53         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
 54         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
 55         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
 56         { "instruction_stctl", VCPU_STAT(instruction_stctl) },
 57         { "instruction_stctg", VCPU_STAT(instruction_stctg) },
 58         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
 59         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
 60         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
 61         { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
 62         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
 63         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
 64         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
 65         { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
 66         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
 67         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
 68         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
 69         { "instruction_spx", VCPU_STAT(instruction_spx) },
 70         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
 71         { "instruction_stap", VCPU_STAT(instruction_stap) },
 72         { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
 73         { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
 74         { "instruction_stsch", VCPU_STAT(instruction_stsch) },
 75         { "instruction_chsc", VCPU_STAT(instruction_chsc) },
 76         { "instruction_essa", VCPU_STAT(instruction_essa) },
 77         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
 78         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
 79         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
 80         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
 81         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
 82         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
 83         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
 84         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
 85         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
 86         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
 87         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
 88         { "diagnose_10", VCPU_STAT(diagnose_10) },
 89         { "diagnose_44", VCPU_STAT(diagnose_44) },
 90         { "diagnose_9c", VCPU_STAT(diagnose_9c) },
 91         { NULL }
 92 };
 93 
 94 unsigned long *vfacilities;
 95 static struct gmap_notifier gmap_notifier;
 96 
 97 /* test availability of vfacility */
 98 int test_vfacility(unsigned long nr)
 99 {
100         return __test_facility(nr, (void *) vfacilities);
101 }
102 
103 /* Section: not file related */
104 int kvm_arch_hardware_enable(void)
105 {
106         /* every s390 is virtualization enabled ;-) */
107         return 0;
108 }
109 
110 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
111 
112 int kvm_arch_hardware_setup(void)
113 {
114         gmap_notifier.notifier_call = kvm_gmap_notifier;
115         gmap_register_ipte_notifier(&gmap_notifier);
116         return 0;
117 }
118 
119 void kvm_arch_hardware_unsetup(void)
120 {
121         gmap_unregister_ipte_notifier(&gmap_notifier);
122 }
123 
124 int kvm_arch_init(void *opaque)
125 {
126         /* Register floating interrupt controller interface. */
127         return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
128 }
129 
130 /* Section: device related */
131 long kvm_arch_dev_ioctl(struct file *filp,
132                         unsigned int ioctl, unsigned long arg)
133 {
134         if (ioctl == KVM_S390_ENABLE_SIE)
135                 return s390_enable_sie();
136         return -EINVAL;
137 }
138 
139 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
140 {
141         int r;
142 
143         switch (ext) {
144         case KVM_CAP_S390_PSW:
145         case KVM_CAP_S390_GMAP:
146         case KVM_CAP_SYNC_MMU:
147 #ifdef CONFIG_KVM_S390_UCONTROL
148         case KVM_CAP_S390_UCONTROL:
149 #endif
150         case KVM_CAP_ASYNC_PF:
151         case KVM_CAP_SYNC_REGS:
152         case KVM_CAP_ONE_REG:
153         case KVM_CAP_ENABLE_CAP:
154         case KVM_CAP_S390_CSS_SUPPORT:
155         case KVM_CAP_IOEVENTFD:
156         case KVM_CAP_DEVICE_CTRL:
157         case KVM_CAP_ENABLE_CAP_VM:
158         case KVM_CAP_S390_IRQCHIP:
159         case KVM_CAP_VM_ATTRIBUTES:
160         case KVM_CAP_MP_STATE:
161                 r = 1;
162                 break;
163         case KVM_CAP_NR_VCPUS:
164         case KVM_CAP_MAX_VCPUS:
165                 r = KVM_MAX_VCPUS;
166                 break;
167         case KVM_CAP_NR_MEMSLOTS:
168                 r = KVM_USER_MEM_SLOTS;
169                 break;
170         case KVM_CAP_S390_COW:
171                 r = MACHINE_HAS_ESOP;
172                 break;
173         default:
174                 r = 0;
175         }
176         return r;
177 }
178 
179 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
180                                         struct kvm_memory_slot *memslot)
181 {
182         gfn_t cur_gfn, last_gfn;
183         unsigned long address;
184         struct gmap *gmap = kvm->arch.gmap;
185 
186         down_read(&gmap->mm->mmap_sem);
187         /* Loop over all guest pages */
188         last_gfn = memslot->base_gfn + memslot->npages;
189         for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
190                 address = gfn_to_hva_memslot(memslot, cur_gfn);
191 
192                 if (gmap_test_and_clear_dirty(address, gmap))
193                         mark_page_dirty(kvm, cur_gfn);
194         }
195         up_read(&gmap->mm->mmap_sem);
196 }
197 
198 /* Section: vm related */
199 /*
200  * Get (and clear) the dirty memory log for a memory slot.
201  */
202 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
203                                struct kvm_dirty_log *log)
204 {
205         int r;
206         unsigned long n;
207         struct kvm_memory_slot *memslot;
208         int is_dirty = 0;
209 
210         if (kvm_is_ucontrol(kvm))
211                 return -EINVAL;
212 
213         mutex_lock(&kvm->slots_lock);
214 
215         r = -EINVAL;
216         if (log->slot >= KVM_USER_MEM_SLOTS)
217                 goto out;
218 
219         memslot = id_to_memslot(kvm->memslots, log->slot);
220         r = -ENOENT;
221         if (!memslot->dirty_bitmap)
222                 goto out;
223 
224         kvm_s390_sync_dirty_log(kvm, memslot);
225         r = kvm_get_dirty_log(kvm, log, &is_dirty);
226         if (r)
227                 goto out;
228 
229         /* Clear the dirty log */
230         if (is_dirty) {
231                 n = kvm_dirty_bitmap_bytes(memslot);
232                 memset(memslot->dirty_bitmap, 0, n);
233         }
234         r = 0;
235 out:
236         mutex_unlock(&kvm->slots_lock);
237         return r;
238 }
239 
240 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
241 {
242         int r;
243 
244         if (cap->flags)
245                 return -EINVAL;
246 
247         switch (cap->cap) {
248         case KVM_CAP_S390_IRQCHIP:
249                 kvm->arch.use_irqchip = 1;
250                 r = 0;
251                 break;
252         default:
253                 r = -EINVAL;
254                 break;
255         }
256         return r;
257 }
258 
259 static int kvm_s390_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
260 {
261         int ret;
262         unsigned int idx;
263         switch (attr->attr) {
264         case KVM_S390_VM_MEM_ENABLE_CMMA:
265                 ret = -EBUSY;
266                 mutex_lock(&kvm->lock);
267                 if (atomic_read(&kvm->online_vcpus) == 0) {
268                         kvm->arch.use_cmma = 1;
269                         ret = 0;
270                 }
271                 mutex_unlock(&kvm->lock);
272                 break;
273         case KVM_S390_VM_MEM_CLR_CMMA:
274                 mutex_lock(&kvm->lock);
275                 idx = srcu_read_lock(&kvm->srcu);
276                 page_table_reset_pgste(kvm->arch.gmap->mm, 0, TASK_SIZE, false);
277                 srcu_read_unlock(&kvm->srcu, idx);
278                 mutex_unlock(&kvm->lock);
279                 ret = 0;
280                 break;
281         default:
282                 ret = -ENXIO;
283                 break;
284         }
285         return ret;
286 }
287 
288 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
289 {
290         int ret;
291 
292         switch (attr->group) {
293         case KVM_S390_VM_MEM_CTRL:
294                 ret = kvm_s390_mem_control(kvm, attr);
295                 break;
296         default:
297                 ret = -ENXIO;
298                 break;
299         }
300 
301         return ret;
302 }
303 
304 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
305 {
306         return -ENXIO;
307 }
308 
309 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
310 {
311         int ret;
312 
313         switch (attr->group) {
314         case KVM_S390_VM_MEM_CTRL:
315                 switch (attr->attr) {
316                 case KVM_S390_VM_MEM_ENABLE_CMMA:
317                 case KVM_S390_VM_MEM_CLR_CMMA:
318                         ret = 0;
319                         break;
320                 default:
321                         ret = -ENXIO;
322                         break;
323                 }
324                 break;
325         default:
326                 ret = -ENXIO;
327                 break;
328         }
329 
330         return ret;
331 }
332 
333 long kvm_arch_vm_ioctl(struct file *filp,
334                        unsigned int ioctl, unsigned long arg)
335 {
336         struct kvm *kvm = filp->private_data;
337         void __user *argp = (void __user *)arg;
338         struct kvm_device_attr attr;
339         int r;
340 
341         switch (ioctl) {
342         case KVM_S390_INTERRUPT: {
343                 struct kvm_s390_interrupt s390int;
344 
345                 r = -EFAULT;
346                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
347                         break;
348                 r = kvm_s390_inject_vm(kvm, &s390int);
349                 break;
350         }
351         case KVM_ENABLE_CAP: {
352                 struct kvm_enable_cap cap;
353                 r = -EFAULT;
354                 if (copy_from_user(&cap, argp, sizeof(cap)))
355                         break;
356                 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
357                 break;
358         }
359         case KVM_CREATE_IRQCHIP: {
360                 struct kvm_irq_routing_entry routing;
361 
362                 r = -EINVAL;
363                 if (kvm->arch.use_irqchip) {
364                         /* Set up dummy routing. */
365                         memset(&routing, 0, sizeof(routing));
366                         kvm_set_irq_routing(kvm, &routing, 0, 0);
367                         r = 0;
368                 }
369                 break;
370         }
371         case KVM_SET_DEVICE_ATTR: {
372                 r = -EFAULT;
373                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
374                         break;
375                 r = kvm_s390_vm_set_attr(kvm, &attr);
376                 break;
377         }
378         case KVM_GET_DEVICE_ATTR: {
379                 r = -EFAULT;
380                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
381                         break;
382                 r = kvm_s390_vm_get_attr(kvm, &attr);
383                 break;
384         }
385         case KVM_HAS_DEVICE_ATTR: {
386                 r = -EFAULT;
387                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
388                         break;
389                 r = kvm_s390_vm_has_attr(kvm, &attr);
390                 break;
391         }
392         default:
393                 r = -ENOTTY;
394         }
395 
396         return r;
397 }
398 
399 static int kvm_s390_crypto_init(struct kvm *kvm)
400 {
401         if (!test_vfacility(76))
402                 return 0;
403 
404         kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
405                                          GFP_KERNEL | GFP_DMA);
406         if (!kvm->arch.crypto.crycb)
407                 return -ENOMEM;
408 
409         kvm->arch.crypto.crycbd = (__u32) (unsigned long) kvm->arch.crypto.crycb |
410                                   CRYCB_FORMAT1;
411 
412         return 0;
413 }
414 
415 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
416 {
417         int rc;
418         char debug_name[16];
419         static unsigned long sca_offset;
420 
421         rc = -EINVAL;
422 #ifdef CONFIG_KVM_S390_UCONTROL
423         if (type & ~KVM_VM_S390_UCONTROL)
424                 goto out_err;
425         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
426                 goto out_err;
427 #else
428         if (type)
429                 goto out_err;
430 #endif
431 
432         rc = s390_enable_sie();
433         if (rc)
434                 goto out_err;
435 
436         rc = -ENOMEM;
437 
438         kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
439         if (!kvm->arch.sca)
440                 goto out_err;
441         spin_lock(&kvm_lock);
442         sca_offset = (sca_offset + 16) & 0x7f0;
443         kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
444         spin_unlock(&kvm_lock);
445 
446         sprintf(debug_name, "kvm-%u", current->pid);
447 
448         kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
449         if (!kvm->arch.dbf)
450                 goto out_nodbf;
451 
452         if (kvm_s390_crypto_init(kvm) < 0)
453                 goto out_crypto;
454 
455         spin_lock_init(&kvm->arch.float_int.lock);
456         INIT_LIST_HEAD(&kvm->arch.float_int.list);
457         init_waitqueue_head(&kvm->arch.ipte_wq);
458 
459         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
460         VM_EVENT(kvm, 3, "%s", "vm created");
461 
462         if (type & KVM_VM_S390_UCONTROL) {
463                 kvm->arch.gmap = NULL;
464         } else {
465                 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
466                 if (!kvm->arch.gmap)
467                         goto out_nogmap;
468                 kvm->arch.gmap->private = kvm;
469                 kvm->arch.gmap->pfault_enabled = 0;
470         }
471 
472         kvm->arch.css_support = 0;
473         kvm->arch.use_irqchip = 0;
474 
475         spin_lock_init(&kvm->arch.start_stop_lock);
476 
477         return 0;
478 out_nogmap:
479         kfree(kvm->arch.crypto.crycb);
480 out_crypto:
481         debug_unregister(kvm->arch.dbf);
482 out_nodbf:
483         free_page((unsigned long)(kvm->arch.sca));
484 out_err:
485         return rc;
486 }
487 
488 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
489 {
490         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
491         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
492         kvm_s390_clear_local_irqs(vcpu);
493         kvm_clear_async_pf_completion_queue(vcpu);
494         if (!kvm_is_ucontrol(vcpu->kvm)) {
495                 clear_bit(63 - vcpu->vcpu_id,
496                           (unsigned long *) &vcpu->kvm->arch.sca->mcn);
497                 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
498                     (__u64) vcpu->arch.sie_block)
499                         vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
500         }
501         smp_mb();
502 
503         if (kvm_is_ucontrol(vcpu->kvm))
504                 gmap_free(vcpu->arch.gmap);
505 
506         if (kvm_s390_cmma_enabled(vcpu->kvm))
507                 kvm_s390_vcpu_unsetup_cmma(vcpu);
508         free_page((unsigned long)(vcpu->arch.sie_block));
509 
510         kvm_vcpu_uninit(vcpu);
511         kmem_cache_free(kvm_vcpu_cache, vcpu);
512 }
513 
514 static void kvm_free_vcpus(struct kvm *kvm)
515 {
516         unsigned int i;
517         struct kvm_vcpu *vcpu;
518 
519         kvm_for_each_vcpu(i, vcpu, kvm)
520                 kvm_arch_vcpu_destroy(vcpu);
521 
522         mutex_lock(&kvm->lock);
523         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
524                 kvm->vcpus[i] = NULL;
525 
526         atomic_set(&kvm->online_vcpus, 0);
527         mutex_unlock(&kvm->lock);
528 }
529 
530 void kvm_arch_destroy_vm(struct kvm *kvm)
531 {
532         kvm_free_vcpus(kvm);
533         free_page((unsigned long)(kvm->arch.sca));
534         debug_unregister(kvm->arch.dbf);
535         kfree(kvm->arch.crypto.crycb);
536         if (!kvm_is_ucontrol(kvm))
537                 gmap_free(kvm->arch.gmap);
538         kvm_s390_destroy_adapters(kvm);
539         kvm_s390_clear_float_irqs(kvm);
540 }
541 
542 /* Section: vcpu related */
543 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
544 {
545         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
546         kvm_clear_async_pf_completion_queue(vcpu);
547         if (kvm_is_ucontrol(vcpu->kvm)) {
548                 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
549                 if (!vcpu->arch.gmap)
550                         return -ENOMEM;
551                 vcpu->arch.gmap->private = vcpu->kvm;
552                 return 0;
553         }
554 
555         vcpu->arch.gmap = vcpu->kvm->arch.gmap;
556         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
557                                     KVM_SYNC_GPRS |
558                                     KVM_SYNC_ACRS |
559                                     KVM_SYNC_CRS |
560                                     KVM_SYNC_ARCH0 |
561                                     KVM_SYNC_PFAULT;
562         return 0;
563 }
564 
565 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
566 {
567         save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
568         save_fp_regs(vcpu->arch.host_fpregs.fprs);
569         save_access_regs(vcpu->arch.host_acrs);
570         restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
571         restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
572         restore_access_regs(vcpu->run->s.regs.acrs);
573         gmap_enable(vcpu->arch.gmap);
574         atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
575 }
576 
577 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
578 {
579         atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
580         gmap_disable(vcpu->arch.gmap);
581         save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
582         save_fp_regs(vcpu->arch.guest_fpregs.fprs);
583         save_access_regs(vcpu->run->s.regs.acrs);
584         restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
585         restore_fp_regs(vcpu->arch.host_fpregs.fprs);
586         restore_access_regs(vcpu->arch.host_acrs);
587 }
588 
589 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
590 {
591         /* this equals initial cpu reset in pop, but we don't switch to ESA */
592         vcpu->arch.sie_block->gpsw.mask = 0UL;
593         vcpu->arch.sie_block->gpsw.addr = 0UL;
594         kvm_s390_set_prefix(vcpu, 0);
595         vcpu->arch.sie_block->cputm     = 0UL;
596         vcpu->arch.sie_block->ckc       = 0UL;
597         vcpu->arch.sie_block->todpr     = 0;
598         memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
599         vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
600         vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
601         vcpu->arch.guest_fpregs.fpc = 0;
602         asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
603         vcpu->arch.sie_block->gbea = 1;
604         vcpu->arch.sie_block->pp = 0;
605         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
606         kvm_clear_async_pf_completion_queue(vcpu);
607         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
608                 kvm_s390_vcpu_stop(vcpu);
609         kvm_s390_clear_local_irqs(vcpu);
610 }
611 
612 int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
613 {
614         return 0;
615 }
616 
617 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
618 {
619         if (!test_vfacility(76))
620                 return;
621 
622         vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
623 }
624 
625 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
626 {
627         free_page(vcpu->arch.sie_block->cbrlo);
628         vcpu->arch.sie_block->cbrlo = 0;
629 }
630 
631 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
632 {
633         vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
634         if (!vcpu->arch.sie_block->cbrlo)
635                 return -ENOMEM;
636 
637         vcpu->arch.sie_block->ecb2 |= 0x80;
638         vcpu->arch.sie_block->ecb2 &= ~0x08;
639         return 0;
640 }
641 
642 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
643 {
644         int rc = 0;
645 
646         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
647                                                     CPUSTAT_SM |
648                                                     CPUSTAT_STOPPED |
649                                                     CPUSTAT_GED);
650         vcpu->arch.sie_block->ecb   = 6;
651         if (test_vfacility(50) && test_vfacility(73))
652                 vcpu->arch.sie_block->ecb |= 0x10;
653 
654         vcpu->arch.sie_block->ecb2  = 8;
655         vcpu->arch.sie_block->eca   = 0xD1002000U;
656         if (sclp_has_siif())
657                 vcpu->arch.sie_block->eca |= 1;
658         vcpu->arch.sie_block->fac   = (int) (long) vfacilities;
659         vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE |
660                                       ICTL_TPROT;
661 
662         if (kvm_s390_cmma_enabled(vcpu->kvm)) {
663                 rc = kvm_s390_vcpu_setup_cmma(vcpu);
664                 if (rc)
665                         return rc;
666         }
667         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
668         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
669         get_cpu_id(&vcpu->arch.cpu_id);
670         vcpu->arch.cpu_id.version = 0xff;
671 
672         kvm_s390_vcpu_crypto_setup(vcpu);
673 
674         return rc;
675 }
676 
677 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
678                                       unsigned int id)
679 {
680         struct kvm_vcpu *vcpu;
681         struct sie_page *sie_page;
682         int rc = -EINVAL;
683 
684         if (id >= KVM_MAX_VCPUS)
685                 goto out;
686 
687         rc = -ENOMEM;
688 
689         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
690         if (!vcpu)
691                 goto out;
692 
693         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
694         if (!sie_page)
695                 goto out_free_cpu;
696 
697         vcpu->arch.sie_block = &sie_page->sie_block;
698         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
699 
700         vcpu->arch.sie_block->icpua = id;
701         if (!kvm_is_ucontrol(kvm)) {
702                 if (!kvm->arch.sca) {
703                         WARN_ON_ONCE(1);
704                         goto out_free_cpu;
705                 }
706                 if (!kvm->arch.sca->cpu[id].sda)
707                         kvm->arch.sca->cpu[id].sda =
708                                 (__u64) vcpu->arch.sie_block;
709                 vcpu->arch.sie_block->scaoh =
710                         (__u32)(((__u64)kvm->arch.sca) >> 32);
711                 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
712                 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
713         }
714 
715         spin_lock_init(&vcpu->arch.local_int.lock);
716         INIT_LIST_HEAD(&vcpu->arch.local_int.list);
717         vcpu->arch.local_int.float_int = &kvm->arch.float_int;
718         vcpu->arch.local_int.wq = &vcpu->wq;
719         vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
720 
721         rc = kvm_vcpu_init(vcpu, kvm, id);
722         if (rc)
723                 goto out_free_sie_block;
724         VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
725                  vcpu->arch.sie_block);
726         trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
727 
728         return vcpu;
729 out_free_sie_block:
730         free_page((unsigned long)(vcpu->arch.sie_block));
731 out_free_cpu:
732         kmem_cache_free(kvm_vcpu_cache, vcpu);
733 out:
734         return ERR_PTR(rc);
735 }
736 
737 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
738 {
739         return kvm_cpu_has_interrupt(vcpu);
740 }
741 
742 void s390_vcpu_block(struct kvm_vcpu *vcpu)
743 {
744         atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
745 }
746 
747 void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
748 {
749         atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
750 }
751 
752 /*
753  * Kick a guest cpu out of SIE and wait until SIE is not running.
754  * If the CPU is not running (e.g. waiting as idle) the function will
755  * return immediately. */
756 void exit_sie(struct kvm_vcpu *vcpu)
757 {
758         atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
759         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
760                 cpu_relax();
761 }
762 
763 /* Kick a guest cpu out of SIE and prevent SIE-reentry */
764 void exit_sie_sync(struct kvm_vcpu *vcpu)
765 {
766         s390_vcpu_block(vcpu);
767         exit_sie(vcpu);
768 }
769 
770 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
771 {
772         int i;
773         struct kvm *kvm = gmap->private;
774         struct kvm_vcpu *vcpu;
775 
776         kvm_for_each_vcpu(i, vcpu, kvm) {
777                 /* match against both prefix pages */
778                 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
779                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
780                         kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
781                         exit_sie_sync(vcpu);
782                 }
783         }
784 }
785 
786 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
787 {
788         /* kvm common code refers to this, but never calls it */
789         BUG();
790         return 0;
791 }
792 
793 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
794                                            struct kvm_one_reg *reg)
795 {
796         int r = -EINVAL;
797 
798         switch (reg->id) {
799         case KVM_REG_S390_TODPR:
800                 r = put_user(vcpu->arch.sie_block->todpr,
801                              (u32 __user *)reg->addr);
802                 break;
803         case KVM_REG_S390_EPOCHDIFF:
804                 r = put_user(vcpu->arch.sie_block->epoch,
805                              (u64 __user *)reg->addr);
806                 break;
807         case KVM_REG_S390_CPU_TIMER:
808                 r = put_user(vcpu->arch.sie_block->cputm,
809                              (u64 __user *)reg->addr);
810                 break;
811         case KVM_REG_S390_CLOCK_COMP:
812                 r = put_user(vcpu->arch.sie_block->ckc,
813                              (u64 __user *)reg->addr);
814                 break;
815         case KVM_REG_S390_PFTOKEN:
816                 r = put_user(vcpu->arch.pfault_token,
817                              (u64 __user *)reg->addr);
818                 break;
819         case KVM_REG_S390_PFCOMPARE:
820                 r = put_user(vcpu->arch.pfault_compare,
821                              (u64 __user *)reg->addr);
822                 break;
823         case KVM_REG_S390_PFSELECT:
824                 r = put_user(vcpu->arch.pfault_select,
825                              (u64 __user *)reg->addr);
826                 break;
827         case KVM_REG_S390_PP:
828                 r = put_user(vcpu->arch.sie_block->pp,
829                              (u64 __user *)reg->addr);
830                 break;
831         case KVM_REG_S390_GBEA:
832                 r = put_user(vcpu->arch.sie_block->gbea,
833                              (u64 __user *)reg->addr);
834                 break;
835         default:
836                 break;
837         }
838 
839         return r;
840 }
841 
842 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
843                                            struct kvm_one_reg *reg)
844 {
845         int r = -EINVAL;
846 
847         switch (reg->id) {
848         case KVM_REG_S390_TODPR:
849                 r = get_user(vcpu->arch.sie_block->todpr,
850                              (u32 __user *)reg->addr);
851                 break;
852         case KVM_REG_S390_EPOCHDIFF:
853                 r = get_user(vcpu->arch.sie_block->epoch,
854                              (u64 __user *)reg->addr);
855                 break;
856         case KVM_REG_S390_CPU_TIMER:
857                 r = get_user(vcpu->arch.sie_block->cputm,
858                              (u64 __user *)reg->addr);
859                 break;
860         case KVM_REG_S390_CLOCK_COMP:
861                 r = get_user(vcpu->arch.sie_block->ckc,
862                              (u64 __user *)reg->addr);
863                 break;
864         case KVM_REG_S390_PFTOKEN:
865                 r = get_user(vcpu->arch.pfault_token,
866                              (u64 __user *)reg->addr);
867                 break;
868         case KVM_REG_S390_PFCOMPARE:
869                 r = get_user(vcpu->arch.pfault_compare,
870                              (u64 __user *)reg->addr);
871                 break;
872         case KVM_REG_S390_PFSELECT:
873                 r = get_user(vcpu->arch.pfault_select,
874                              (u64 __user *)reg->addr);
875                 break;
876         case KVM_REG_S390_PP:
877                 r = get_user(vcpu->arch.sie_block->pp,
878                              (u64 __user *)reg->addr);
879                 break;
880         case KVM_REG_S390_GBEA:
881                 r = get_user(vcpu->arch.sie_block->gbea,
882                              (u64 __user *)reg->addr);
883                 break;
884         default:
885                 break;
886         }
887 
888         return r;
889 }
890 
891 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
892 {
893         kvm_s390_vcpu_initial_reset(vcpu);
894         return 0;
895 }
896 
897 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
898 {
899         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
900         return 0;
901 }
902 
903 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
904 {
905         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
906         return 0;
907 }
908 
909 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
910                                   struct kvm_sregs *sregs)
911 {
912         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
913         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
914         restore_access_regs(vcpu->run->s.regs.acrs);
915         return 0;
916 }
917 
918 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
919                                   struct kvm_sregs *sregs)
920 {
921         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
922         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
923         return 0;
924 }
925 
926 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
927 {
928         if (test_fp_ctl(fpu->fpc))
929                 return -EINVAL;
930         memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
931         vcpu->arch.guest_fpregs.fpc = fpu->fpc;
932         restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
933         restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
934         return 0;
935 }
936 
937 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
938 {
939         memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
940         fpu->fpc = vcpu->arch.guest_fpregs.fpc;
941         return 0;
942 }
943 
944 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
945 {
946         int rc = 0;
947 
948         if (!is_vcpu_stopped(vcpu))
949                 rc = -EBUSY;
950         else {
951                 vcpu->run->psw_mask = psw.mask;
952                 vcpu->run->psw_addr = psw.addr;
953         }
954         return rc;
955 }
956 
957 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
958                                   struct kvm_translation *tr)
959 {
960         return -EINVAL; /* not implemented yet */
961 }
962 
963 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
964                               KVM_GUESTDBG_USE_HW_BP | \
965                               KVM_GUESTDBG_ENABLE)
966 
967 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
968                                         struct kvm_guest_debug *dbg)
969 {
970         int rc = 0;
971 
972         vcpu->guest_debug = 0;
973         kvm_s390_clear_bp_data(vcpu);
974 
975         if (dbg->control & ~VALID_GUESTDBG_FLAGS)
976                 return -EINVAL;
977 
978         if (dbg->control & KVM_GUESTDBG_ENABLE) {
979                 vcpu->guest_debug = dbg->control;
980                 /* enforce guest PER */
981                 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
982 
983                 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
984                         rc = kvm_s390_import_bp_data(vcpu, dbg);
985         } else {
986                 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
987                 vcpu->arch.guestdbg.last_bp = 0;
988         }
989 
990         if (rc) {
991                 vcpu->guest_debug = 0;
992                 kvm_s390_clear_bp_data(vcpu);
993                 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
994         }
995 
996         return rc;
997 }
998 
999 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1000                                     struct kvm_mp_state *mp_state)
1001 {
1002         /* CHECK_STOP and LOAD are not supported yet */
1003         return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1004                                        KVM_MP_STATE_OPERATING;
1005 }
1006 
1007 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1008                                     struct kvm_mp_state *mp_state)
1009 {
1010         int rc = 0;
1011 
1012         /* user space knows about this interface - let it control the state */
1013         vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1014 
1015         switch (mp_state->mp_state) {
1016         case KVM_MP_STATE_STOPPED:
1017                 kvm_s390_vcpu_stop(vcpu);
1018                 break;
1019         case KVM_MP_STATE_OPERATING:
1020                 kvm_s390_vcpu_start(vcpu);
1021                 break;
1022         case KVM_MP_STATE_LOAD:
1023         case KVM_MP_STATE_CHECK_STOP:
1024                 /* fall through - CHECK_STOP and LOAD are not supported yet */
1025         default:
1026                 rc = -ENXIO;
1027         }
1028 
1029         return rc;
1030 }
1031 
1032 bool kvm_s390_cmma_enabled(struct kvm *kvm)
1033 {
1034         if (!MACHINE_IS_LPAR)
1035                 return false;
1036         /* only enable for z10 and later */
1037         if (!MACHINE_HAS_EDAT1)
1038                 return false;
1039         if (!kvm->arch.use_cmma)
1040                 return false;
1041         return true;
1042 }
1043 
1044 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1045 {
1046         return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1047 }
1048 
1049 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1050 {
1051 retry:
1052         s390_vcpu_unblock(vcpu);
1053         /*
1054          * We use MMU_RELOAD just to re-arm the ipte notifier for the
1055          * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1056          * This ensures that the ipte instruction for this request has
1057          * already finished. We might race against a second unmapper that
1058          * wants to set the blocking bit. Lets just retry the request loop.
1059          */
1060         if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1061                 int rc;
1062                 rc = gmap_ipte_notify(vcpu->arch.gmap,
1063                                       kvm_s390_get_prefix(vcpu),
1064                                       PAGE_SIZE * 2);
1065                 if (rc)
1066                         return rc;
1067                 goto retry;
1068         }
1069 
1070         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1071                 vcpu->arch.sie_block->ihcpu = 0xffff;
1072                 goto retry;
1073         }
1074 
1075         if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1076                 if (!ibs_enabled(vcpu)) {
1077                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1078                         atomic_set_mask(CPUSTAT_IBS,
1079                                         &vcpu->arch.sie_block->cpuflags);
1080                 }
1081                 goto retry;
1082         }
1083 
1084         if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1085                 if (ibs_enabled(vcpu)) {
1086                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1087                         atomic_clear_mask(CPUSTAT_IBS,
1088                                           &vcpu->arch.sie_block->cpuflags);
1089                 }
1090                 goto retry;
1091         }
1092 
1093         /* nothing to do, just clear the request */
1094         clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1095 
1096         return 0;
1097 }
1098 
1099 /**
1100  * kvm_arch_fault_in_page - fault-in guest page if necessary
1101  * @vcpu: The corresponding virtual cpu
1102  * @gpa: Guest physical address
1103  * @writable: Whether the page should be writable or not
1104  *
1105  * Make sure that a guest page has been faulted-in on the host.
1106  *
1107  * Return: Zero on success, negative error code otherwise.
1108  */
1109 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
1110 {
1111         return gmap_fault(vcpu->arch.gmap, gpa,
1112                           writable ? FAULT_FLAG_WRITE : 0);
1113 }
1114 
1115 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1116                                       unsigned long token)
1117 {
1118         struct kvm_s390_interrupt inti;
1119         inti.parm64 = token;
1120 
1121         if (start_token) {
1122                 inti.type = KVM_S390_INT_PFAULT_INIT;
1123                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &inti));
1124         } else {
1125                 inti.type = KVM_S390_INT_PFAULT_DONE;
1126                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1127         }
1128 }
1129 
1130 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1131                                      struct kvm_async_pf *work)
1132 {
1133         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1134         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1135 }
1136 
1137 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1138                                  struct kvm_async_pf *work)
1139 {
1140         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1141         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1142 }
1143 
1144 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1145                                struct kvm_async_pf *work)
1146 {
1147         /* s390 will always inject the page directly */
1148 }
1149 
1150 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1151 {
1152         /*
1153          * s390 will always inject the page directly,
1154          * but we still want check_async_completion to cleanup
1155          */
1156         return true;
1157 }
1158 
1159 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1160 {
1161         hva_t hva;
1162         struct kvm_arch_async_pf arch;
1163         int rc;
1164 
1165         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1166                 return 0;
1167         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1168             vcpu->arch.pfault_compare)
1169                 return 0;
1170         if (psw_extint_disabled(vcpu))
1171                 return 0;
1172         if (kvm_cpu_has_interrupt(vcpu))
1173                 return 0;
1174         if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1175                 return 0;
1176         if (!vcpu->arch.gmap->pfault_enabled)
1177                 return 0;
1178 
1179         hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1180         hva += current->thread.gmap_addr & ~PAGE_MASK;
1181         if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
1182                 return 0;
1183 
1184         rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1185         return rc;
1186 }
1187 
1188 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
1189 {
1190         int rc, cpuflags;
1191 
1192         /*
1193          * On s390 notifications for arriving pages will be delivered directly
1194          * to the guest but the house keeping for completed pfaults is
1195          * handled outside the worker.
1196          */
1197         kvm_check_async_pf_completion(vcpu);
1198 
1199         memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
1200 
1201         if (need_resched())
1202                 schedule();
1203 
1204         if (test_cpu_flag(CIF_MCCK_PENDING))
1205                 s390_handle_mcck();
1206 
1207         if (!kvm_is_ucontrol(vcpu->kvm)) {
1208                 rc = kvm_s390_deliver_pending_interrupts(vcpu);
1209                 if (rc)
1210                         return rc;
1211         }
1212 
1213         rc = kvm_s390_handle_requests(vcpu);
1214         if (rc)
1215                 return rc;
1216 
1217         if (guestdbg_enabled(vcpu)) {
1218                 kvm_s390_backup_guest_per_regs(vcpu);
1219                 kvm_s390_patch_guest_per_regs(vcpu);
1220         }
1221 
1222         vcpu->arch.sie_block->icptcode = 0;
1223         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1224         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1225         trace_kvm_s390_sie_enter(vcpu, cpuflags);
1226 
1227         return 0;
1228 }
1229 
1230 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1231 {
1232         int rc = -1;
1233 
1234         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1235                    vcpu->arch.sie_block->icptcode);
1236         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1237 
1238         if (guestdbg_enabled(vcpu))
1239                 kvm_s390_restore_guest_per_regs(vcpu);
1240 
1241         if (exit_reason >= 0) {
1242                 rc = 0;
1243         } else if (kvm_is_ucontrol(vcpu->kvm)) {
1244                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1245                 vcpu->run->s390_ucontrol.trans_exc_code =
1246                                                 current->thread.gmap_addr;
1247                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1248                 rc = -EREMOTE;
1249 
1250         } else if (current->thread.gmap_pfault) {
1251                 trace_kvm_s390_major_guest_pfault(vcpu);
1252                 current->thread.gmap_pfault = 0;
1253                 if (kvm_arch_setup_async_pf(vcpu)) {
1254                         rc = 0;
1255                 } else {
1256                         gpa_t gpa = current->thread.gmap_addr;
1257                         rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1258                 }
1259         }
1260 
1261         if (rc == -1) {
1262                 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1263                 trace_kvm_s390_sie_fault(vcpu);
1264                 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1265         }
1266 
1267         memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
1268 
1269         if (rc == 0) {
1270                 if (kvm_is_ucontrol(vcpu->kvm))
1271                         /* Don't exit for host interrupts. */
1272                         rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
1273                 else
1274                         rc = kvm_handle_sie_intercept(vcpu);
1275         }
1276 
1277         return rc;
1278 }
1279 
1280 static int __vcpu_run(struct kvm_vcpu *vcpu)
1281 {
1282         int rc, exit_reason;
1283 
1284         /*
1285          * We try to hold kvm->srcu during most of vcpu_run (except when run-
1286          * ning the guest), so that memslots (and other stuff) are protected
1287          */
1288         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1289 
1290         do {
1291                 rc = vcpu_pre_run(vcpu);
1292                 if (rc)
1293                         break;
1294 
1295                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1296                 /*
1297                  * As PF_VCPU will be used in fault handler, between
1298                  * guest_enter and guest_exit should be no uaccess.
1299                  */
1300                 preempt_disable();
1301                 kvm_guest_enter();
1302                 preempt_enable();
1303                 exit_reason = sie64a(vcpu->arch.sie_block,
1304                                      vcpu->run->s.regs.gprs);
1305                 kvm_guest_exit();
1306                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1307 
1308                 rc = vcpu_post_run(vcpu, exit_reason);
1309         } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
1310 
1311         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1312         return rc;
1313 }
1314 
1315 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1316 {
1317         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1318         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
1319         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
1320                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1321         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1322                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
1323                 /* some control register changes require a tlb flush */
1324                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1325         }
1326         if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
1327                 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
1328                 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
1329                 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
1330                 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
1331                 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
1332         }
1333         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
1334                 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
1335                 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
1336                 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
1337         }
1338         kvm_run->kvm_dirty_regs = 0;
1339 }
1340 
1341 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1342 {
1343         kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
1344         kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
1345         kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
1346         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
1347         kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
1348         kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
1349         kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
1350         kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
1351         kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
1352         kvm_run->s.regs.pft = vcpu->arch.pfault_token;
1353         kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
1354         kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
1355 }
1356 
1357 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1358 {
1359         int rc;
1360         sigset_t sigsaved;
1361 
1362         if (guestdbg_exit_pending(vcpu)) {
1363                 kvm_s390_prepare_debug_exit(vcpu);
1364                 return 0;
1365         }
1366 
1367         if (vcpu->sigset_active)
1368                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1369 
1370         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
1371                 kvm_s390_vcpu_start(vcpu);
1372         } else if (is_vcpu_stopped(vcpu)) {
1373                 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
1374                                    vcpu->vcpu_id);
1375                 return -EINVAL;
1376         }
1377 
1378         sync_regs(vcpu, kvm_run);
1379 
1380         might_fault();
1381         rc = __vcpu_run(vcpu);
1382 
1383         if (signal_pending(current) && !rc) {
1384                 kvm_run->exit_reason = KVM_EXIT_INTR;
1385                 rc = -EINTR;
1386         }
1387 
1388         if (guestdbg_exit_pending(vcpu) && !rc)  {
1389                 kvm_s390_prepare_debug_exit(vcpu);
1390                 rc = 0;
1391         }
1392 
1393         if (rc == -EOPNOTSUPP) {
1394                 /* intercept cannot be handled in-kernel, prepare kvm-run */
1395                 kvm_run->exit_reason         = KVM_EXIT_S390_SIEIC;
1396                 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
1397                 kvm_run->s390_sieic.ipa      = vcpu->arch.sie_block->ipa;
1398                 kvm_run->s390_sieic.ipb      = vcpu->arch.sie_block->ipb;
1399                 rc = 0;
1400         }
1401 
1402         if (rc == -EREMOTE) {
1403                 /* intercept was handled, but userspace support is needed
1404                  * kvm_run has been prepared by the handler */
1405                 rc = 0;
1406         }
1407 
1408         store_regs(vcpu, kvm_run);
1409 
1410         if (vcpu->sigset_active)
1411                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1412 
1413         vcpu->stat.exit_userspace++;
1414         return rc;
1415 }
1416 
1417 /*
1418  * store status at address
1419  * we use have two special cases:
1420  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1421  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1422  */
1423 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
1424 {
1425         unsigned char archmode = 1;
1426         unsigned int px;
1427         u64 clkcomp;
1428         int rc;
1429 
1430         if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
1431                 if (write_guest_abs(vcpu, 163, &archmode, 1))
1432                         return -EFAULT;
1433                 gpa = SAVE_AREA_BASE;
1434         } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
1435                 if (write_guest_real(vcpu, 163, &archmode, 1))
1436                         return -EFAULT;
1437                 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
1438         }
1439         rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
1440                              vcpu->arch.guest_fpregs.fprs, 128);
1441         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
1442                               vcpu->run->s.regs.gprs, 128);
1443         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
1444                               &vcpu->arch.sie_block->gpsw, 16);
1445         px = kvm_s390_get_prefix(vcpu);
1446         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
1447                               &px, 4);
1448         rc |= write_guest_abs(vcpu,
1449                               gpa + offsetof(struct save_area, fp_ctrl_reg),
1450                               &vcpu->arch.guest_fpregs.fpc, 4);
1451         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
1452                               &vcpu->arch.sie_block->todpr, 4);
1453         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
1454                               &vcpu->arch.sie_block->cputm, 8);
1455         clkcomp = vcpu->arch.sie_block->ckc >> 8;
1456         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
1457                               &clkcomp, 8);
1458         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
1459                               &vcpu->run->s.regs.acrs, 64);
1460         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
1461                               &vcpu->arch.sie_block->gcr, 128);
1462         return rc ? -EFAULT : 0;
1463 }
1464 
1465 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1466 {
1467         /*
1468          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1469          * copying in vcpu load/put. Lets update our copies before we save
1470          * it into the save area
1471          */
1472         save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1473         save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1474         save_access_regs(vcpu->run->s.regs.acrs);
1475 
1476         return kvm_s390_store_status_unloaded(vcpu, addr);
1477 }
1478 
1479 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1480 {
1481         kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
1482         kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu);
1483         exit_sie_sync(vcpu);
1484 }
1485 
1486 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
1487 {
1488         unsigned int i;
1489         struct kvm_vcpu *vcpu;
1490 
1491         kvm_for_each_vcpu(i, vcpu, kvm) {
1492                 __disable_ibs_on_vcpu(vcpu);
1493         }
1494 }
1495 
1496 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1497 {
1498         kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
1499         kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu);
1500         exit_sie_sync(vcpu);
1501 }
1502 
1503 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
1504 {
1505         int i, online_vcpus, started_vcpus = 0;
1506 
1507         if (!is_vcpu_stopped(vcpu))
1508                 return;
1509 
1510         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
1511         /* Only one cpu at a time may enter/leave the STOPPED state. */
1512         spin_lock(&vcpu->kvm->arch.start_stop_lock);
1513         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1514 
1515         for (i = 0; i < online_vcpus; i++) {
1516                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
1517                         started_vcpus++;
1518         }
1519 
1520         if (started_vcpus == 0) {
1521                 /* we're the only active VCPU -> speed it up */
1522                 __enable_ibs_on_vcpu(vcpu);
1523         } else if (started_vcpus == 1) {
1524                 /*
1525                  * As we are starting a second VCPU, we have to disable
1526                  * the IBS facility on all VCPUs to remove potentially
1527                  * oustanding ENABLE requests.
1528                  */
1529                 __disable_ibs_on_all_vcpus(vcpu->kvm);
1530         }
1531 
1532         atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
1533         /*
1534          * Another VCPU might have used IBS while we were offline.
1535          * Let's play safe and flush the VCPU at startup.
1536          */
1537         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1538         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
1539         return;
1540 }
1541 
1542 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
1543 {
1544         int i, online_vcpus, started_vcpus = 0;
1545         struct kvm_vcpu *started_vcpu = NULL;
1546 
1547         if (is_vcpu_stopped(vcpu))
1548                 return;
1549 
1550         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
1551         /* Only one cpu at a time may enter/leave the STOPPED state. */
1552         spin_lock(&vcpu->kvm->arch.start_stop_lock);
1553         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1554 
1555         /* Need to lock access to action_bits to avoid a SIGP race condition */
1556         spin_lock(&vcpu->arch.local_int.lock);
1557         atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
1558 
1559         /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
1560         vcpu->arch.local_int.action_bits &=
1561                                  ~(ACTION_STOP_ON_STOP | ACTION_STORE_ON_STOP);
1562         spin_unlock(&vcpu->arch.local_int.lock);
1563 
1564         __disable_ibs_on_vcpu(vcpu);
1565 
1566         for (i = 0; i < online_vcpus; i++) {
1567                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
1568                         started_vcpus++;
1569                         started_vcpu = vcpu->kvm->vcpus[i];
1570                 }
1571         }
1572 
1573         if (started_vcpus == 1) {
1574                 /*
1575                  * As we only have one VCPU left, we want to enable the
1576                  * IBS facility for that VCPU to speed it up.
1577                  */
1578                 __enable_ibs_on_vcpu(started_vcpu);
1579         }
1580 
1581         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
1582         return;
1583 }
1584 
1585 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1586                                      struct kvm_enable_cap *cap)
1587 {
1588         int r;
1589 
1590         if (cap->flags)
1591                 return -EINVAL;
1592 
1593         switch (cap->cap) {
1594         case KVM_CAP_S390_CSS_SUPPORT:
1595                 if (!vcpu->kvm->arch.css_support) {
1596                         vcpu->kvm->arch.css_support = 1;
1597                         trace_kvm_s390_enable_css(vcpu->kvm);
1598                 }
1599                 r = 0;
1600                 break;
1601         default:
1602                 r = -EINVAL;
1603                 break;
1604         }
1605         return r;
1606 }
1607 
1608 long kvm_arch_vcpu_ioctl(struct file *filp,
1609                          unsigned int ioctl, unsigned long arg)
1610 {
1611         struct kvm_vcpu *vcpu = filp->private_data;
1612         void __user *argp = (void __user *)arg;
1613         int idx;
1614         long r;
1615 
1616         switch (ioctl) {
1617         case KVM_S390_INTERRUPT: {
1618                 struct kvm_s390_interrupt s390int;
1619 
1620                 r = -EFAULT;
1621                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1622                         break;
1623                 r = kvm_s390_inject_vcpu(vcpu, &s390int);
1624                 break;
1625         }
1626         case KVM_S390_STORE_STATUS:
1627                 idx = srcu_read_lock(&vcpu->kvm->srcu);
1628                 r = kvm_s390_vcpu_store_status(vcpu, arg);
1629                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
1630                 break;
1631         case KVM_S390_SET_INITIAL_PSW: {
1632                 psw_t psw;
1633 
1634                 r = -EFAULT;
1635                 if (copy_from_user(&psw, argp, sizeof(psw)))
1636                         break;
1637                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
1638                 break;
1639         }
1640         case KVM_S390_INITIAL_RESET:
1641                 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
1642                 break;
1643         case KVM_SET_ONE_REG:
1644         case KVM_GET_ONE_REG: {
1645                 struct kvm_one_reg reg;
1646                 r = -EFAULT;
1647                 if (copy_from_user(&reg, argp, sizeof(reg)))
1648                         break;
1649                 if (ioctl == KVM_SET_ONE_REG)
1650                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
1651                 else
1652                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
1653                 break;
1654         }
1655 #ifdef CONFIG_KVM_S390_UCONTROL
1656         case KVM_S390_UCAS_MAP: {
1657                 struct kvm_s390_ucas_mapping ucasmap;
1658 
1659                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1660                         r = -EFAULT;
1661                         break;
1662                 }
1663 
1664                 if (!kvm_is_ucontrol(vcpu->kvm)) {
1665                         r = -EINVAL;
1666                         break;
1667                 }
1668 
1669                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
1670                                      ucasmap.vcpu_addr, ucasmap.length);
1671                 break;
1672         }
1673         case KVM_S390_UCAS_UNMAP: {
1674                 struct kvm_s390_ucas_mapping ucasmap;
1675 
1676                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1677                         r = -EFAULT;
1678                         break;
1679                 }
1680 
1681                 if (!kvm_is_ucontrol(vcpu->kvm)) {
1682                         r = -EINVAL;
1683                         break;
1684                 }
1685 
1686                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
1687                         ucasmap.length);
1688                 break;
1689         }
1690 #endif
1691         case KVM_S390_VCPU_FAULT: {
1692                 r = gmap_fault(vcpu->arch.gmap, arg, 0);
1693                 break;
1694         }
1695         case KVM_ENABLE_CAP:
1696         {
1697                 struct kvm_enable_cap cap;
1698                 r = -EFAULT;
1699                 if (copy_from_user(&cap, argp, sizeof(cap)))
1700                         break;
1701                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1702                 break;
1703         }
1704         default:
1705                 r = -ENOTTY;
1706         }
1707         return r;
1708 }
1709 
1710 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
1711 {
1712 #ifdef CONFIG_KVM_S390_UCONTROL
1713         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
1714                  && (kvm_is_ucontrol(vcpu->kvm))) {
1715                 vmf->page = virt_to_page(vcpu->arch.sie_block);
1716                 get_page(vmf->page);
1717                 return 0;
1718         }
1719 #endif
1720         return VM_FAULT_SIGBUS;
1721 }
1722 
1723 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
1724                             unsigned long npages)
1725 {
1726         return 0;
1727 }
1728 
1729 /* Section: memory related */
1730 int kvm_arch_prepare_memory_region(struct kvm *kvm,
1731                                    struct kvm_memory_slot *memslot,
1732                                    struct kvm_userspace_memory_region *mem,
1733                                    enum kvm_mr_change change)
1734 {
1735         /* A few sanity checks. We can have memory slots which have to be
1736            located/ended at a segment boundary (1MB). The memory in userland is
1737            ok to be fragmented into various different vmas. It is okay to mmap()
1738            and munmap() stuff in this slot after doing this call at any time */
1739 
1740         if (mem->userspace_addr & 0xffffful)
1741                 return -EINVAL;
1742 
1743         if (mem->memory_size & 0xffffful)
1744                 return -EINVAL;
1745 
1746         return 0;
1747 }
1748 
1749 void kvm_arch_commit_memory_region(struct kvm *kvm,
1750                                 struct kvm_userspace_memory_region *mem,
1751                                 const struct kvm_memory_slot *old,
1752                                 enum kvm_mr_change change)
1753 {
1754         int rc;
1755 
1756         /* If the basics of the memslot do not change, we do not want
1757          * to update the gmap. Every update causes several unnecessary
1758          * segment translation exceptions. This is usually handled just
1759          * fine by the normal fault handler + gmap, but it will also
1760          * cause faults on the prefix page of running guest CPUs.
1761          */
1762         if (old->userspace_addr == mem->userspace_addr &&
1763             old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
1764             old->npages * PAGE_SIZE == mem->memory_size)
1765                 return;
1766 
1767         rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
1768                 mem->guest_phys_addr, mem->memory_size);
1769         if (rc)
1770                 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
1771         return;
1772 }
1773 
1774 static int __init kvm_s390_init(void)
1775 {
1776         int ret;
1777         ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
1778         if (ret)
1779                 return ret;
1780 
1781         /*
1782          * guests can ask for up to 255+1 double words, we need a full page
1783          * to hold the maximum amount of facilities. On the other hand, we
1784          * only set facilities that are known to work in KVM.
1785          */
1786         vfacilities = (unsigned long *) get_zeroed_page(GFP_KERNEL|GFP_DMA);
1787         if (!vfacilities) {
1788                 kvm_exit();
1789                 return -ENOMEM;
1790         }
1791         memcpy(vfacilities, S390_lowcore.stfle_fac_list, 16);
1792         vfacilities[0] &= 0xff82fffbf47c2000UL;
1793         vfacilities[1] &= 0x005c000000000000UL;
1794         return 0;
1795 }
1796 
1797 static void __exit kvm_s390_exit(void)
1798 {
1799         free_page((unsigned long) vfacilities);
1800         kvm_exit();
1801 }
1802 
1803 module_init(kvm_s390_init);
1804 module_exit(kvm_s390_exit);
1805 
1806 /*
1807  * Enable autoloading of the kvm module.
1808  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
1809  * since x86 takes a different approach.
1810  */
1811 #include <linux/miscdevice.h>
1812 MODULE_ALIAS_MISCDEV(KVM_MINOR);
1813 MODULE_ALIAS("devname:kvm");
1814 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp