~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/s390/kvm/kvm-s390.c

Version: ~ [ linux-5.12 ] ~ [ linux-5.11.16 ] ~ [ linux-5.10.32 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.114 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.188 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.231 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.267 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.267 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  * hosting IBM Z kernel virtual machines (s390x)
  4  *
  5  * Copyright IBM Corp. 2008, 2017
  6  *
  7  *    Author(s): Carsten Otte <cotte@de.ibm.com>
  8  *               Christian Borntraeger <borntraeger@de.ibm.com>
  9  *               Heiko Carstens <heiko.carstens@de.ibm.com>
 10  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
 11  *               Jason J. Herne <jjherne@us.ibm.com>
 12  */
 13 
 14 #include <linux/compiler.h>
 15 #include <linux/err.h>
 16 #include <linux/fs.h>
 17 #include <linux/hrtimer.h>
 18 #include <linux/init.h>
 19 #include <linux/kvm.h>
 20 #include <linux/kvm_host.h>
 21 #include <linux/mman.h>
 22 #include <linux/module.h>
 23 #include <linux/moduleparam.h>
 24 #include <linux/random.h>
 25 #include <linux/slab.h>
 26 #include <linux/timer.h>
 27 #include <linux/vmalloc.h>
 28 #include <linux/bitmap.h>
 29 #include <linux/sched/signal.h>
 30 #include <linux/string.h>
 31 
 32 #include <asm/asm-offsets.h>
 33 #include <asm/lowcore.h>
 34 #include <asm/stp.h>
 35 #include <asm/pgtable.h>
 36 #include <asm/gmap.h>
 37 #include <asm/nmi.h>
 38 #include <asm/switch_to.h>
 39 #include <asm/isc.h>
 40 #include <asm/sclp.h>
 41 #include <asm/cpacf.h>
 42 #include <asm/timex.h>
 43 #include "kvm-s390.h"
 44 #include "gaccess.h"
 45 
 46 #define KMSG_COMPONENT "kvm-s390"
 47 #undef pr_fmt
 48 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
 49 
 50 #define CREATE_TRACE_POINTS
 51 #include "trace.h"
 52 #include "trace-s390.h"
 53 
 54 #define MEM_OP_MAX_SIZE 65536   /* Maximum transfer size for KVM_S390_MEM_OP */
 55 #define LOCAL_IRQS 32
 56 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
 57                            (KVM_MAX_VCPUS + LOCAL_IRQS))
 58 
 59 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
 60 
 61 struct kvm_stats_debugfs_item debugfs_entries[] = {
 62         { "userspace_handled", VCPU_STAT(exit_userspace) },
 63         { "exit_null", VCPU_STAT(exit_null) },
 64         { "exit_validity", VCPU_STAT(exit_validity) },
 65         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
 66         { "exit_external_request", VCPU_STAT(exit_external_request) },
 67         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
 68         { "exit_instruction", VCPU_STAT(exit_instruction) },
 69         { "exit_pei", VCPU_STAT(exit_pei) },
 70         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
 71         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
 72         { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
 73         { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
 74         { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
 75         { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
 76         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
 77         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
 78         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
 79         { "instruction_stctl", VCPU_STAT(instruction_stctl) },
 80         { "instruction_stctg", VCPU_STAT(instruction_stctg) },
 81         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
 82         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
 83         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
 84         { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
 85         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
 86         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
 87         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
 88         { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
 89         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
 90         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
 91         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
 92         { "instruction_spx", VCPU_STAT(instruction_spx) },
 93         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
 94         { "instruction_stap", VCPU_STAT(instruction_stap) },
 95         { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
 96         { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
 97         { "instruction_stsch", VCPU_STAT(instruction_stsch) },
 98         { "instruction_chsc", VCPU_STAT(instruction_chsc) },
 99         { "instruction_essa", VCPU_STAT(instruction_essa) },
100         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
101         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
102         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
103         { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
104         { "instruction_sie", VCPU_STAT(instruction_sie) },
105         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
106         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
107         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
108         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
109         { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
110         { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
111         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
112         { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
113         { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
114         { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
115         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
116         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
117         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
118         { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
119         { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
120         { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
121         { "diagnose_10", VCPU_STAT(diagnose_10) },
122         { "diagnose_44", VCPU_STAT(diagnose_44) },
123         { "diagnose_9c", VCPU_STAT(diagnose_9c) },
124         { "diagnose_258", VCPU_STAT(diagnose_258) },
125         { "diagnose_308", VCPU_STAT(diagnose_308) },
126         { "diagnose_500", VCPU_STAT(diagnose_500) },
127         { NULL }
128 };
129 
130 struct kvm_s390_tod_clock_ext {
131         __u8 epoch_idx;
132         __u64 tod;
133         __u8 reserved[7];
134 } __packed;
135 
136 /* allow nested virtualization in KVM (if enabled by user space) */
137 static int nested;
138 module_param(nested, int, S_IRUGO);
139 MODULE_PARM_DESC(nested, "Nested virtualization support");
140 
141 /* upper facilities limit for kvm */
142 unsigned long kvm_s390_fac_list_mask[16] = { FACILITIES_KVM };
143 
144 unsigned long kvm_s390_fac_list_mask_size(void)
145 {
146         BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
147         return ARRAY_SIZE(kvm_s390_fac_list_mask);
148 }
149 
150 /* available cpu features supported by kvm */
151 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
152 /* available subfunctions indicated via query / "test bit" */
153 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
154 
155 static struct gmap_notifier gmap_notifier;
156 static struct gmap_notifier vsie_gmap_notifier;
157 debug_info_t *kvm_s390_dbf;
158 
159 /* Section: not file related */
160 int kvm_arch_hardware_enable(void)
161 {
162         /* every s390 is virtualization enabled ;-) */
163         return 0;
164 }
165 
166 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
167                               unsigned long end);
168 
169 static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
170 {
171         u8 delta_idx = 0;
172 
173         /*
174          * The TOD jumps by delta, we have to compensate this by adding
175          * -delta to the epoch.
176          */
177         delta = -delta;
178 
179         /* sign-extension - we're adding to signed values below */
180         if ((s64)delta < 0)
181                 delta_idx = -1;
182 
183         scb->epoch += delta;
184         if (scb->ecd & ECD_MEF) {
185                 scb->epdx += delta_idx;
186                 if (scb->epoch < delta)
187                         scb->epdx += 1;
188         }
189 }
190 
191 /*
192  * This callback is executed during stop_machine(). All CPUs are therefore
193  * temporarily stopped. In order not to change guest behavior, we have to
194  * disable preemption whenever we touch the epoch of kvm and the VCPUs,
195  * so a CPU won't be stopped while calculating with the epoch.
196  */
197 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
198                           void *v)
199 {
200         struct kvm *kvm;
201         struct kvm_vcpu *vcpu;
202         int i;
203         unsigned long long *delta = v;
204 
205         list_for_each_entry(kvm, &vm_list, vm_list) {
206                 kvm_for_each_vcpu(i, vcpu, kvm) {
207                         kvm_clock_sync_scb(vcpu->arch.sie_block, *delta);
208                         if (i == 0) {
209                                 kvm->arch.epoch = vcpu->arch.sie_block->epoch;
210                                 kvm->arch.epdx = vcpu->arch.sie_block->epdx;
211                         }
212                         if (vcpu->arch.cputm_enabled)
213                                 vcpu->arch.cputm_start += *delta;
214                         if (vcpu->arch.vsie_block)
215                                 kvm_clock_sync_scb(vcpu->arch.vsie_block,
216                                                    *delta);
217                 }
218         }
219         return NOTIFY_OK;
220 }
221 
222 static struct notifier_block kvm_clock_notifier = {
223         .notifier_call = kvm_clock_sync,
224 };
225 
226 int kvm_arch_hardware_setup(void)
227 {
228         gmap_notifier.notifier_call = kvm_gmap_notifier;
229         gmap_register_pte_notifier(&gmap_notifier);
230         vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
231         gmap_register_pte_notifier(&vsie_gmap_notifier);
232         atomic_notifier_chain_register(&s390_epoch_delta_notifier,
233                                        &kvm_clock_notifier);
234         return 0;
235 }
236 
237 void kvm_arch_hardware_unsetup(void)
238 {
239         gmap_unregister_pte_notifier(&gmap_notifier);
240         gmap_unregister_pte_notifier(&vsie_gmap_notifier);
241         atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
242                                          &kvm_clock_notifier);
243 }
244 
245 static void allow_cpu_feat(unsigned long nr)
246 {
247         set_bit_inv(nr, kvm_s390_available_cpu_feat);
248 }
249 
250 static inline int plo_test_bit(unsigned char nr)
251 {
252         register unsigned long r0 asm("") = (unsigned long) nr | 0x100;
253         int cc;
254 
255         asm volatile(
256                 /* Parameter registers are ignored for "test bit" */
257                 "       plo     0,0,0,0(0)\n"
258                 "       ipm     %0\n"
259                 "       srl     %0,28\n"
260                 : "=d" (cc)
261                 : "d" (r0)
262                 : "cc");
263         return cc == 0;
264 }
265 
266 static void kvm_s390_cpu_feat_init(void)
267 {
268         int i;
269 
270         for (i = 0; i < 256; ++i) {
271                 if (plo_test_bit(i))
272                         kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
273         }
274 
275         if (test_facility(28)) /* TOD-clock steering */
276                 ptff(kvm_s390_available_subfunc.ptff,
277                      sizeof(kvm_s390_available_subfunc.ptff),
278                      PTFF_QAF);
279 
280         if (test_facility(17)) { /* MSA */
281                 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
282                               kvm_s390_available_subfunc.kmac);
283                 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
284                               kvm_s390_available_subfunc.kmc);
285                 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
286                               kvm_s390_available_subfunc.km);
287                 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
288                               kvm_s390_available_subfunc.kimd);
289                 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
290                               kvm_s390_available_subfunc.klmd);
291         }
292         if (test_facility(76)) /* MSA3 */
293                 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
294                               kvm_s390_available_subfunc.pckmo);
295         if (test_facility(77)) { /* MSA4 */
296                 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
297                               kvm_s390_available_subfunc.kmctr);
298                 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
299                               kvm_s390_available_subfunc.kmf);
300                 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
301                               kvm_s390_available_subfunc.kmo);
302                 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
303                               kvm_s390_available_subfunc.pcc);
304         }
305         if (test_facility(57)) /* MSA5 */
306                 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
307                               kvm_s390_available_subfunc.ppno);
308 
309         if (test_facility(146)) /* MSA8 */
310                 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
311                               kvm_s390_available_subfunc.kma);
312 
313         if (MACHINE_HAS_ESOP)
314                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
315         /*
316          * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
317          * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
318          */
319         if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
320             !test_facility(3) || !nested)
321                 return;
322         allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
323         if (sclp.has_64bscao)
324                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
325         if (sclp.has_siif)
326                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
327         if (sclp.has_gpere)
328                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
329         if (sclp.has_gsls)
330                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
331         if (sclp.has_ib)
332                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
333         if (sclp.has_cei)
334                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
335         if (sclp.has_ibs)
336                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
337         if (sclp.has_kss)
338                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
339         /*
340          * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
341          * all skey handling functions read/set the skey from the PGSTE
342          * instead of the real storage key.
343          *
344          * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
345          * pages being detected as preserved although they are resident.
346          *
347          * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
348          * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
349          *
350          * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
351          * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
352          * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
353          *
354          * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
355          * cannot easily shadow the SCA because of the ipte lock.
356          */
357 }
358 
359 int kvm_arch_init(void *opaque)
360 {
361         kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
362         if (!kvm_s390_dbf)
363                 return -ENOMEM;
364 
365         if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
366                 debug_unregister(kvm_s390_dbf);
367                 return -ENOMEM;
368         }
369 
370         kvm_s390_cpu_feat_init();
371 
372         /* Register floating interrupt controller interface. */
373         return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
374 }
375 
376 void kvm_arch_exit(void)
377 {
378         debug_unregister(kvm_s390_dbf);
379 }
380 
381 /* Section: device related */
382 long kvm_arch_dev_ioctl(struct file *filp,
383                         unsigned int ioctl, unsigned long arg)
384 {
385         if (ioctl == KVM_S390_ENABLE_SIE)
386                 return s390_enable_sie();
387         return -EINVAL;
388 }
389 
390 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
391 {
392         int r;
393 
394         switch (ext) {
395         case KVM_CAP_S390_PSW:
396         case KVM_CAP_S390_GMAP:
397         case KVM_CAP_SYNC_MMU:
398 #ifdef CONFIG_KVM_S390_UCONTROL
399         case KVM_CAP_S390_UCONTROL:
400 #endif
401         case KVM_CAP_ASYNC_PF:
402         case KVM_CAP_SYNC_REGS:
403         case KVM_CAP_ONE_REG:
404         case KVM_CAP_ENABLE_CAP:
405         case KVM_CAP_S390_CSS_SUPPORT:
406         case KVM_CAP_IOEVENTFD:
407         case KVM_CAP_DEVICE_CTRL:
408         case KVM_CAP_ENABLE_CAP_VM:
409         case KVM_CAP_S390_IRQCHIP:
410         case KVM_CAP_VM_ATTRIBUTES:
411         case KVM_CAP_MP_STATE:
412         case KVM_CAP_IMMEDIATE_EXIT:
413         case KVM_CAP_S390_INJECT_IRQ:
414         case KVM_CAP_S390_USER_SIGP:
415         case KVM_CAP_S390_USER_STSI:
416         case KVM_CAP_S390_SKEYS:
417         case KVM_CAP_S390_IRQ_STATE:
418         case KVM_CAP_S390_USER_INSTR0:
419         case KVM_CAP_S390_CMMA_MIGRATION:
420         case KVM_CAP_S390_AIS:
421         case KVM_CAP_S390_AIS_MIGRATION:
422                 r = 1;
423                 break;
424         case KVM_CAP_S390_MEM_OP:
425                 r = MEM_OP_MAX_SIZE;
426                 break;
427         case KVM_CAP_NR_VCPUS:
428         case KVM_CAP_MAX_VCPUS:
429                 r = KVM_S390_BSCA_CPU_SLOTS;
430                 if (!kvm_s390_use_sca_entries())
431                         r = KVM_MAX_VCPUS;
432                 else if (sclp.has_esca && sclp.has_64bscao)
433                         r = KVM_S390_ESCA_CPU_SLOTS;
434                 break;
435         case KVM_CAP_NR_MEMSLOTS:
436                 r = KVM_USER_MEM_SLOTS;
437                 break;
438         case KVM_CAP_S390_COW:
439                 r = MACHINE_HAS_ESOP;
440                 break;
441         case KVM_CAP_S390_VECTOR_REGISTERS:
442                 r = MACHINE_HAS_VX;
443                 break;
444         case KVM_CAP_S390_RI:
445                 r = test_facility(64);
446                 break;
447         case KVM_CAP_S390_GS:
448                 r = test_facility(133);
449                 break;
450         case KVM_CAP_S390_BPB:
451                 r = test_facility(82);
452                 break;
453         default:
454                 r = 0;
455         }
456         return r;
457 }
458 
459 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
460                                         struct kvm_memory_slot *memslot)
461 {
462         gfn_t cur_gfn, last_gfn;
463         unsigned long address;
464         struct gmap *gmap = kvm->arch.gmap;
465 
466         /* Loop over all guest pages */
467         last_gfn = memslot->base_gfn + memslot->npages;
468         for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
469                 address = gfn_to_hva_memslot(memslot, cur_gfn);
470 
471                 if (test_and_clear_guest_dirty(gmap->mm, address))
472                         mark_page_dirty(kvm, cur_gfn);
473                 if (fatal_signal_pending(current))
474                         return;
475                 cond_resched();
476         }
477 }
478 
479 /* Section: vm related */
480 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
481 
482 /*
483  * Get (and clear) the dirty memory log for a memory slot.
484  */
485 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
486                                struct kvm_dirty_log *log)
487 {
488         int r;
489         unsigned long n;
490         struct kvm_memslots *slots;
491         struct kvm_memory_slot *memslot;
492         int is_dirty = 0;
493 
494         if (kvm_is_ucontrol(kvm))
495                 return -EINVAL;
496 
497         mutex_lock(&kvm->slots_lock);
498 
499         r = -EINVAL;
500         if (log->slot >= KVM_USER_MEM_SLOTS)
501                 goto out;
502 
503         slots = kvm_memslots(kvm);
504         memslot = id_to_memslot(slots, log->slot);
505         r = -ENOENT;
506         if (!memslot->dirty_bitmap)
507                 goto out;
508 
509         kvm_s390_sync_dirty_log(kvm, memslot);
510         r = kvm_get_dirty_log(kvm, log, &is_dirty);
511         if (r)
512                 goto out;
513 
514         /* Clear the dirty log */
515         if (is_dirty) {
516                 n = kvm_dirty_bitmap_bytes(memslot);
517                 memset(memslot->dirty_bitmap, 0, n);
518         }
519         r = 0;
520 out:
521         mutex_unlock(&kvm->slots_lock);
522         return r;
523 }
524 
525 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
526 {
527         unsigned int i;
528         struct kvm_vcpu *vcpu;
529 
530         kvm_for_each_vcpu(i, vcpu, kvm) {
531                 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
532         }
533 }
534 
535 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
536 {
537         int r;
538 
539         if (cap->flags)
540                 return -EINVAL;
541 
542         switch (cap->cap) {
543         case KVM_CAP_S390_IRQCHIP:
544                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
545                 kvm->arch.use_irqchip = 1;
546                 r = 0;
547                 break;
548         case KVM_CAP_S390_USER_SIGP:
549                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
550                 kvm->arch.user_sigp = 1;
551                 r = 0;
552                 break;
553         case KVM_CAP_S390_VECTOR_REGISTERS:
554                 mutex_lock(&kvm->lock);
555                 if (kvm->created_vcpus) {
556                         r = -EBUSY;
557                 } else if (MACHINE_HAS_VX) {
558                         set_kvm_facility(kvm->arch.model.fac_mask, 129);
559                         set_kvm_facility(kvm->arch.model.fac_list, 129);
560                         if (test_facility(134)) {
561                                 set_kvm_facility(kvm->arch.model.fac_mask, 134);
562                                 set_kvm_facility(kvm->arch.model.fac_list, 134);
563                         }
564                         if (test_facility(135)) {
565                                 set_kvm_facility(kvm->arch.model.fac_mask, 135);
566                                 set_kvm_facility(kvm->arch.model.fac_list, 135);
567                         }
568                         r = 0;
569                 } else
570                         r = -EINVAL;
571                 mutex_unlock(&kvm->lock);
572                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
573                          r ? "(not available)" : "(success)");
574                 break;
575         case KVM_CAP_S390_RI:
576                 r = -EINVAL;
577                 mutex_lock(&kvm->lock);
578                 if (kvm->created_vcpus) {
579                         r = -EBUSY;
580                 } else if (test_facility(64)) {
581                         set_kvm_facility(kvm->arch.model.fac_mask, 64);
582                         set_kvm_facility(kvm->arch.model.fac_list, 64);
583                         r = 0;
584                 }
585                 mutex_unlock(&kvm->lock);
586                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
587                          r ? "(not available)" : "(success)");
588                 break;
589         case KVM_CAP_S390_AIS:
590                 mutex_lock(&kvm->lock);
591                 if (kvm->created_vcpus) {
592                         r = -EBUSY;
593                 } else {
594                         set_kvm_facility(kvm->arch.model.fac_mask, 72);
595                         set_kvm_facility(kvm->arch.model.fac_list, 72);
596                         r = 0;
597                 }
598                 mutex_unlock(&kvm->lock);
599                 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
600                          r ? "(not available)" : "(success)");
601                 break;
602         case KVM_CAP_S390_GS:
603                 r = -EINVAL;
604                 mutex_lock(&kvm->lock);
605                 if (atomic_read(&kvm->online_vcpus)) {
606                         r = -EBUSY;
607                 } else if (test_facility(133)) {
608                         set_kvm_facility(kvm->arch.model.fac_mask, 133);
609                         set_kvm_facility(kvm->arch.model.fac_list, 133);
610                         r = 0;
611                 }
612                 mutex_unlock(&kvm->lock);
613                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
614                          r ? "(not available)" : "(success)");
615                 break;
616         case KVM_CAP_S390_USER_STSI:
617                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
618                 kvm->arch.user_stsi = 1;
619                 r = 0;
620                 break;
621         case KVM_CAP_S390_USER_INSTR0:
622                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
623                 kvm->arch.user_instr0 = 1;
624                 icpt_operexc_on_all_vcpus(kvm);
625                 r = 0;
626                 break;
627         default:
628                 r = -EINVAL;
629                 break;
630         }
631         return r;
632 }
633 
634 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
635 {
636         int ret;
637 
638         switch (attr->attr) {
639         case KVM_S390_VM_MEM_LIMIT_SIZE:
640                 ret = 0;
641                 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
642                          kvm->arch.mem_limit);
643                 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
644                         ret = -EFAULT;
645                 break;
646         default:
647                 ret = -ENXIO;
648                 break;
649         }
650         return ret;
651 }
652 
653 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
654 {
655         int ret;
656         unsigned int idx;
657         switch (attr->attr) {
658         case KVM_S390_VM_MEM_ENABLE_CMMA:
659                 ret = -ENXIO;
660                 if (!sclp.has_cmma)
661                         break;
662 
663                 ret = -EBUSY;
664                 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
665                 mutex_lock(&kvm->lock);
666                 if (!kvm->created_vcpus) {
667                         kvm->arch.use_cmma = 1;
668                         ret = 0;
669                 }
670                 mutex_unlock(&kvm->lock);
671                 break;
672         case KVM_S390_VM_MEM_CLR_CMMA:
673                 ret = -ENXIO;
674                 if (!sclp.has_cmma)
675                         break;
676                 ret = -EINVAL;
677                 if (!kvm->arch.use_cmma)
678                         break;
679 
680                 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
681                 mutex_lock(&kvm->lock);
682                 idx = srcu_read_lock(&kvm->srcu);
683                 s390_reset_cmma(kvm->arch.gmap->mm);
684                 srcu_read_unlock(&kvm->srcu, idx);
685                 mutex_unlock(&kvm->lock);
686                 ret = 0;
687                 break;
688         case KVM_S390_VM_MEM_LIMIT_SIZE: {
689                 unsigned long new_limit;
690 
691                 if (kvm_is_ucontrol(kvm))
692                         return -EINVAL;
693 
694                 if (get_user(new_limit, (u64 __user *)attr->addr))
695                         return -EFAULT;
696 
697                 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
698                     new_limit > kvm->arch.mem_limit)
699                         return -E2BIG;
700 
701                 if (!new_limit)
702                         return -EINVAL;
703 
704                 /* gmap_create takes last usable address */
705                 if (new_limit != KVM_S390_NO_MEM_LIMIT)
706                         new_limit -= 1;
707 
708                 ret = -EBUSY;
709                 mutex_lock(&kvm->lock);
710                 if (!kvm->created_vcpus) {
711                         /* gmap_create will round the limit up */
712                         struct gmap *new = gmap_create(current->mm, new_limit);
713 
714                         if (!new) {
715                                 ret = -ENOMEM;
716                         } else {
717                                 gmap_remove(kvm->arch.gmap);
718                                 new->private = kvm;
719                                 kvm->arch.gmap = new;
720                                 ret = 0;
721                         }
722                 }
723                 mutex_unlock(&kvm->lock);
724                 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
725                 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
726                          (void *) kvm->arch.gmap->asce);
727                 break;
728         }
729         default:
730                 ret = -ENXIO;
731                 break;
732         }
733         return ret;
734 }
735 
736 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
737 
738 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
739 {
740         struct kvm_vcpu *vcpu;
741         int i;
742 
743         if (!test_kvm_facility(kvm, 76))
744                 return -EINVAL;
745 
746         mutex_lock(&kvm->lock);
747         switch (attr->attr) {
748         case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
749                 get_random_bytes(
750                         kvm->arch.crypto.crycb->aes_wrapping_key_mask,
751                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
752                 kvm->arch.crypto.aes_kw = 1;
753                 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
754                 break;
755         case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
756                 get_random_bytes(
757                         kvm->arch.crypto.crycb->dea_wrapping_key_mask,
758                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
759                 kvm->arch.crypto.dea_kw = 1;
760                 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
761                 break;
762         case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
763                 kvm->arch.crypto.aes_kw = 0;
764                 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
765                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
766                 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
767                 break;
768         case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
769                 kvm->arch.crypto.dea_kw = 0;
770                 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
771                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
772                 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
773                 break;
774         default:
775                 mutex_unlock(&kvm->lock);
776                 return -ENXIO;
777         }
778 
779         kvm_for_each_vcpu(i, vcpu, kvm) {
780                 kvm_s390_vcpu_crypto_setup(vcpu);
781                 exit_sie(vcpu);
782         }
783         mutex_unlock(&kvm->lock);
784         return 0;
785 }
786 
787 static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
788 {
789         int cx;
790         struct kvm_vcpu *vcpu;
791 
792         kvm_for_each_vcpu(cx, vcpu, kvm)
793                 kvm_s390_sync_request(req, vcpu);
794 }
795 
796 /*
797  * Must be called with kvm->srcu held to avoid races on memslots, and with
798  * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
799  */
800 static int kvm_s390_vm_start_migration(struct kvm *kvm)
801 {
802         struct kvm_s390_migration_state *mgs;
803         struct kvm_memory_slot *ms;
804         /* should be the only one */
805         struct kvm_memslots *slots;
806         unsigned long ram_pages;
807         int slotnr;
808 
809         /* migration mode already enabled */
810         if (kvm->arch.migration_state)
811                 return 0;
812 
813         slots = kvm_memslots(kvm);
814         if (!slots || !slots->used_slots)
815                 return -EINVAL;
816 
817         mgs = kzalloc(sizeof(*mgs), GFP_KERNEL);
818         if (!mgs)
819                 return -ENOMEM;
820         kvm->arch.migration_state = mgs;
821 
822         if (kvm->arch.use_cmma) {
823                 /*
824                  * Get the first slot. They are reverse sorted by base_gfn, so
825                  * the first slot is also the one at the end of the address
826                  * space. We have verified above that at least one slot is
827                  * present.
828                  */
829                 ms = slots->memslots;
830                 /* round up so we only use full longs */
831                 ram_pages = roundup(ms->base_gfn + ms->npages, BITS_PER_LONG);
832                 /* allocate enough bytes to store all the bits */
833                 mgs->pgste_bitmap = vmalloc(ram_pages / 8);
834                 if (!mgs->pgste_bitmap) {
835                         kfree(mgs);
836                         kvm->arch.migration_state = NULL;
837                         return -ENOMEM;
838                 }
839 
840                 mgs->bitmap_size = ram_pages;
841                 atomic64_set(&mgs->dirty_pages, ram_pages);
842                 /* mark all the pages in active slots as dirty */
843                 for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
844                         ms = slots->memslots + slotnr;
845                         bitmap_set(mgs->pgste_bitmap, ms->base_gfn, ms->npages);
846                 }
847 
848                 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
849         }
850         return 0;
851 }
852 
853 /*
854  * Must be called with kvm->slots_lock to avoid races with ourselves and
855  * kvm_s390_vm_start_migration.
856  */
857 static int kvm_s390_vm_stop_migration(struct kvm *kvm)
858 {
859         struct kvm_s390_migration_state *mgs;
860 
861         /* migration mode already disabled */
862         if (!kvm->arch.migration_state)
863                 return 0;
864         mgs = kvm->arch.migration_state;
865         kvm->arch.migration_state = NULL;
866 
867         if (kvm->arch.use_cmma) {
868                 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
869                 /* We have to wait for the essa emulation to finish */
870                 synchronize_srcu(&kvm->srcu);
871                 vfree(mgs->pgste_bitmap);
872         }
873         kfree(mgs);
874         return 0;
875 }
876 
877 static int kvm_s390_vm_set_migration(struct kvm *kvm,
878                                      struct kvm_device_attr *attr)
879 {
880         int res = -ENXIO;
881 
882         mutex_lock(&kvm->slots_lock);
883         switch (attr->attr) {
884         case KVM_S390_VM_MIGRATION_START:
885                 res = kvm_s390_vm_start_migration(kvm);
886                 break;
887         case KVM_S390_VM_MIGRATION_STOP:
888                 res = kvm_s390_vm_stop_migration(kvm);
889                 break;
890         default:
891                 break;
892         }
893         mutex_unlock(&kvm->slots_lock);
894 
895         return res;
896 }
897 
898 static int kvm_s390_vm_get_migration(struct kvm *kvm,
899                                      struct kvm_device_attr *attr)
900 {
901         u64 mig = (kvm->arch.migration_state != NULL);
902 
903         if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
904                 return -ENXIO;
905 
906         if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
907                 return -EFAULT;
908         return 0;
909 }
910 
911 static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
912 {
913         struct kvm_s390_vm_tod_clock gtod;
914 
915         if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
916                 return -EFAULT;
917 
918         if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
919                 return -EINVAL;
920         kvm_s390_set_tod_clock(kvm, &gtod);
921 
922         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
923                 gtod.epoch_idx, gtod.tod);
924 
925         return 0;
926 }
927 
928 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
929 {
930         u8 gtod_high;
931 
932         if (copy_from_user(&gtod_high, (void __user *)attr->addr,
933                                            sizeof(gtod_high)))
934                 return -EFAULT;
935 
936         if (gtod_high != 0)
937                 return -EINVAL;
938         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
939 
940         return 0;
941 }
942 
943 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
944 {
945         struct kvm_s390_vm_tod_clock gtod = { 0 };
946 
947         if (copy_from_user(&gtod.tod, (void __user *)attr->addr,
948                            sizeof(gtod.tod)))
949                 return -EFAULT;
950 
951         kvm_s390_set_tod_clock(kvm, &gtod);
952         VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
953         return 0;
954 }
955 
956 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
957 {
958         int ret;
959 
960         if (attr->flags)
961                 return -EINVAL;
962 
963         switch (attr->attr) {
964         case KVM_S390_VM_TOD_EXT:
965                 ret = kvm_s390_set_tod_ext(kvm, attr);
966                 break;
967         case KVM_S390_VM_TOD_HIGH:
968                 ret = kvm_s390_set_tod_high(kvm, attr);
969                 break;
970         case KVM_S390_VM_TOD_LOW:
971                 ret = kvm_s390_set_tod_low(kvm, attr);
972                 break;
973         default:
974                 ret = -ENXIO;
975                 break;
976         }
977         return ret;
978 }
979 
980 static void kvm_s390_get_tod_clock_ext(struct kvm *kvm,
981                                         struct kvm_s390_vm_tod_clock *gtod)
982 {
983         struct kvm_s390_tod_clock_ext htod;
984 
985         preempt_disable();
986 
987         get_tod_clock_ext((char *)&htod);
988 
989         gtod->tod = htod.tod + kvm->arch.epoch;
990         gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
991 
992         if (gtod->tod < htod.tod)
993                 gtod->epoch_idx += 1;
994 
995         preempt_enable();
996 }
997 
998 static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
999 {
1000         struct kvm_s390_vm_tod_clock gtod;
1001 
1002         memset(&gtod, 0, sizeof(gtod));
1003 
1004         if (test_kvm_facility(kvm, 139))
1005                 kvm_s390_get_tod_clock_ext(kvm, &gtod);
1006         else
1007                 gtod.tod = kvm_s390_get_tod_clock_fast(kvm);
1008 
1009         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1010                 return -EFAULT;
1011 
1012         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
1013                 gtod.epoch_idx, gtod.tod);
1014         return 0;
1015 }
1016 
1017 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1018 {
1019         u8 gtod_high = 0;
1020 
1021         if (copy_to_user((void __user *)attr->addr, &gtod_high,
1022                                          sizeof(gtod_high)))
1023                 return -EFAULT;
1024         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
1025 
1026         return 0;
1027 }
1028 
1029 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1030 {
1031         u64 gtod;
1032 
1033         gtod = kvm_s390_get_tod_clock_fast(kvm);
1034         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1035                 return -EFAULT;
1036         VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
1037 
1038         return 0;
1039 }
1040 
1041 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1042 {
1043         int ret;
1044 
1045         if (attr->flags)
1046                 return -EINVAL;
1047 
1048         switch (attr->attr) {
1049         case KVM_S390_VM_TOD_EXT:
1050                 ret = kvm_s390_get_tod_ext(kvm, attr);
1051                 break;
1052         case KVM_S390_VM_TOD_HIGH:
1053                 ret = kvm_s390_get_tod_high(kvm, attr);
1054                 break;
1055         case KVM_S390_VM_TOD_LOW:
1056                 ret = kvm_s390_get_tod_low(kvm, attr);
1057                 break;
1058         default:
1059                 ret = -ENXIO;
1060                 break;
1061         }
1062         return ret;
1063 }
1064 
1065 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1066 {
1067         struct kvm_s390_vm_cpu_processor *proc;
1068         u16 lowest_ibc, unblocked_ibc;
1069         int ret = 0;
1070 
1071         mutex_lock(&kvm->lock);
1072         if (kvm->created_vcpus) {
1073                 ret = -EBUSY;
1074                 goto out;
1075         }
1076         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1077         if (!proc) {
1078                 ret = -ENOMEM;
1079                 goto out;
1080         }
1081         if (!copy_from_user(proc, (void __user *)attr->addr,
1082                             sizeof(*proc))) {
1083                 kvm->arch.model.cpuid = proc->cpuid;
1084                 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1085                 unblocked_ibc = sclp.ibc & 0xfff;
1086                 if (lowest_ibc && proc->ibc) {
1087                         if (proc->ibc > unblocked_ibc)
1088                                 kvm->arch.model.ibc = unblocked_ibc;
1089                         else if (proc->ibc < lowest_ibc)
1090                                 kvm->arch.model.ibc = lowest_ibc;
1091                         else
1092                                 kvm->arch.model.ibc = proc->ibc;
1093                 }
1094                 memcpy(kvm->arch.model.fac_list, proc->fac_list,
1095                        S390_ARCH_FAC_LIST_SIZE_BYTE);
1096                 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1097                          kvm->arch.model.ibc,
1098                          kvm->arch.model.cpuid);
1099                 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1100                          kvm->arch.model.fac_list[0],
1101                          kvm->arch.model.fac_list[1],
1102                          kvm->arch.model.fac_list[2]);
1103         } else
1104                 ret = -EFAULT;
1105         kfree(proc);
1106 out:
1107         mutex_unlock(&kvm->lock);
1108         return ret;
1109 }
1110 
1111 static int kvm_s390_set_processor_feat(struct kvm *kvm,
1112                                        struct kvm_device_attr *attr)
1113 {
1114         struct kvm_s390_vm_cpu_feat data;
1115         int ret = -EBUSY;
1116 
1117         if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1118                 return -EFAULT;
1119         if (!bitmap_subset((unsigned long *) data.feat,
1120                            kvm_s390_available_cpu_feat,
1121                            KVM_S390_VM_CPU_FEAT_NR_BITS))
1122                 return -EINVAL;
1123 
1124         mutex_lock(&kvm->lock);
1125         if (!atomic_read(&kvm->online_vcpus)) {
1126                 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
1127                             KVM_S390_VM_CPU_FEAT_NR_BITS);
1128                 ret = 0;
1129         }
1130         mutex_unlock(&kvm->lock);
1131         return ret;
1132 }
1133 
1134 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1135                                           struct kvm_device_attr *attr)
1136 {
1137         /*
1138          * Once supported by kernel + hw, we have to store the subfunctions
1139          * in kvm->arch and remember that user space configured them.
1140          */
1141         return -ENXIO;
1142 }
1143 
1144 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1145 {
1146         int ret = -ENXIO;
1147 
1148         switch (attr->attr) {
1149         case KVM_S390_VM_CPU_PROCESSOR:
1150                 ret = kvm_s390_set_processor(kvm, attr);
1151                 break;
1152         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1153                 ret = kvm_s390_set_processor_feat(kvm, attr);
1154                 break;
1155         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1156                 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1157                 break;
1158         }
1159         return ret;
1160 }
1161 
1162 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1163 {
1164         struct kvm_s390_vm_cpu_processor *proc;
1165         int ret = 0;
1166 
1167         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1168         if (!proc) {
1169                 ret = -ENOMEM;
1170                 goto out;
1171         }
1172         proc->cpuid = kvm->arch.model.cpuid;
1173         proc->ibc = kvm->arch.model.ibc;
1174         memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1175                S390_ARCH_FAC_LIST_SIZE_BYTE);
1176         VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1177                  kvm->arch.model.ibc,
1178                  kvm->arch.model.cpuid);
1179         VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1180                  kvm->arch.model.fac_list[0],
1181                  kvm->arch.model.fac_list[1],
1182                  kvm->arch.model.fac_list[2]);
1183         if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1184                 ret = -EFAULT;
1185         kfree(proc);
1186 out:
1187         return ret;
1188 }
1189 
1190 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1191 {
1192         struct kvm_s390_vm_cpu_machine *mach;
1193         int ret = 0;
1194 
1195         mach = kzalloc(sizeof(*mach), GFP_KERNEL);
1196         if (!mach) {
1197                 ret = -ENOMEM;
1198                 goto out;
1199         }
1200         get_cpu_id((struct cpuid *) &mach->cpuid);
1201         mach->ibc = sclp.ibc;
1202         memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
1203                S390_ARCH_FAC_LIST_SIZE_BYTE);
1204         memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
1205                sizeof(S390_lowcore.stfle_fac_list));
1206         VM_EVENT(kvm, 3, "GET: host ibc:  0x%4.4x, host cpuid:  0x%16.16llx",
1207                  kvm->arch.model.ibc,
1208                  kvm->arch.model.cpuid);
1209         VM_EVENT(kvm, 3, "GET: host facmask:  0x%16.16llx.%16.16llx.%16.16llx",
1210                  mach->fac_mask[0],
1211                  mach->fac_mask[1],
1212                  mach->fac_mask[2]);
1213         VM_EVENT(kvm, 3, "GET: host faclist:  0x%16.16llx.%16.16llx.%16.16llx",
1214                  mach->fac_list[0],
1215                  mach->fac_list[1],
1216                  mach->fac_list[2]);
1217         if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1218                 ret = -EFAULT;
1219         kfree(mach);
1220 out:
1221         return ret;
1222 }
1223 
1224 static int kvm_s390_get_processor_feat(struct kvm *kvm,
1225                                        struct kvm_device_attr *attr)
1226 {
1227         struct kvm_s390_vm_cpu_feat data;
1228 
1229         bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1230                     KVM_S390_VM_CPU_FEAT_NR_BITS);
1231         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1232                 return -EFAULT;
1233         return 0;
1234 }
1235 
1236 static int kvm_s390_get_machine_feat(struct kvm *kvm,
1237                                      struct kvm_device_attr *attr)
1238 {
1239         struct kvm_s390_vm_cpu_feat data;
1240 
1241         bitmap_copy((unsigned long *) data.feat,
1242                     kvm_s390_available_cpu_feat,
1243                     KVM_S390_VM_CPU_FEAT_NR_BITS);
1244         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1245                 return -EFAULT;
1246         return 0;
1247 }
1248 
1249 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1250                                           struct kvm_device_attr *attr)
1251 {
1252         /*
1253          * Once we can actually configure subfunctions (kernel + hw support),
1254          * we have to check if they were already set by user space, if so copy
1255          * them from kvm->arch.
1256          */
1257         return -ENXIO;
1258 }
1259 
1260 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1261                                         struct kvm_device_attr *attr)
1262 {
1263         if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1264             sizeof(struct kvm_s390_vm_cpu_subfunc)))
1265                 return -EFAULT;
1266         return 0;
1267 }
1268 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1269 {
1270         int ret = -ENXIO;
1271 
1272         switch (attr->attr) {
1273         case KVM_S390_VM_CPU_PROCESSOR:
1274                 ret = kvm_s390_get_processor(kvm, attr);
1275                 break;
1276         case KVM_S390_VM_CPU_MACHINE:
1277                 ret = kvm_s390_get_machine(kvm, attr);
1278                 break;
1279         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1280                 ret = kvm_s390_get_processor_feat(kvm, attr);
1281                 break;
1282         case KVM_S390_VM_CPU_MACHINE_FEAT:
1283                 ret = kvm_s390_get_machine_feat(kvm, attr);
1284                 break;
1285         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1286                 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1287                 break;
1288         case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1289                 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1290                 break;
1291         }
1292         return ret;
1293 }
1294 
1295 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1296 {
1297         int ret;
1298 
1299         switch (attr->group) {
1300         case KVM_S390_VM_MEM_CTRL:
1301                 ret = kvm_s390_set_mem_control(kvm, attr);
1302                 break;
1303         case KVM_S390_VM_TOD:
1304                 ret = kvm_s390_set_tod(kvm, attr);
1305                 break;
1306         case KVM_S390_VM_CPU_MODEL:
1307                 ret = kvm_s390_set_cpu_model(kvm, attr);
1308                 break;
1309         case KVM_S390_VM_CRYPTO:
1310                 ret = kvm_s390_vm_set_crypto(kvm, attr);
1311                 break;
1312         case KVM_S390_VM_MIGRATION:
1313                 ret = kvm_s390_vm_set_migration(kvm, attr);
1314                 break;
1315         default:
1316                 ret = -ENXIO;
1317                 break;
1318         }
1319 
1320         return ret;
1321 }
1322 
1323 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1324 {
1325         int ret;
1326 
1327         switch (attr->group) {
1328         case KVM_S390_VM_MEM_CTRL:
1329                 ret = kvm_s390_get_mem_control(kvm, attr);
1330                 break;
1331         case KVM_S390_VM_TOD:
1332                 ret = kvm_s390_get_tod(kvm, attr);
1333                 break;
1334         case KVM_S390_VM_CPU_MODEL:
1335                 ret = kvm_s390_get_cpu_model(kvm, attr);
1336                 break;
1337         case KVM_S390_VM_MIGRATION:
1338                 ret = kvm_s390_vm_get_migration(kvm, attr);
1339                 break;
1340         default:
1341                 ret = -ENXIO;
1342                 break;
1343         }
1344 
1345         return ret;
1346 }
1347 
1348 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1349 {
1350         int ret;
1351 
1352         switch (attr->group) {
1353         case KVM_S390_VM_MEM_CTRL:
1354                 switch (attr->attr) {
1355                 case KVM_S390_VM_MEM_ENABLE_CMMA:
1356                 case KVM_S390_VM_MEM_CLR_CMMA:
1357                         ret = sclp.has_cmma ? 0 : -ENXIO;
1358                         break;
1359                 case KVM_S390_VM_MEM_LIMIT_SIZE:
1360                         ret = 0;
1361                         break;
1362                 default:
1363                         ret = -ENXIO;
1364                         break;
1365                 }
1366                 break;
1367         case KVM_S390_VM_TOD:
1368                 switch (attr->attr) {
1369                 case KVM_S390_VM_TOD_LOW:
1370                 case KVM_S390_VM_TOD_HIGH:
1371                         ret = 0;
1372                         break;
1373                 default:
1374                         ret = -ENXIO;
1375                         break;
1376                 }
1377                 break;
1378         case KVM_S390_VM_CPU_MODEL:
1379                 switch (attr->attr) {
1380                 case KVM_S390_VM_CPU_PROCESSOR:
1381                 case KVM_S390_VM_CPU_MACHINE:
1382                 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1383                 case KVM_S390_VM_CPU_MACHINE_FEAT:
1384                 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1385                         ret = 0;
1386                         break;
1387                 /* configuring subfunctions is not supported yet */
1388                 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1389                 default:
1390                         ret = -ENXIO;
1391                         break;
1392                 }
1393                 break;
1394         case KVM_S390_VM_CRYPTO:
1395                 switch (attr->attr) {
1396                 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1397                 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1398                 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1399                 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1400                         ret = 0;
1401                         break;
1402                 default:
1403                         ret = -ENXIO;
1404                         break;
1405                 }
1406                 break;
1407         case KVM_S390_VM_MIGRATION:
1408                 ret = 0;
1409                 break;
1410         default:
1411                 ret = -ENXIO;
1412                 break;
1413         }
1414 
1415         return ret;
1416 }
1417 
1418 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1419 {
1420         uint8_t *keys;
1421         uint64_t hva;
1422         int srcu_idx, i, r = 0;
1423 
1424         if (args->flags != 0)
1425                 return -EINVAL;
1426 
1427         /* Is this guest using storage keys? */
1428         if (!mm_use_skey(current->mm))
1429                 return KVM_S390_GET_SKEYS_NONE;
1430 
1431         /* Enforce sane limit on memory allocation */
1432         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1433                 return -EINVAL;
1434 
1435         keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1436         if (!keys)
1437                 return -ENOMEM;
1438 
1439         down_read(&current->mm->mmap_sem);
1440         srcu_idx = srcu_read_lock(&kvm->srcu);
1441         for (i = 0; i < args->count; i++) {
1442                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1443                 if (kvm_is_error_hva(hva)) {
1444                         r = -EFAULT;
1445                         break;
1446                 }
1447 
1448                 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1449                 if (r)
1450                         break;
1451         }
1452         srcu_read_unlock(&kvm->srcu, srcu_idx);
1453         up_read(&current->mm->mmap_sem);
1454 
1455         if (!r) {
1456                 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1457                                  sizeof(uint8_t) * args->count);
1458                 if (r)
1459                         r = -EFAULT;
1460         }
1461 
1462         kvfree(keys);
1463         return r;
1464 }
1465 
1466 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1467 {
1468         uint8_t *keys;
1469         uint64_t hva;
1470         int srcu_idx, i, r = 0;
1471 
1472         if (args->flags != 0)
1473                 return -EINVAL;
1474 
1475         /* Enforce sane limit on memory allocation */
1476         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1477                 return -EINVAL;
1478 
1479         keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1480         if (!keys)
1481                 return -ENOMEM;
1482 
1483         r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1484                            sizeof(uint8_t) * args->count);
1485         if (r) {
1486                 r = -EFAULT;
1487                 goto out;
1488         }
1489 
1490         /* Enable storage key handling for the guest */
1491         r = s390_enable_skey();
1492         if (r)
1493                 goto out;
1494 
1495         down_read(&current->mm->mmap_sem);
1496         srcu_idx = srcu_read_lock(&kvm->srcu);
1497         for (i = 0; i < args->count; i++) {
1498                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1499                 if (kvm_is_error_hva(hva)) {
1500                         r = -EFAULT;
1501                         break;
1502                 }
1503 
1504                 /* Lowest order bit is reserved */
1505                 if (keys[i] & 0x01) {
1506                         r = -EINVAL;
1507                         break;
1508                 }
1509 
1510                 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1511                 if (r)
1512                         break;
1513         }
1514         srcu_read_unlock(&kvm->srcu, srcu_idx);
1515         up_read(&current->mm->mmap_sem);
1516 out:
1517         kvfree(keys);
1518         return r;
1519 }
1520 
1521 /*
1522  * Base address and length must be sent at the start of each block, therefore
1523  * it's cheaper to send some clean data, as long as it's less than the size of
1524  * two longs.
1525  */
1526 #define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
1527 /* for consistency */
1528 #define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1529 
1530 /*
1531  * This function searches for the next page with dirty CMMA attributes, and
1532  * saves the attributes in the buffer up to either the end of the buffer or
1533  * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
1534  * no trailing clean bytes are saved.
1535  * In case no dirty bits were found, or if CMMA was not enabled or used, the
1536  * output buffer will indicate 0 as length.
1537  */
1538 static int kvm_s390_get_cmma_bits(struct kvm *kvm,
1539                                   struct kvm_s390_cmma_log *args)
1540 {
1541         struct kvm_s390_migration_state *s = kvm->arch.migration_state;
1542         unsigned long bufsize, hva, pgstev, i, next, cur;
1543         int srcu_idx, peek, r = 0, rr;
1544         u8 *res;
1545 
1546         cur = args->start_gfn;
1547         i = next = pgstev = 0;
1548 
1549         if (unlikely(!kvm->arch.use_cmma))
1550                 return -ENXIO;
1551         /* Invalid/unsupported flags were specified */
1552         if (args->flags & ~KVM_S390_CMMA_PEEK)
1553                 return -EINVAL;
1554         /* Migration mode query, and we are not doing a migration */
1555         peek = !!(args->flags & KVM_S390_CMMA_PEEK);
1556         if (!peek && !s)
1557                 return -EINVAL;
1558         /* CMMA is disabled or was not used, or the buffer has length zero */
1559         bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
1560         if (!bufsize || !kvm->mm->context.use_cmma) {
1561                 memset(args, 0, sizeof(*args));
1562                 return 0;
1563         }
1564 
1565         if (!peek) {
1566                 /* We are not peeking, and there are no dirty pages */
1567                 if (!atomic64_read(&s->dirty_pages)) {
1568                         memset(args, 0, sizeof(*args));
1569                         return 0;
1570                 }
1571                 cur = find_next_bit(s->pgste_bitmap, s->bitmap_size,
1572                                     args->start_gfn);
1573                 if (cur >= s->bitmap_size)      /* nothing found, loop back */
1574                         cur = find_next_bit(s->pgste_bitmap, s->bitmap_size, 0);
1575                 if (cur >= s->bitmap_size) {    /* again! (very unlikely) */
1576                         memset(args, 0, sizeof(*args));
1577                         return 0;
1578                 }
1579                 next = find_next_bit(s->pgste_bitmap, s->bitmap_size, cur + 1);
1580         }
1581 
1582         res = vmalloc(bufsize);
1583         if (!res)
1584                 return -ENOMEM;
1585 
1586         args->start_gfn = cur;
1587 
1588         down_read(&kvm->mm->mmap_sem);
1589         srcu_idx = srcu_read_lock(&kvm->srcu);
1590         while (i < bufsize) {
1591                 hva = gfn_to_hva(kvm, cur);
1592                 if (kvm_is_error_hva(hva)) {
1593                         r = -EFAULT;
1594                         break;
1595                 }
1596                 /* decrement only if we actually flipped the bit to 0 */
1597                 if (!peek && test_and_clear_bit(cur, s->pgste_bitmap))
1598                         atomic64_dec(&s->dirty_pages);
1599                 r = get_pgste(kvm->mm, hva, &pgstev);
1600                 if (r < 0)
1601                         pgstev = 0;
1602                 /* save the value */
1603                 res[i++] = (pgstev >> 24) & 0x43;
1604                 /*
1605                  * if the next bit is too far away, stop.
1606                  * if we reached the previous "next", find the next one
1607                  */
1608                 if (!peek) {
1609                         if (next > cur + KVM_S390_MAX_BIT_DISTANCE)
1610                                 break;
1611                         if (cur == next)
1612                                 next = find_next_bit(s->pgste_bitmap,
1613                                                      s->bitmap_size, cur + 1);
1614                 /* reached the end of the bitmap or of the buffer, stop */
1615                         if ((next >= s->bitmap_size) ||
1616                             (next >= args->start_gfn + bufsize))
1617                                 break;
1618                 }
1619                 cur++;
1620         }
1621         srcu_read_unlock(&kvm->srcu, srcu_idx);
1622         up_read(&kvm->mm->mmap_sem);
1623         args->count = i;
1624         args->remaining = s ? atomic64_read(&s->dirty_pages) : 0;
1625 
1626         rr = copy_to_user((void __user *)args->values, res, args->count);
1627         if (rr)
1628                 r = -EFAULT;
1629 
1630         vfree(res);
1631         return r;
1632 }
1633 
1634 /*
1635  * This function sets the CMMA attributes for the given pages. If the input
1636  * buffer has zero length, no action is taken, otherwise the attributes are
1637  * set and the mm->context.use_cmma flag is set.
1638  */
1639 static int kvm_s390_set_cmma_bits(struct kvm *kvm,
1640                                   const struct kvm_s390_cmma_log *args)
1641 {
1642         unsigned long hva, mask, pgstev, i;
1643         uint8_t *bits;
1644         int srcu_idx, r = 0;
1645 
1646         mask = args->mask;
1647 
1648         if (!kvm->arch.use_cmma)
1649                 return -ENXIO;
1650         /* invalid/unsupported flags */
1651         if (args->flags != 0)
1652                 return -EINVAL;
1653         /* Enforce sane limit on memory allocation */
1654         if (args->count > KVM_S390_CMMA_SIZE_MAX)
1655                 return -EINVAL;
1656         /* Nothing to do */
1657         if (args->count == 0)
1658                 return 0;
1659 
1660         bits = vmalloc(sizeof(*bits) * args->count);
1661         if (!bits)
1662                 return -ENOMEM;
1663 
1664         r = copy_from_user(bits, (void __user *)args->values, args->count);
1665         if (r) {
1666                 r = -EFAULT;
1667                 goto out;
1668         }
1669 
1670         down_read(&kvm->mm->mmap_sem);
1671         srcu_idx = srcu_read_lock(&kvm->srcu);
1672         for (i = 0; i < args->count; i++) {
1673                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1674                 if (kvm_is_error_hva(hva)) {
1675                         r = -EFAULT;
1676                         break;
1677                 }
1678 
1679                 pgstev = bits[i];
1680                 pgstev = pgstev << 24;
1681                 mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
1682                 set_pgste_bits(kvm->mm, hva, mask, pgstev);
1683         }
1684         srcu_read_unlock(&kvm->srcu, srcu_idx);
1685         up_read(&kvm->mm->mmap_sem);
1686 
1687         if (!kvm->mm->context.use_cmma) {
1688                 down_write(&kvm->mm->mmap_sem);
1689                 kvm->mm->context.use_cmma = 1;
1690                 up_write(&kvm->mm->mmap_sem);
1691         }
1692 out:
1693         vfree(bits);
1694         return r;
1695 }
1696 
1697 long kvm_arch_vm_ioctl(struct file *filp,
1698                        unsigned int ioctl, unsigned long arg)
1699 {
1700         struct kvm *kvm = filp->private_data;
1701         void __user *argp = (void __user *)arg;
1702         struct kvm_device_attr attr;
1703         int r;
1704 
1705         switch (ioctl) {
1706         case KVM_S390_INTERRUPT: {
1707                 struct kvm_s390_interrupt s390int;
1708 
1709                 r = -EFAULT;
1710                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1711                         break;
1712                 r = kvm_s390_inject_vm(kvm, &s390int);
1713                 break;
1714         }
1715         case KVM_ENABLE_CAP: {
1716                 struct kvm_enable_cap cap;
1717                 r = -EFAULT;
1718                 if (copy_from_user(&cap, argp, sizeof(cap)))
1719                         break;
1720                 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1721                 break;
1722         }
1723         case KVM_CREATE_IRQCHIP: {
1724                 struct kvm_irq_routing_entry routing;
1725 
1726                 r = -EINVAL;
1727                 if (kvm->arch.use_irqchip) {
1728                         /* Set up dummy routing. */
1729                         memset(&routing, 0, sizeof(routing));
1730                         r = kvm_set_irq_routing(kvm, &routing, 0, 0);
1731                 }
1732                 break;
1733         }
1734         case KVM_SET_DEVICE_ATTR: {
1735                 r = -EFAULT;
1736                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1737                         break;
1738                 r = kvm_s390_vm_set_attr(kvm, &attr);
1739                 break;
1740         }
1741         case KVM_GET_DEVICE_ATTR: {
1742                 r = -EFAULT;
1743                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1744                         break;
1745                 r = kvm_s390_vm_get_attr(kvm, &attr);
1746                 break;
1747         }
1748         case KVM_HAS_DEVICE_ATTR: {
1749                 r = -EFAULT;
1750                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1751                         break;
1752                 r = kvm_s390_vm_has_attr(kvm, &attr);
1753                 break;
1754         }
1755         case KVM_S390_GET_SKEYS: {
1756                 struct kvm_s390_skeys args;
1757 
1758                 r = -EFAULT;
1759                 if (copy_from_user(&args, argp,
1760                                    sizeof(struct kvm_s390_skeys)))
1761                         break;
1762                 r = kvm_s390_get_skeys(kvm, &args);
1763                 break;
1764         }
1765         case KVM_S390_SET_SKEYS: {
1766                 struct kvm_s390_skeys args;
1767 
1768                 r = -EFAULT;
1769                 if (copy_from_user(&args, argp,
1770                                    sizeof(struct kvm_s390_skeys)))
1771                         break;
1772                 r = kvm_s390_set_skeys(kvm, &args);
1773                 break;
1774         }
1775         case KVM_S390_GET_CMMA_BITS: {
1776                 struct kvm_s390_cmma_log args;
1777 
1778                 r = -EFAULT;
1779                 if (copy_from_user(&args, argp, sizeof(args)))
1780                         break;
1781                 mutex_lock(&kvm->slots_lock);
1782                 r = kvm_s390_get_cmma_bits(kvm, &args);
1783                 mutex_unlock(&kvm->slots_lock);
1784                 if (!r) {
1785                         r = copy_to_user(argp, &args, sizeof(args));
1786                         if (r)
1787                                 r = -EFAULT;
1788                 }
1789                 break;
1790         }
1791         case KVM_S390_SET_CMMA_BITS: {
1792                 struct kvm_s390_cmma_log args;
1793 
1794                 r = -EFAULT;
1795                 if (copy_from_user(&args, argp, sizeof(args)))
1796                         break;
1797                 mutex_lock(&kvm->slots_lock);
1798                 r = kvm_s390_set_cmma_bits(kvm, &args);
1799                 mutex_unlock(&kvm->slots_lock);
1800                 break;
1801         }
1802         default:
1803                 r = -ENOTTY;
1804         }
1805 
1806         return r;
1807 }
1808 
1809 static int kvm_s390_query_ap_config(u8 *config)
1810 {
1811         u32 fcn_code = 0x04000000UL;
1812         u32 cc = 0;
1813 
1814         memset(config, 0, 128);
1815         asm volatile(
1816                 "lgr 0,%1\n"
1817                 "lgr 2,%2\n"
1818                 ".long 0xb2af0000\n"            /* PQAP(QCI) */
1819                 "0: ipm %0\n"
1820                 "srl %0,28\n"
1821                 "1:\n"
1822                 EX_TABLE(0b, 1b)
1823                 : "+r" (cc)
1824                 : "r" (fcn_code), "r" (config)
1825                 : "cc", "", "2", "memory"
1826         );
1827 
1828         return cc;
1829 }
1830 
1831 static int kvm_s390_apxa_installed(void)
1832 {
1833         u8 config[128];
1834         int cc;
1835 
1836         if (test_facility(12)) {
1837                 cc = kvm_s390_query_ap_config(config);
1838 
1839                 if (cc)
1840                         pr_err("PQAP(QCI) failed with cc=%d", cc);
1841                 else
1842                         return config[0] & 0x40;
1843         }
1844 
1845         return 0;
1846 }
1847 
1848 static void kvm_s390_set_crycb_format(struct kvm *kvm)
1849 {
1850         kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1851 
1852         if (kvm_s390_apxa_installed())
1853                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1854         else
1855                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1856 }
1857 
1858 static u64 kvm_s390_get_initial_cpuid(void)
1859 {
1860         struct cpuid cpuid;
1861 
1862         get_cpu_id(&cpuid);
1863         cpuid.version = 0xff;
1864         return *((u64 *) &cpuid);
1865 }
1866 
1867 static void kvm_s390_crypto_init(struct kvm *kvm)
1868 {
1869         if (!test_kvm_facility(kvm, 76))
1870                 return;
1871 
1872         kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
1873         kvm_s390_set_crycb_format(kvm);
1874 
1875         /* Enable AES/DEA protected key functions by default */
1876         kvm->arch.crypto.aes_kw = 1;
1877         kvm->arch.crypto.dea_kw = 1;
1878         get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1879                          sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1880         get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1881                          sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1882 }
1883 
1884 static void sca_dispose(struct kvm *kvm)
1885 {
1886         if (kvm->arch.use_esca)
1887                 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
1888         else
1889                 free_page((unsigned long)(kvm->arch.sca));
1890         kvm->arch.sca = NULL;
1891 }
1892 
1893 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1894 {
1895         gfp_t alloc_flags = GFP_KERNEL;
1896         int i, rc;
1897         char debug_name[16];
1898         static unsigned long sca_offset;
1899 
1900         rc = -EINVAL;
1901 #ifdef CONFIG_KVM_S390_UCONTROL
1902         if (type & ~KVM_VM_S390_UCONTROL)
1903                 goto out_err;
1904         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1905                 goto out_err;
1906 #else
1907         if (type)
1908                 goto out_err;
1909 #endif
1910 
1911         rc = s390_enable_sie();
1912         if (rc)
1913                 goto out_err;
1914 
1915         rc = -ENOMEM;
1916 
1917         kvm->arch.use_esca = 0; /* start with basic SCA */
1918         if (!sclp.has_64bscao)
1919                 alloc_flags |= GFP_DMA;
1920         rwlock_init(&kvm->arch.sca_lock);
1921         kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
1922         if (!kvm->arch.sca)
1923                 goto out_err;
1924         spin_lock(&kvm_lock);
1925         sca_offset += 16;
1926         if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
1927                 sca_offset = 0;
1928         kvm->arch.sca = (struct bsca_block *)
1929                         ((char *) kvm->arch.sca + sca_offset);
1930         spin_unlock(&kvm_lock);
1931 
1932         sprintf(debug_name, "kvm-%u", current->pid);
1933 
1934         kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1935         if (!kvm->arch.dbf)
1936                 goto out_err;
1937 
1938         kvm->arch.sie_page2 =
1939              (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1940         if (!kvm->arch.sie_page2)
1941                 goto out_err;
1942 
1943         /* Populate the facility mask initially. */
1944         memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
1945                sizeof(S390_lowcore.stfle_fac_list));
1946         for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1947                 if (i < kvm_s390_fac_list_mask_size())
1948                         kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
1949                 else
1950                         kvm->arch.model.fac_mask[i] = 0UL;
1951         }
1952 
1953         /* Populate the facility list initially. */
1954         kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1955         memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
1956                S390_ARCH_FAC_LIST_SIZE_BYTE);
1957 
1958         /* we are always in czam mode - even on pre z14 machines */
1959         set_kvm_facility(kvm->arch.model.fac_mask, 138);
1960         set_kvm_facility(kvm->arch.model.fac_list, 138);
1961         /* we emulate STHYI in kvm */
1962         set_kvm_facility(kvm->arch.model.fac_mask, 74);
1963         set_kvm_facility(kvm->arch.model.fac_list, 74);
1964         if (MACHINE_HAS_TLB_GUEST) {
1965                 set_kvm_facility(kvm->arch.model.fac_mask, 147);
1966                 set_kvm_facility(kvm->arch.model.fac_list, 147);
1967         }
1968 
1969         kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
1970         kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1971 
1972         kvm_s390_crypto_init(kvm);
1973 
1974         mutex_init(&kvm->arch.float_int.ais_lock);
1975         kvm->arch.float_int.simm = 0;
1976         kvm->arch.float_int.nimm = 0;
1977         spin_lock_init(&kvm->arch.float_int.lock);
1978         for (i = 0; i < FIRQ_LIST_COUNT; i++)
1979                 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1980         init_waitqueue_head(&kvm->arch.ipte_wq);
1981         mutex_init(&kvm->arch.ipte_mutex);
1982 
1983         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1984         VM_EVENT(kvm, 3, "vm created with type %lu", type);
1985 
1986         if (type & KVM_VM_S390_UCONTROL) {
1987                 kvm->arch.gmap = NULL;
1988                 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
1989         } else {
1990                 if (sclp.hamax == U64_MAX)
1991                         kvm->arch.mem_limit = TASK_SIZE_MAX;
1992                 else
1993                         kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
1994                                                     sclp.hamax + 1);
1995                 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
1996                 if (!kvm->arch.gmap)
1997                         goto out_err;
1998                 kvm->arch.gmap->private = kvm;
1999                 kvm->arch.gmap->pfault_enabled = 0;
2000         }
2001 
2002         kvm->arch.css_support = 0;
2003         kvm->arch.use_irqchip = 0;
2004         kvm->arch.epoch = 0;
2005 
2006         spin_lock_init(&kvm->arch.start_stop_lock);
2007         kvm_s390_vsie_init(kvm);
2008         KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
2009 
2010         return 0;
2011 out_err:
2012         free_page((unsigned long)kvm->arch.sie_page2);
2013         debug_unregister(kvm->arch.dbf);
2014         sca_dispose(kvm);
2015         KVM_EVENT(3, "creation of vm failed: %d", rc);
2016         return rc;
2017 }
2018 
2019 bool kvm_arch_has_vcpu_debugfs(void)
2020 {
2021         return false;
2022 }
2023 
2024 int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
2025 {
2026         return 0;
2027 }
2028 
2029 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
2030 {
2031         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
2032         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
2033         kvm_s390_clear_local_irqs(vcpu);
2034         kvm_clear_async_pf_completion_queue(vcpu);
2035         if (!kvm_is_ucontrol(vcpu->kvm))
2036                 sca_del_vcpu(vcpu);
2037 
2038         if (kvm_is_ucontrol(vcpu->kvm))
2039                 gmap_remove(vcpu->arch.gmap);
2040 
2041         if (vcpu->kvm->arch.use_cmma)
2042                 kvm_s390_vcpu_unsetup_cmma(vcpu);
2043         free_page((unsigned long)(vcpu->arch.sie_block));
2044 
2045         kvm_vcpu_uninit(vcpu);
2046         kmem_cache_free(kvm_vcpu_cache, vcpu);
2047 }
2048 
2049 static void kvm_free_vcpus(struct kvm *kvm)
2050 {
2051         unsigned int i;
2052         struct kvm_vcpu *vcpu;
2053 
2054         kvm_for_each_vcpu(i, vcpu, kvm)
2055                 kvm_arch_vcpu_destroy(vcpu);
2056 
2057         mutex_lock(&kvm->lock);
2058         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
2059                 kvm->vcpus[i] = NULL;
2060 
2061         atomic_set(&kvm->online_vcpus, 0);
2062         mutex_unlock(&kvm->lock);
2063 }
2064 
2065 void kvm_arch_destroy_vm(struct kvm *kvm)
2066 {
2067         kvm_free_vcpus(kvm);
2068         sca_dispose(kvm);
2069         debug_unregister(kvm->arch.dbf);
2070         free_page((unsigned long)kvm->arch.sie_page2);
2071         if (!kvm_is_ucontrol(kvm))
2072                 gmap_remove(kvm->arch.gmap);
2073         kvm_s390_destroy_adapters(kvm);
2074         kvm_s390_clear_float_irqs(kvm);
2075         kvm_s390_vsie_destroy(kvm);
2076         if (kvm->arch.migration_state) {
2077                 vfree(kvm->arch.migration_state->pgste_bitmap);
2078                 kfree(kvm->arch.migration_state);
2079         }
2080         KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
2081 }
2082 
2083 /* Section: vcpu related */
2084 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
2085 {
2086         vcpu->arch.gmap = gmap_create(current->mm, -1UL);
2087         if (!vcpu->arch.gmap)
2088                 return -ENOMEM;
2089         vcpu->arch.gmap->private = vcpu->kvm;
2090 
2091         return 0;
2092 }
2093 
2094 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
2095 {
2096         if (!kvm_s390_use_sca_entries())
2097                 return;
2098         read_lock(&vcpu->kvm->arch.sca_lock);
2099         if (vcpu->kvm->arch.use_esca) {
2100                 struct esca_block *sca = vcpu->kvm->arch.sca;
2101 
2102                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2103                 sca->cpu[vcpu->vcpu_id].sda = 0;
2104         } else {
2105                 struct bsca_block *sca = vcpu->kvm->arch.sca;
2106 
2107                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2108                 sca->cpu[vcpu->vcpu_id].sda = 0;
2109         }
2110         read_unlock(&vcpu->kvm->arch.sca_lock);
2111 }
2112 
2113 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
2114 {
2115         if (!kvm_s390_use_sca_entries()) {
2116                 struct bsca_block *sca = vcpu->kvm->arch.sca;
2117 
2118                 /* we still need the basic sca for the ipte control */
2119                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2120                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2121                 return;
2122         }
2123         read_lock(&vcpu->kvm->arch.sca_lock);
2124         if (vcpu->kvm->arch.use_esca) {
2125                 struct esca_block *sca = vcpu->kvm->arch.sca;
2126 
2127                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2128                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2129                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
2130                 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2131                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2132         } else {
2133                 struct bsca_block *sca = vcpu->kvm->arch.sca;
2134 
2135                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2136                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2137                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2138                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2139         }
2140         read_unlock(&vcpu->kvm->arch.sca_lock);
2141 }
2142 
2143 /* Basic SCA to Extended SCA data copy routines */
2144 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
2145 {
2146         d->sda = s->sda;
2147         d->sigp_ctrl.c = s->sigp_ctrl.c;
2148         d->sigp_ctrl.scn = s->sigp_ctrl.scn;
2149 }
2150 
2151 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
2152 {
2153         int i;
2154 
2155         d->ipte_control = s->ipte_control;
2156         d->mcn[0] = s->mcn;
2157         for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
2158                 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
2159 }
2160 
2161 static int sca_switch_to_extended(struct kvm *kvm)
2162 {
2163         struct bsca_block *old_sca = kvm->arch.sca;
2164         struct esca_block *new_sca;
2165         struct kvm_vcpu *vcpu;
2166         unsigned int vcpu_idx;
2167         u32 scaol, scaoh;
2168 
2169         new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
2170         if (!new_sca)
2171                 return -ENOMEM;
2172 
2173         scaoh = (u32)((u64)(new_sca) >> 32);
2174         scaol = (u32)(u64)(new_sca) & ~0x3fU;
2175 
2176         kvm_s390_vcpu_block_all(kvm);
2177         write_lock(&kvm->arch.sca_lock);
2178 
2179         sca_copy_b_to_e(new_sca, old_sca);
2180 
2181         kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
2182                 vcpu->arch.sie_block->scaoh = scaoh;
2183                 vcpu->arch.sie_block->scaol = scaol;
2184                 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2185         }
2186         kvm->arch.sca = new_sca;
2187         kvm->arch.use_esca = 1;
2188 
2189         write_unlock(&kvm->arch.sca_lock);
2190         kvm_s390_vcpu_unblock_all(kvm);
2191 
2192         free_page((unsigned long)old_sca);
2193 
2194         VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
2195                  old_sca, kvm->arch.sca);
2196         return 0;
2197 }
2198 
2199 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
2200 {
2201         int rc;
2202 
2203         if (!kvm_s390_use_sca_entries()) {
2204                 if (id < KVM_MAX_VCPUS)
2205                         return true;
2206                 return false;
2207         }
2208         if (id < KVM_S390_BSCA_CPU_SLOTS)
2209                 return true;
2210         if (!sclp.has_esca || !sclp.has_64bscao)
2211                 return false;
2212 
2213         mutex_lock(&kvm->lock);
2214         rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
2215         mutex_unlock(&kvm->lock);
2216 
2217         return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
2218 }
2219 
2220 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
2221 {
2222         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2223         kvm_clear_async_pf_completion_queue(vcpu);
2224         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
2225                                     KVM_SYNC_GPRS |
2226                                     KVM_SYNC_ACRS |
2227                                     KVM_SYNC_CRS |
2228                                     KVM_SYNC_ARCH0 |
2229                                     KVM_SYNC_PFAULT;
2230         kvm_s390_set_prefix(vcpu, 0);
2231         if (test_kvm_facility(vcpu->kvm, 64))
2232                 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
2233         if (test_kvm_facility(vcpu->kvm, 82))
2234                 vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
2235         if (test_kvm_facility(vcpu->kvm, 133))
2236                 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
2237         /* fprs can be synchronized via vrs, even if the guest has no vx. With
2238          * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
2239          */
2240         if (MACHINE_HAS_VX)
2241                 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
2242         else
2243                 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
2244 
2245         if (kvm_is_ucontrol(vcpu->kvm))
2246                 return __kvm_ucontrol_vcpu_init(vcpu);
2247 
2248         return 0;
2249 }
2250 
2251 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2252 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2253 {
2254         WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
2255         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2256         vcpu->arch.cputm_start = get_tod_clock_fast();
2257         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2258 }
2259 
2260 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2261 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2262 {
2263         WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
2264         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2265         vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2266         vcpu->arch.cputm_start = 0;
2267         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2268 }
2269 
2270 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2271 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2272 {
2273         WARN_ON_ONCE(vcpu->arch.cputm_enabled);
2274         vcpu->arch.cputm_enabled = true;
2275         __start_cpu_timer_accounting(vcpu);
2276 }
2277 
2278 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2279 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2280 {
2281         WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
2282         __stop_cpu_timer_accounting(vcpu);
2283         vcpu->arch.cputm_enabled = false;
2284 }
2285 
2286 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2287 {
2288         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2289         __enable_cpu_timer_accounting(vcpu);
2290         preempt_enable();
2291 }
2292 
2293 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2294 {
2295         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2296         __disable_cpu_timer_accounting(vcpu);
2297         preempt_enable();
2298 }
2299 
2300 /* set the cpu timer - may only be called from the VCPU thread itself */
2301 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
2302 {
2303         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2304         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2305         if (vcpu->arch.cputm_enabled)
2306                 vcpu->arch.cputm_start = get_tod_clock_fast();
2307         vcpu->arch.sie_block->cputm = cputm;
2308         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2309         preempt_enable();
2310 }
2311 
2312 /* update and get the cpu timer - can also be called from other VCPU threads */
2313 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
2314 {
2315         unsigned int seq;
2316         __u64 value;
2317 
2318         if (unlikely(!vcpu->arch.cputm_enabled))
2319                 return vcpu->arch.sie_block->cputm;
2320 
2321         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2322         do {
2323                 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
2324                 /*
2325                  * If the writer would ever execute a read in the critical
2326                  * section, e.g. in irq context, we have a deadlock.
2327                  */
2328                 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
2329                 value = vcpu->arch.sie_block->cputm;
2330                 /* if cputm_start is 0, accounting is being started/stopped */
2331                 if (likely(vcpu->arch.cputm_start))
2332                         value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2333         } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
2334         preempt_enable();
2335         return value;
2336 }
2337 
2338 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2339 {
2340 
2341         gmap_enable(vcpu->arch.enabled_gmap);
2342         atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
2343         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2344                 __start_cpu_timer_accounting(vcpu);
2345         vcpu->cpu = cpu;
2346 }
2347 
2348 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
2349 {
2350         vcpu->cpu = -1;
2351         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2352                 __stop_cpu_timer_accounting(vcpu);
2353         atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
2354         vcpu->arch.enabled_gmap = gmap_get_enabled();
2355         gmap_disable(vcpu->arch.enabled_gmap);
2356 
2357 }
2358 
2359 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
2360 {
2361         /* this equals initial cpu reset in pop, but we don't switch to ESA */
2362         vcpu->arch.sie_block->gpsw.mask = 0UL;
2363         vcpu->arch.sie_block->gpsw.addr = 0UL;
2364         kvm_s390_set_prefix(vcpu, 0);
2365         kvm_s390_set_cpu_timer(vcpu, 0);
2366         vcpu->arch.sie_block->ckc       = 0UL;
2367         vcpu->arch.sie_block->todpr     = 0;
2368         memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
2369         vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
2370         vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
2371         /* make sure the new fpc will be lazily loaded */
2372         save_fpu_regs();
2373         current->thread.fpu.fpc = 0;
2374         vcpu->arch.sie_block->gbea = 1;
2375         vcpu->arch.sie_block->pp = 0;
2376         vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
2377         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2378         kvm_clear_async_pf_completion_queue(vcpu);
2379         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
2380                 kvm_s390_vcpu_stop(vcpu);
2381         kvm_s390_clear_local_irqs(vcpu);
2382 }
2383 
2384 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
2385 {
2386         mutex_lock(&vcpu->kvm->lock);
2387         preempt_disable();
2388         vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
2389         vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
2390         preempt_enable();
2391         mutex_unlock(&vcpu->kvm->lock);
2392         if (!kvm_is_ucontrol(vcpu->kvm)) {
2393                 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
2394                 sca_add_vcpu(vcpu);
2395         }
2396         if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
2397                 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2398         /* make vcpu_load load the right gmap on the first trigger */
2399         vcpu->arch.enabled_gmap = vcpu->arch.gmap;
2400 }
2401 
2402 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
2403 {
2404         if (!test_kvm_facility(vcpu->kvm, 76))
2405                 return;
2406 
2407         vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
2408 
2409         if (vcpu->kvm->arch.crypto.aes_kw)
2410                 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
2411         if (vcpu->kvm->arch.crypto.dea_kw)
2412                 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
2413 
2414         vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
2415 }
2416 
2417 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
2418 {
2419         free_page(vcpu->arch.sie_block->cbrlo);
2420         vcpu->arch.sie_block->cbrlo = 0;
2421 }
2422 
2423 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
2424 {
2425         vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
2426         if (!vcpu->arch.sie_block->cbrlo)
2427                 return -ENOMEM;
2428 
2429         vcpu->arch.sie_block->ecb2 &= ~ECB2_PFMFI;
2430         return 0;
2431 }
2432 
2433 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
2434 {
2435         struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
2436 
2437         vcpu->arch.sie_block->ibc = model->ibc;
2438         if (test_kvm_facility(vcpu->kvm, 7))
2439                 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
2440 }
2441 
2442 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
2443 {
2444         int rc = 0;
2445 
2446         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
2447                                                     CPUSTAT_SM |
2448                                                     CPUSTAT_STOPPED);
2449 
2450         if (test_kvm_facility(vcpu->kvm, 78))
2451                 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
2452         else if (test_kvm_facility(vcpu->kvm, 8))
2453                 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
2454 
2455         kvm_s390_vcpu_setup_model(vcpu);
2456 
2457         /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
2458         if (MACHINE_HAS_ESOP)
2459                 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
2460         if (test_kvm_facility(vcpu->kvm, 9))
2461                 vcpu->arch.sie_block->ecb |= ECB_SRSI;
2462         if (test_kvm_facility(vcpu->kvm, 73))
2463                 vcpu->arch.sie_block->ecb |= ECB_TE;
2464 
2465         if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
2466                 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
2467         if (test_kvm_facility(vcpu->kvm, 130))
2468                 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
2469         vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
2470         if (sclp.has_cei)
2471                 vcpu->arch.sie_block->eca |= ECA_CEI;
2472         if (sclp.has_ib)
2473                 vcpu->arch.sie_block->eca |= ECA_IB;
2474         if (sclp.has_siif)
2475                 vcpu->arch.sie_block->eca |= ECA_SII;
2476         if (sclp.has_sigpif)
2477                 vcpu->arch.sie_block->eca |= ECA_SIGPI;
2478         if (test_kvm_facility(vcpu->kvm, 129)) {
2479                 vcpu->arch.sie_block->eca |= ECA_VX;
2480                 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
2481         }
2482         if (test_kvm_facility(vcpu->kvm, 139))
2483                 vcpu->arch.sie_block->ecd |= ECD_MEF;
2484 
2485         vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
2486                                         | SDNXC;
2487         vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
2488 
2489         if (sclp.has_kss)
2490                 atomic_or(CPUSTAT_KSS, &vcpu->arch.sie_block->cpuflags);
2491         else
2492                 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
2493 
2494         if (vcpu->kvm->arch.use_cmma) {
2495                 rc = kvm_s390_vcpu_setup_cmma(vcpu);
2496                 if (rc)
2497                         return rc;
2498         }
2499         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2500         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
2501 
2502         kvm_s390_vcpu_crypto_setup(vcpu);
2503 
2504         return rc;
2505 }
2506 
2507 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
2508                                       unsigned int id)
2509 {
2510         struct kvm_vcpu *vcpu;
2511         struct sie_page *sie_page;
2512         int rc = -EINVAL;
2513 
2514         if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
2515                 goto out;
2516 
2517         rc = -ENOMEM;
2518 
2519         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
2520         if (!vcpu)
2521                 goto out;
2522 
2523         BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
2524         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
2525         if (!sie_page)
2526                 goto out_free_cpu;
2527 
2528         vcpu->arch.sie_block = &sie_page->sie_block;
2529         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
2530 
2531         /* the real guest size will always be smaller than msl */
2532         vcpu->arch.sie_block->mso = 0;
2533         vcpu->arch.sie_block->msl = sclp.hamax;
2534 
2535         vcpu->arch.sie_block->icpua = id;
2536         spin_lock_init(&vcpu->arch.local_int.lock);
2537         vcpu->arch.local_int.float_int = &kvm->arch.float_int;
2538         vcpu->arch.local_int.wq = &vcpu->wq;
2539         vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
2540         seqcount_init(&vcpu->arch.cputm_seqcount);
2541 
2542         rc = kvm_vcpu_init(vcpu, kvm, id);
2543         if (rc)
2544                 goto out_free_sie_block;
2545         VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
2546                  vcpu->arch.sie_block);
2547         trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
2548 
2549         return vcpu;
2550 out_free_sie_block:
2551         free_page((unsigned long)(vcpu->arch.sie_block));
2552 out_free_cpu:
2553         kmem_cache_free(kvm_vcpu_cache, vcpu);
2554 out:
2555         return ERR_PTR(rc);
2556 }
2557 
2558 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
2559 {
2560         return kvm_s390_vcpu_has_irq(vcpu, 0);
2561 }
2562 
2563 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
2564 {
2565         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
2566 }
2567 
2568 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
2569 {
2570         atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2571         exit_sie(vcpu);
2572 }
2573 
2574 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
2575 {
2576         atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2577 }
2578 
2579 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
2580 {
2581         atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2582         exit_sie(vcpu);
2583 }
2584 
2585 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2586 {
2587         atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2588 }
2589 
2590 /*
2591  * Kick a guest cpu out of SIE and wait until SIE is not running.
2592  * If the CPU is not running (e.g. waiting as idle) the function will
2593  * return immediately. */
2594 void exit_sie(struct kvm_vcpu *vcpu)
2595 {
2596         atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
2597         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2598                 cpu_relax();
2599 }
2600 
2601 /* Kick a guest cpu out of SIE to process a request synchronously */
2602 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
2603 {
2604         kvm_make_request(req, vcpu);
2605         kvm_s390_vcpu_request(vcpu);
2606 }
2607 
2608 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
2609                               unsigned long end)
2610 {
2611         struct kvm *kvm = gmap->private;
2612         struct kvm_vcpu *vcpu;
2613         unsigned long prefix;
2614         int i;
2615 
2616         if (gmap_is_shadow(gmap))
2617                 return;
2618         if (start >= 1UL << 31)
2619                 /* We are only interested in prefix pages */
2620                 return;
2621         kvm_for_each_vcpu(i, vcpu, kvm) {
2622                 /* match against both prefix pages */
2623                 prefix = kvm_s390_get_prefix(vcpu);
2624                 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
2625                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
2626                                    start, end);
2627                         kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2628                 }
2629         }
2630 }
2631 
2632 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2633 {
2634         /* kvm common code refers to this, but never calls it */
2635         BUG();
2636         return 0;
2637 }
2638 
2639 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2640                                            struct kvm_one_reg *reg)
2641 {
2642         int r = -EINVAL;
2643 
2644         switch (reg->id) {
2645         case KVM_REG_S390_TODPR:
2646                 r = put_user(vcpu->arch.sie_block->todpr,
2647                              (u32 __user *)reg->addr);
2648                 break;
2649         case KVM_REG_S390_EPOCHDIFF:
2650                 r = put_user(vcpu->arch.sie_block->epoch,
2651                              (u64 __user *)reg->addr);
2652                 break;
2653         case KVM_REG_S390_CPU_TIMER:
2654                 r = put_user(kvm_s390_get_cpu_timer(vcpu),
2655                              (u64 __user *)reg->addr);
2656                 break;
2657         case KVM_REG_S390_CLOCK_COMP:
2658                 r = put_user(vcpu->arch.sie_block->ckc,
2659                              (u64 __user *)reg->addr);
2660                 break;
2661         case KVM_REG_S390_PFTOKEN:
2662                 r = put_user(vcpu->arch.pfault_token,
2663                              (u64 __user *)reg->addr);
2664                 break;
2665         case KVM_REG_S390_PFCOMPARE:
2666                 r = put_user(vcpu->arch.pfault_compare,
2667                              (u64 __user *)reg->addr);
2668                 break;
2669         case KVM_REG_S390_PFSELECT:
2670                 r = put_user(vcpu->arch.pfault_select,
2671                              (u64 __user *)reg->addr);
2672                 break;
2673         case KVM_REG_S390_PP:
2674                 r = put_user(vcpu->arch.sie_block->pp,
2675                              (u64 __user *)reg->addr);
2676                 break;
2677         case KVM_REG_S390_GBEA:
2678                 r = put_user(vcpu->arch.sie_block->gbea,
2679                              (u64 __user *)reg->addr);
2680                 break;
2681         default:
2682                 break;
2683         }
2684 
2685         return r;
2686 }
2687 
2688 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2689                                            struct kvm_one_reg *reg)
2690 {
2691         int r = -EINVAL;
2692         __u64 val;
2693 
2694         switch (reg->id) {
2695         case KVM_REG_S390_TODPR:
2696                 r = get_user(vcpu->arch.sie_block->todpr,
2697                              (u32 __user *)reg->addr);
2698                 break;
2699         case KVM_REG_S390_EPOCHDIFF:
2700                 r = get_user(vcpu->arch.sie_block->epoch,
2701                              (u64 __user *)reg->addr);
2702                 break;
2703         case KVM_REG_S390_CPU_TIMER:
2704                 r = get_user(val, (u64 __user *)reg->addr);
2705                 if (!r)
2706                         kvm_s390_set_cpu_timer(vcpu, val);
2707                 break;
2708         case KVM_REG_S390_CLOCK_COMP:
2709                 r = get_user(vcpu->arch.sie_block->ckc,
2710                              (u64 __user *)reg->addr);
2711                 break;
2712         case KVM_REG_S390_PFTOKEN:
2713                 r = get_user(vcpu->arch.pfault_token,
2714                              (u64 __user *)reg->addr);
2715                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2716                         kvm_clear_async_pf_completion_queue(vcpu);
2717                 break;
2718         case KVM_REG_S390_PFCOMPARE:
2719                 r = get_user(vcpu->arch.pfault_compare,
2720                              (u64 __user *)reg->addr);
2721                 break;
2722         case KVM_REG_S390_PFSELECT:
2723                 r = get_user(vcpu->arch.pfault_select,
2724                              (u64 __user *)reg->addr);
2725                 break;
2726         case KVM_REG_S390_PP:
2727                 r = get_user(vcpu->arch.sie_block->pp,
2728                              (u64 __user *)reg->addr);
2729                 break;
2730         case KVM_REG_S390_GBEA:
2731                 r = get_user(vcpu->arch.sie_block->gbea,
2732                              (u64 __user *)reg->addr);
2733                 break;
2734         default:
2735                 break;
2736         }
2737 
2738         return r;
2739 }
2740 
2741 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2742 {
2743         kvm_s390_vcpu_initial_reset(vcpu);
2744         return 0;
2745 }
2746 
2747 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2748 {
2749         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
2750         return 0;
2751 }
2752 
2753 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2754 {
2755         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
2756         return 0;
2757 }
2758 
2759 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2760                                   struct kvm_sregs *sregs)
2761 {
2762         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
2763         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
2764         return 0;
2765 }
2766 
2767 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2768                                   struct kvm_sregs *sregs)
2769 {
2770         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
2771         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
2772         return 0;
2773 }
2774 
2775 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2776 {
2777         if (test_fp_ctl(fpu->fpc))
2778                 return -EINVAL;
2779         vcpu->run->s.regs.fpc = fpu->fpc;
2780         if (MACHINE_HAS_VX)
2781                 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
2782                                  (freg_t *) fpu->fprs);
2783         else
2784                 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
2785         return 0;
2786 }
2787 
2788 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2789 {
2790         /* make sure we have the latest values */
2791         save_fpu_regs();
2792         if (MACHINE_HAS_VX)
2793                 convert_vx_to_fp((freg_t *) fpu->fprs,
2794                                  (__vector128 *) vcpu->run->s.regs.vrs);
2795         else
2796                 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
2797         fpu->fpc = vcpu->run->s.regs.fpc;
2798         return 0;
2799 }
2800 
2801 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2802 {
2803         int rc = 0;
2804 
2805         if (!is_vcpu_stopped(vcpu))
2806                 rc = -EBUSY;
2807         else {
2808                 vcpu->run->psw_mask = psw.mask;
2809                 vcpu->run->psw_addr = psw.addr;
2810         }
2811         return rc;
2812 }
2813 
2814 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2815                                   struct kvm_translation *tr)
2816 {
2817         return -EINVAL; /* not implemented yet */
2818 }
2819 
2820 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2821                               KVM_GUESTDBG_USE_HW_BP | \
2822                               KVM_GUESTDBG_ENABLE)
2823 
2824 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2825                                         struct kvm_guest_debug *dbg)
2826 {
2827         int rc = 0;
2828 
2829         vcpu->guest_debug = 0;
2830         kvm_s390_clear_bp_data(vcpu);
2831 
2832         if (dbg->control & ~VALID_GUESTDBG_FLAGS)
2833                 return -EINVAL;
2834         if (!sclp.has_gpere)
2835                 return -EINVAL;
2836 
2837         if (dbg->control & KVM_GUESTDBG_ENABLE) {
2838                 vcpu->guest_debug = dbg->control;
2839                 /* enforce guest PER */
2840                 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2841 
2842                 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2843                         rc = kvm_s390_import_bp_data(vcpu, dbg);
2844         } else {
2845                 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2846                 vcpu->arch.guestdbg.last_bp = 0;
2847         }
2848 
2849         if (rc) {
2850                 vcpu->guest_debug = 0;
2851                 kvm_s390_clear_bp_data(vcpu);
2852                 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2853         }
2854 
2855         return rc;
2856 }
2857 
2858 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2859                                     struct kvm_mp_state *mp_state)
2860 {
2861         /* CHECK_STOP and LOAD are not supported yet */
2862         return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2863                                        KVM_MP_STATE_OPERATING;
2864 }
2865 
2866 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2867                                     struct kvm_mp_state *mp_state)
2868 {
2869         int rc = 0;
2870 
2871         /* user space knows about this interface - let it control the state */
2872         vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2873 
2874         switch (mp_state->mp_state) {
2875         case KVM_MP_STATE_STOPPED:
2876                 kvm_s390_vcpu_stop(vcpu);
2877                 break;
2878         case KVM_MP_STATE_OPERATING:
2879                 kvm_s390_vcpu_start(vcpu);
2880                 break;
2881         case KVM_MP_STATE_LOAD:
2882         case KVM_MP_STATE_CHECK_STOP:
2883                 /* fall through - CHECK_STOP and LOAD are not supported yet */
2884         default:
2885                 rc = -ENXIO;
2886         }
2887 
2888         return rc;
2889 }
2890 
2891 static bool ibs_enabled(struct kvm_vcpu *vcpu)
2892 {
2893         return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2894 }
2895 
2896 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2897 {
2898 retry:
2899         kvm_s390_vcpu_request_handled(vcpu);
2900         if (!kvm_request_pending(vcpu))
2901                 return 0;
2902         /*
2903          * We use MMU_RELOAD just to re-arm the ipte notifier for the
2904          * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2905          * This ensures that the ipte instruction for this request has
2906          * already finished. We might race against a second unmapper that
2907          * wants to set the blocking bit. Lets just retry the request loop.
2908          */
2909         if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2910                 int rc;
2911                 rc = gmap_mprotect_notify(vcpu->arch.gmap,
2912                                           kvm_s390_get_prefix(vcpu),
2913                                           PAGE_SIZE * 2, PROT_WRITE);
2914                 if (rc) {
2915                         kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
2916                         return rc;
2917                 }
2918                 goto retry;
2919         }
2920 
2921         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2922                 vcpu->arch.sie_block->ihcpu = 0xffff;
2923                 goto retry;
2924         }
2925 
2926         if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2927                 if (!ibs_enabled(vcpu)) {
2928                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
2929                         atomic_or(CPUSTAT_IBS,
2930                                         &vcpu->arch.sie_block->cpuflags);
2931                 }
2932                 goto retry;
2933         }
2934 
2935         if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2936                 if (ibs_enabled(vcpu)) {
2937                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
2938                         atomic_andnot(CPUSTAT_IBS,
2939                                           &vcpu->arch.sie_block->cpuflags);
2940                 }
2941                 goto retry;
2942         }
2943 
2944         if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
2945                 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2946                 goto retry;
2947         }
2948 
2949         if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
2950                 /*
2951                  * Disable CMMA virtualization; we will emulate the ESSA
2952                  * instruction manually, in order to provide additional
2953                  * functionalities needed for live migration.
2954                  */
2955                 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
2956                 goto retry;
2957         }
2958 
2959         if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
2960                 /*
2961                  * Re-enable CMMA virtualization if CMMA is available and
2962                  * was used.
2963                  */
2964                 if ((vcpu->kvm->arch.use_cmma) &&
2965                     (vcpu->kvm->mm->context.use_cmma))
2966                         vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
2967                 goto retry;
2968         }
2969 
2970         /* nothing to do, just clear the request */
2971         kvm_clear_request(KVM_REQ_UNHALT, vcpu);
2972 
2973         return 0;
2974 }
2975 
2976 void kvm_s390_set_tod_clock(struct kvm *kvm,
2977                             const struct kvm_s390_vm_tod_clock *gtod)
2978 {
2979         struct kvm_vcpu *vcpu;
2980         struct kvm_s390_tod_clock_ext htod;
2981         int i;
2982 
2983         mutex_lock(&kvm->lock);
2984         preempt_disable();
2985 
2986         get_tod_clock_ext((char *)&htod);
2987 
2988         kvm->arch.epoch = gtod->tod - htod.tod;
2989         kvm->arch.epdx = 0;
2990         if (test_kvm_facility(kvm, 139)) {
2991                 kvm->arch.epdx = gtod->epoch_idx - htod.epoch_idx;
2992                 if (kvm->arch.epoch > gtod->tod)
2993                         kvm->arch.epdx -= 1;
2994         }
2995 
2996         kvm_s390_vcpu_block_all(kvm);
2997         kvm_for_each_vcpu(i, vcpu, kvm) {
2998                 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2999                 vcpu->arch.sie_block->epdx  = kvm->arch.epdx;
3000         }
3001 
3002         kvm_s390_vcpu_unblock_all(kvm);
3003         preempt_enable();
3004         mutex_unlock(&kvm->lock);
3005 }
3006 
3007 /**
3008  * kvm_arch_fault_in_page - fault-in guest page if necessary
3009  * @vcpu: The corresponding virtual cpu
3010  * @gpa: Guest physical address
3011  * @writable: Whether the page should be writable or not
3012  *
3013  * Make sure that a guest page has been faulted-in on the host.
3014  *
3015  * Return: Zero on success, negative error code otherwise.
3016  */
3017 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
3018 {
3019         return gmap_fault(vcpu->arch.gmap, gpa,
3020                           writable ? FAULT_FLAG_WRITE : 0);
3021 }
3022 
3023 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
3024                                       unsigned long token)
3025 {
3026         struct kvm_s390_interrupt inti;
3027         struct kvm_s390_irq irq;
3028 
3029         if (start_token) {
3030                 irq.u.ext.ext_params2 = token;
3031                 irq.type = KVM_S390_INT_PFAULT_INIT;
3032                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3033         } else {
3034                 inti.type = KVM_S390_INT_PFAULT_DONE;
3035                 inti.parm64 = token;
3036                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
3037         }
3038 }
3039 
3040 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
3041                                      struct kvm_async_pf *work)
3042 {
3043         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
3044         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
3045 }
3046 
3047 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
3048                                  struct kvm_async_pf *work)
3049 {
3050         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
3051         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
3052 }
3053 
3054 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
3055                                struct kvm_async_pf *work)
3056 {
3057         /* s390 will always inject the page directly */
3058 }
3059 
3060 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
3061 {
3062         /*
3063          * s390 will always inject the page directly,
3064          * but we still want check_async_completion to cleanup
3065          */
3066         return true;
3067 }
3068 
3069 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
3070 {
3071         hva_t hva;
3072         struct kvm_arch_async_pf arch;
3073         int rc;
3074 
3075         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3076                 return 0;
3077         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
3078             vcpu->arch.pfault_compare)
3079                 return 0;
3080         if (psw_extint_disabled(vcpu))
3081                 return 0;
3082         if (kvm_s390_vcpu_has_irq(vcpu, 0))
3083                 return 0;
3084         if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
3085                 return 0;
3086         if (!vcpu->arch.gmap->pfault_enabled)
3087                 return 0;
3088 
3089         hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
3090         hva += current->thread.gmap_addr & ~PAGE_MASK;
3091         if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3092                 return 0;
3093 
3094         rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
3095         return rc;
3096 }
3097 
3098 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
3099 {
3100         int rc, cpuflags;
3101 
3102         /*
3103          * On s390 notifications for arriving pages will be delivered directly
3104          * to the guest but the house keeping for completed pfaults is
3105          * handled outside the worker.
3106          */
3107         kvm_check_async_pf_completion(vcpu);
3108 
3109         vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
3110         vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
3111 
3112         if (need_resched())
3113                 schedule();
3114 
3115         if (test_cpu_flag(CIF_MCCK_PENDING))
3116                 s390_handle_mcck();
3117 
3118         if (!kvm_is_ucontrol(vcpu->kvm)) {
3119                 rc = kvm_s390_deliver_pending_interrupts(vcpu);
3120                 if (rc)
3121                         return rc;
3122         }
3123 
3124         rc = kvm_s390_handle_requests(vcpu);
3125         if (rc)
3126                 return rc;
3127 
3128         if (guestdbg_enabled(vcpu)) {
3129                 kvm_s390_backup_guest_per_regs(vcpu);
3130                 kvm_s390_patch_guest_per_regs(vcpu);
3131         }
3132 
3133         vcpu->arch.sie_block->icptcode = 0;
3134         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
3135         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
3136         trace_kvm_s390_sie_enter(vcpu, cpuflags);
3137 
3138         return 0;
3139 }
3140 
3141 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
3142 {
3143         struct kvm_s390_pgm_info pgm_info = {
3144                 .code = PGM_ADDRESSING,
3145         };
3146         u8 opcode, ilen;
3147         int rc;
3148 
3149         VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
3150         trace_kvm_s390_sie_fault(vcpu);
3151 
3152         /*
3153          * We want to inject an addressing exception, which is defined as a
3154          * suppressing or terminating exception. However, since we came here
3155          * by a DAT access exception, the PSW still points to the faulting
3156          * instruction since DAT exceptions are nullifying. So we've got
3157          * to look up the current opcode to get the length of the instruction
3158          * to be able to forward the PSW.
3159          */
3160         rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
3161         ilen = insn_length(opcode);
3162         if (rc < 0) {
3163                 return rc;
3164         } else if (rc) {
3165                 /* Instruction-Fetching Exceptions - we can't detect the ilen.
3166                  * Forward by arbitrary ilc, injection will take care of
3167                  * nullification if necessary.
3168                  */
3169                 pgm_info = vcpu->arch.pgm;
3170                 ilen = 4;
3171         }
3172         pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
3173         kvm_s390_forward_psw(vcpu, ilen);
3174         return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
3175 }
3176 
3177 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
3178 {
3179         struct mcck_volatile_info *mcck_info;
3180         struct sie_page *sie_page;
3181 
3182         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
3183                    vcpu->arch.sie_block->icptcode);
3184         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
3185 
3186         if (guestdbg_enabled(vcpu))
3187                 kvm_s390_restore_guest_per_regs(vcpu);
3188 
3189         vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
3190         vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
3191 
3192         if (exit_reason == -EINTR) {
3193                 VCPU_EVENT(vcpu, 3, "%s", "machine check");
3194                 sie_page = container_of(vcpu->arch.sie_block,
3195                                         struct sie_page, sie_block);
3196                 mcck_info = &sie_page->mcck_info;
3197                 kvm_s390_reinject_machine_check(vcpu, mcck_info);
3198                 return 0;
3199         }
3200 
3201         if (vcpu->arch.sie_block->icptcode > 0) {
3202                 int rc = kvm_handle_sie_intercept(vcpu);
3203 
3204                 if (rc != -EOPNOTSUPP)
3205                         return rc;
3206                 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
3207                 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
3208                 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
3209                 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
3210                 return -EREMOTE;
3211         } else if (exit_reason != -EFAULT) {
3212                 vcpu->stat.exit_null++;
3213                 return 0;
3214         } else if (kvm_is_ucontrol(vcpu->kvm)) {
3215                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
3216                 vcpu->run->s390_ucontrol.trans_exc_code =
3217                                                 current->thread.gmap_addr;
3218                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
3219                 return -EREMOTE;
3220         } else if (current->thread.gmap_pfault) {
3221                 trace_kvm_s390_major_guest_pfault(vcpu);
3222                 current->thread.gmap_pfault = 0;
3223                 if (kvm_arch_setup_async_pf(vcpu))
3224                         return 0;
3225                 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
3226         }
3227         return vcpu_post_run_fault_in_sie(vcpu);
3228 }
3229 
3230 static int __vcpu_run(struct kvm_vcpu *vcpu)
3231 {
3232         int rc, exit_reason;
3233 
3234         /*
3235          * We try to hold kvm->srcu during most of vcpu_run (except when run-
3236          * ning the guest), so that memslots (and other stuff) are protected
3237          */
3238         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3239 
3240         do {
3241                 rc = vcpu_pre_run(vcpu);
3242                 if (rc)
3243                         break;
3244 
3245                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
3246                 /*
3247                  * As PF_VCPU will be used in fault handler, between
3248                  * guest_enter and guest_exit should be no uaccess.
3249                  */
3250                 local_irq_disable();
3251                 guest_enter_irqoff();
3252                 __disable_cpu_timer_accounting(vcpu);
3253                 local_irq_enable();
3254                 exit_reason = sie64a(vcpu->arch.sie_block,
3255                                      vcpu->run->s.regs.gprs);
3256                 local_irq_disable();
3257                 __enable_cpu_timer_accounting(vcpu);
3258                 guest_exit_irqoff();
3259                 local_irq_enable();
3260                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3261 
3262                 rc = vcpu_post_run(vcpu, exit_reason);
3263         } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3264 
3265         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
3266         return rc;
3267 }
3268 
3269 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3270 {
3271         struct runtime_instr_cb *riccb;
3272         struct gs_cb *gscb;
3273 
3274         riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
3275         gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
3276         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
3277         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
3278         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
3279                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
3280         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
3281                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
3282                 /* some control register changes require a tlb flush */
3283                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3284         }
3285         if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
3286                 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
3287                 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
3288                 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
3289                 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
3290                 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
3291         }
3292         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
3293                 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
3294                 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
3295                 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
3296                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3297                         kvm_clear_async_pf_completion_queue(vcpu);
3298         }
3299         /*
3300          * If userspace sets the riccb (e.g. after migration) to a valid state,
3301          * we should enable RI here instead of doing the lazy enablement.
3302          */
3303         if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
3304             test_kvm_facility(vcpu->kvm, 64) &&
3305             riccb->v &&
3306             !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
3307                 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
3308                 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
3309         }
3310         /*
3311          * If userspace sets the gscb (e.g. after migration) to non-zero,
3312          * we should enable GS here instead of doing the lazy enablement.
3313          */
3314         if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
3315             test_kvm_facility(vcpu->kvm, 133) &&
3316             gscb->gssm &&
3317             !vcpu->arch.gs_enabled) {
3318                 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
3319                 vcpu->arch.sie_block->ecb |= ECB_GS;
3320                 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3321                 vcpu->arch.gs_enabled = 1;
3322         }
3323         if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) &&
3324             test_kvm_facility(vcpu->kvm, 82)) {
3325                 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
3326                 vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
3327         }
3328         save_access_regs(vcpu->arch.host_acrs);
3329         restore_access_regs(vcpu->run->s.regs.acrs);
3330         /* save host (userspace) fprs/vrs */
3331         save_fpu_regs();
3332         vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
3333         vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
3334         if (MACHINE_HAS_VX)
3335                 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
3336         else
3337                 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
3338         current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
3339         if (test_fp_ctl(current->thread.fpu.fpc))
3340                 /* User space provided an invalid FPC, let's clear it */
3341                 current->thread.fpu.fpc = 0;
3342         if (MACHINE_HAS_GS) {
3343                 preempt_disable();
3344                 __ctl_set_bit(2, 4);
3345                 if (current->thread.gs_cb) {
3346                         vcpu->arch.host_gscb = current->thread.gs_cb;
3347                         save_gs_cb(vcpu->arch.host_gscb);
3348                 }
3349                 if (vcpu->arch.gs_enabled) {
3350                         current->thread.gs_cb = (struct gs_cb *)
3351                                                 &vcpu->run->s.regs.gscb;
3352                         restore_gs_cb(current->thread.gs_cb);
3353                 }
3354                 preempt_enable();
3355         }
3356 
3357         kvm_run->kvm_dirty_regs = 0;
3358 }
3359 
3360 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3361 {
3362         kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
3363         kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
3364         kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
3365         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
3366         kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
3367         kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
3368         kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
3369         kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
3370         kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
3371         kvm_run->s.regs.pft = vcpu->arch.pfault_token;
3372         kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
3373         kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
3374         kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
3375         save_access_regs(vcpu->run->s.regs.acrs);
3376         restore_access_regs(vcpu->arch.host_acrs);
3377         /* Save guest register state */
3378         save_fpu_regs();
3379         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3380         /* Restore will be done lazily at return */
3381         current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
3382         current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
3383         if (MACHINE_HAS_GS) {
3384                 __ctl_set_bit(2, 4);
3385                 if (vcpu->arch.gs_enabled)
3386                         save_gs_cb(current->thread.gs_cb);
3387                 preempt_disable();
3388                 current->thread.gs_cb = vcpu->arch.host_gscb;
3389                 restore_gs_cb(vcpu->arch.host_gscb);
3390                 preempt_enable();
3391                 if (!vcpu->arch.host_gscb)
3392                         __ctl_clear_bit(2, 4);
3393                 vcpu->arch.host_gscb = NULL;
3394         }
3395 
3396 }
3397 
3398 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3399 {
3400         int rc;
3401 
3402         if (kvm_run->immediate_exit)
3403                 return -EINTR;
3404 
3405         if (guestdbg_exit_pending(vcpu)) {
3406                 kvm_s390_prepare_debug_exit(vcpu);
3407                 return 0;
3408         }
3409 
3410         kvm_sigset_activate(vcpu);
3411 
3412         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
3413                 kvm_s390_vcpu_start(vcpu);
3414         } else if (is_vcpu_stopped(vcpu)) {
3415                 pr_err_ratelimited("can't run stopped vcpu %d\n",
3416                                    vcpu->vcpu_id);
3417                 return -EINVAL;
3418         }
3419 
3420         sync_regs(vcpu, kvm_run);
3421         enable_cpu_timer_accounting(vcpu);
3422 
3423         might_fault();
3424         rc = __vcpu_run(vcpu);
3425 
3426         if (signal_pending(current) && !rc) {
3427                 kvm_run->exit_reason = KVM_EXIT_INTR;
3428                 rc = -EINTR;
3429         }
3430 
3431         if (guestdbg_exit_pending(vcpu) && !rc)  {
3432                 kvm_s390_prepare_debug_exit(vcpu);
3433                 rc = 0;
3434         }
3435 
3436         if (rc == -EREMOTE) {
3437                 /* userspace support is needed, kvm_run has been prepared */
3438                 rc = 0;
3439         }
3440 
3441         disable_cpu_timer_accounting(vcpu);
3442         store_regs(vcpu, kvm_run);
3443 
3444         kvm_sigset_deactivate(vcpu);
3445 
3446         vcpu->stat.exit_userspace++;
3447         return rc;
3448 }
3449 
3450 /*
3451  * store status at address
3452  * we use have two special cases:
3453  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
3454  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
3455  */
3456 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
3457 {
3458         unsigned char archmode = 1;
3459         freg_t fprs[NUM_FPRS];
3460         unsigned int px;
3461         u64 clkcomp, cputm;
3462         int rc;
3463 
3464         px = kvm_s390_get_prefix(vcpu);
3465         if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
3466                 if (write_guest_abs(vcpu, 163, &archmode, 1))
3467                         return -EFAULT;
3468                 gpa = 0;
3469         } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
3470                 if (write_guest_real(vcpu, 163, &archmode, 1))
3471                         return -EFAULT;
3472                 gpa = px;
3473         } else
3474                 gpa -= __LC_FPREGS_SAVE_AREA;
3475 
3476         /* manually convert vector registers if necessary */
3477         if (MACHINE_HAS_VX) {
3478                 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
3479                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
3480                                      fprs, 128);
3481         } else {
3482                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
3483                                      vcpu->run->s.regs.fprs, 128);
3484         }
3485         rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
3486                               vcpu->run->s.regs.gprs, 128);
3487         rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
3488                               &vcpu->arch.sie_block->gpsw, 16);
3489         rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
3490                               &px, 4);
3491         rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
3492                               &vcpu->run->s.regs.fpc, 4);
3493         rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
3494                               &vcpu->arch.sie_block->todpr, 4);
3495         cputm = kvm_s390_get_cpu_timer(vcpu);
3496         rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
3497                               &cputm, 8);
3498         clkcomp = vcpu->arch.sie_block->ckc >> 8;
3499         rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
3500                               &clkcomp, 8);
3501         rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
3502                               &vcpu->run->s.regs.acrs, 64);
3503         rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
3504                               &vcpu->arch.sie_block->gcr, 128);
3505         return rc ? -EFAULT : 0;
3506 }
3507 
3508 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
3509 {
3510         /*
3511          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
3512          * switch in the run ioctl. Let's update our copies before we save
3513          * it into the save area
3514          */
3515         save_fpu_regs();
3516         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3517         save_access_regs(vcpu->run->s.regs.acrs);
3518 
3519         return kvm_s390_store_status_unloaded(vcpu, addr);
3520 }
3521 
3522 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3523 {
3524         kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
3525         kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
3526 }
3527 
3528 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
3529 {
3530         unsigned int i;
3531         struct kvm_vcpu *vcpu;
3532 
3533         kvm_for_each_vcpu(i, vcpu, kvm) {
3534                 __disable_ibs_on_vcpu(vcpu);
3535         }
3536 }
3537 
3538 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3539 {
3540         if (!sclp.has_ibs)
3541                 return;
3542         kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
3543         kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
3544 }
3545 
3546 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
3547 {
3548         int i, online_vcpus, started_vcpus = 0;
3549 
3550         if (!is_vcpu_stopped(vcpu))
3551                 return;
3552 
3553         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
3554         /* Only one cpu at a time may enter/leave the STOPPED state. */
3555         spin_lock(&vcpu->kvm->arch.start_stop_lock);
3556         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3557 
3558         for (i = 0; i < online_vcpus; i++) {
3559                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
3560                         started_vcpus++;
3561         }
3562 
3563         if (started_vcpus == 0) {
3564                 /* we're the only active VCPU -> speed it up */
3565                 __enable_ibs_on_vcpu(vcpu);
3566         } else if (started_vcpus == 1) {
3567                 /*
3568                  * As we are starting a second VCPU, we have to disable
3569                  * the IBS facility on all VCPUs to remove potentially
3570                  * oustanding ENABLE requests.
3571                  */
3572                 __disable_ibs_on_all_vcpus(vcpu->kvm);
3573         }
3574 
3575         atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
3576         /*
3577          * Another VCPU might have used IBS while we were offline.
3578          * Let's play safe and flush the VCPU at startup.
3579          */
3580         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3581         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
3582         return;
3583 }
3584 
3585 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
3586 {
3587         int i, online_vcpus, started_vcpus = 0;
3588         struct kvm_vcpu *started_vcpu = NULL;
3589 
3590         if (is_vcpu_stopped(vcpu))
3591                 return;
3592 
3593         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
3594         /* Only one cpu at a time may enter/leave the STOPPED state. */
3595         spin_lock(&vcpu->kvm->arch.start_stop_lock);
3596         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3597 
3598         /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
3599         kvm_s390_clear_stop_irq(vcpu);
3600 
3601         atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
3602         __disable_ibs_on_vcpu(vcpu);
3603 
3604         for (i = 0; i < online_vcpus; i++) {
3605                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
3606                         started_vcpus++;
3607                         started_vcpu = vcpu->kvm->vcpus[i];
3608                 }
3609         }
3610 
3611         if (started_vcpus == 1) {
3612                 /*
3613                  * As we only have one VCPU left, we want to enable the
3614                  * IBS facility for that VCPU to speed it up.
3615                  */
3616                 __enable_ibs_on_vcpu(started_vcpu);
3617         }
3618 
3619         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
3620         return;
3621 }
3622 
3623 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
3624                                      struct kvm_enable_cap *cap)
3625 {
3626         int r;
3627 
3628         if (cap->flags)
3629                 return -EINVAL;
3630 
3631         switch (cap->cap) {
3632         case KVM_CAP_S390_CSS_SUPPORT:
3633                 if (!vcpu->kvm->arch.css_support) {
3634                         vcpu->kvm->arch.css_support = 1;
3635                         VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
3636                         trace_kvm_s390_enable_css(vcpu->kvm);
3637                 }
3638                 r = 0;
3639                 break;
3640         default:
3641                 r = -EINVAL;
3642                 break;
3643         }
3644         return r;
3645 }
3646 
3647 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
3648                                   struct kvm_s390_mem_op *mop)
3649 {
3650         void __user *uaddr = (void __user *)mop->buf;
3651         void *tmpbuf = NULL;
3652         int r, srcu_idx;
3653         const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
3654                                     | KVM_S390_MEMOP_F_CHECK_ONLY;
3655 
3656         if (mop->flags & ~supported_flags)
3657                 return -EINVAL;
3658 
3659         if (mop->size > MEM_OP_MAX_SIZE)
3660                 return -E2BIG;
3661 
3662         if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
3663                 tmpbuf = vmalloc(mop->size);
3664                 if (!tmpbuf)
3665                         return -ENOMEM;
3666         }
3667 
3668         srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3669 
3670         switch (mop->op) {
3671         case KVM_S390_MEMOP_LOGICAL_READ:
3672                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
3673                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3674                                             mop->size, GACC_FETCH);
3675                         break;
3676                 }
3677                 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3678                 if (r == 0) {
3679                         if (copy_to_user(uaddr, tmpbuf, mop->size))
3680                                 r = -EFAULT;
3681                 }
3682                 break;
3683         case KVM_S390_MEMOP_LOGICAL_WRITE:
3684                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
3685                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3686                                             mop->size, GACC_STORE);
3687                         break;
3688                 }
3689                 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
3690                         r = -EFAULT;
3691                         break;
3692                 }
3693                 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3694                 break;
3695         default:
3696                 r = -EINVAL;
3697         }
3698 
3699         srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
3700 
3701         if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
3702                 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
3703 
3704         vfree(tmpbuf);
3705         return r;
3706 }
3707 
3708 long kvm_arch_vcpu_ioctl(struct file *filp,
3709                          unsigned int ioctl, unsigned long arg)
3710 {
3711         struct kvm_vcpu *vcpu = filp->private_data;
3712         void __user *argp = (void __user *)arg;
3713         int idx;
3714         long r;
3715 
3716         switch (ioctl) {
3717         case KVM_S390_IRQ: {
3718                 struct kvm_s390_irq s390irq;
3719 
3720                 r = -EFAULT;
3721                 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
3722                         break;
3723                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3724                 break;
3725         }
3726         case KVM_S390_INTERRUPT: {
3727                 struct kvm_s390_interrupt s390int;
3728                 struct kvm_s390_irq s390irq;
3729 
3730                 r = -EFAULT;
3731                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
3732                         break;
3733                 if (s390int_to_s390irq(&s390int, &s390irq))
3734                         return -EINVAL;
3735                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3736                 break;
3737         }
3738         case KVM_S390_STORE_STATUS:
3739                 idx = srcu_read_lock(&vcpu->kvm->srcu);
3740                 r = kvm_s390_vcpu_store_status(vcpu, arg);
3741                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
3742                 break;
3743         case KVM_S390_SET_INITIAL_PSW: {
3744                 psw_t psw;
3745 
3746                 r = -EFAULT;
3747                 if (copy_from_user(&psw, argp, sizeof(psw)))
3748                         break;
3749                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3750                 break;
3751         }
3752         case KVM_S390_INITIAL_RESET:
3753                 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3754                 break;
3755         case KVM_SET_ONE_REG:
3756         case KVM_GET_ONE_REG: {
3757                 struct kvm_one_reg reg;
3758                 r = -EFAULT;
3759                 if (copy_from_user(&reg, argp, sizeof(reg)))
3760                         break;
3761                 if (ioctl == KVM_SET_ONE_REG)
3762                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3763                 else
3764                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3765                 break;
3766         }
3767 #ifdef CONFIG_KVM_S390_UCONTROL
3768         case KVM_S390_UCAS_MAP: {
3769                 struct kvm_s390_ucas_mapping ucasmap;
3770 
3771                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3772                         r = -EFAULT;
3773                         break;
3774                 }
3775 
3776                 if (!kvm_is_ucontrol(vcpu->kvm)) {
3777                         r = -EINVAL;
3778                         break;
3779                 }
3780 
3781                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3782                                      ucasmap.vcpu_addr, ucasmap.length);
3783                 break;
3784         }
3785         case KVM_S390_UCAS_UNMAP: {
3786                 struct kvm_s390_ucas_mapping ucasmap;
3787 
3788                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3789                         r = -EFAULT;
3790                         break;
3791                 }
3792 
3793                 if (!kvm_is_ucontrol(vcpu->kvm)) {
3794                         r = -EINVAL;
3795                         break;
3796                 }
3797 
3798                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3799                         ucasmap.length);
3800                 break;
3801         }
3802 #endif
3803         case KVM_S390_VCPU_FAULT: {
3804                 r = gmap_fault(vcpu->arch.gmap, arg, 0);
3805                 break;
3806         }
3807         case KVM_ENABLE_CAP:
3808         {
3809                 struct kvm_enable_cap cap;
3810                 r = -EFAULT;
3811                 if (copy_from_user(&cap, argp, sizeof(cap)))
3812                         break;
3813                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3814                 break;
3815         }
3816         case KVM_S390_MEM_OP: {
3817                 struct kvm_s390_mem_op mem_op;
3818 
3819                 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3820                         r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3821                 else
3822                         r = -EFAULT;
3823                 break;
3824         }
3825         case KVM_S390_SET_IRQ_STATE: {
3826                 struct kvm_s390_irq_state irq_state;
3827 
3828                 r = -EFAULT;
3829                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3830                         break;
3831                 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3832                     irq_state.len == 0 ||
3833                     irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3834                         r = -EINVAL;
3835                         break;
3836                 }
3837                 /* do not use irq_state.flags, it will break old QEMUs */
3838                 r = kvm_s390_set_irq_state(vcpu,
3839                                            (void __user *) irq_state.buf,
3840                                            irq_state.len);
3841                 break;
3842         }
3843         case KVM_S390_GET_IRQ_STATE: {
3844                 struct kvm_s390_irq_state irq_state;
3845 
3846                 r = -EFAULT;
3847                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3848                         break;
3849                 if (irq_state.len == 0) {
3850                         r = -EINVAL;
3851                         break;
3852                 }
3853                 /* do not use irq_state.flags, it will break old QEMUs */
3854                 r = kvm_s390_get_irq_state(vcpu,
3855                                            (__u8 __user *)  irq_state.buf,
3856                                            irq_state.len);
3857                 break;
3858         }
3859         default:
3860                 r = -ENOTTY;
3861         }
3862         return r;
3863 }
3864 
3865 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3866 {
3867 #ifdef CONFIG_KVM_S390_UCONTROL
3868         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3869                  && (kvm_is_ucontrol(vcpu->kvm))) {
3870                 vmf->page = virt_to_page(vcpu->arch.sie_block);
3871                 get_page(vmf->page);
3872                 return 0;
3873         }
3874 #endif
3875         return VM_FAULT_SIGBUS;
3876 }
3877 
3878 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3879                             unsigned long npages)
3880 {
3881         return 0;
3882 }
3883 
3884 /* Section: memory related */
3885 int kvm_arch_prepare_memory_region(struct kvm *kvm,
3886                                    struct kvm_memory_slot *memslot,
3887                                    const struct kvm_userspace_memory_region *mem,
3888                                    enum kvm_mr_change change)
3889 {
3890         /* A few sanity checks. We can have memory slots which have to be
3891            located/ended at a segment boundary (1MB). The memory in userland is
3892            ok to be fragmented into various different vmas. It is okay to mmap()
3893            and munmap() stuff in this slot after doing this call at any time */
3894 
3895         if (mem->userspace_addr & 0xffffful)
3896                 return -EINVAL;
3897 
3898         if (mem->memory_size & 0xffffful)
3899                 return -EINVAL;
3900 
3901         if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3902                 return -EINVAL;
3903 
3904         return 0;
3905 }
3906 
3907 void kvm_arch_commit_memory_region(struct kvm *kvm,
3908                                 const struct kvm_userspace_memory_region *mem,
3909                                 const struct kvm_memory_slot *old,
3910                                 const struct kvm_memory_slot *new,
3911                                 enum kvm_mr_change change)
3912 {
3913         int rc;
3914 
3915         /* If the basics of the memslot do not change, we do not want
3916          * to update the gmap. Every update causes several unnecessary
3917          * segment translation exceptions. This is usually handled just
3918          * fine by the normal fault handler + gmap, but it will also
3919          * cause faults on the prefix page of running guest CPUs.
3920          */
3921         if (old->userspace_addr == mem->userspace_addr &&
3922             old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3923             old->npages * PAGE_SIZE == mem->memory_size)
3924                 return;
3925 
3926         rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3927                 mem->guest_phys_addr, mem->memory_size);
3928         if (rc)
3929                 pr_warn("failed to commit memory region\n");
3930         return;
3931 }
3932 
3933 static inline unsigned long nonhyp_mask(int i)
3934 {
3935         unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3936 
3937         return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3938 }
3939 
3940 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3941 {
3942         vcpu->valid_wakeup = false;
3943 }
3944 
3945 static int __init kvm_s390_init(void)
3946 {
3947         int i;
3948 
3949         if (!sclp.has_sief2) {
3950                 pr_info("SIE not available\n");
3951                 return -ENODEV;
3952         }
3953 
3954         for (i = 0; i < 16; i++)
3955                 kvm_s390_fac_list_mask[i] |=
3956                         S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3957 
3958         return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
3959 }
3960 
3961 static void __exit kvm_s390_exit(void)
3962 {
3963         kvm_exit();
3964 }
3965 
3966 module_init(kvm_s390_init);
3967 module_exit(kvm_s390_exit);
3968 
3969 /*
3970  * Enable autoloading of the kvm module.
3971  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3972  * since x86 takes a different approach.
3973  */
3974 #include <linux/miscdevice.h>
3975 MODULE_ALIAS_MISCDEV(KVM_MINOR);
3976 MODULE_ALIAS("devname:kvm");
3977 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp