~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/mips/kvm/trap_emul.c

Version: ~ [ linux-5.14-rc3 ] ~ [ linux-5.13.5 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.53 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.135 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.198 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.240 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.276 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.276 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * This file is subject to the terms and conditions of the GNU General Public
  3  * License.  See the file "COPYING" in the main directory of this archive
  4  * for more details.
  5  *
  6  * KVM/MIPS: Deliver/Emulate exceptions to the guest kernel
  7  *
  8  * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
  9  * Authors: Sanjay Lal <sanjayl@kymasys.com>
 10  */
 11 
 12 #include <linux/errno.h>
 13 #include <linux/err.h>
 14 #include <linux/kvm_host.h>
 15 #include <linux/log2.h>
 16 #include <linux/uaccess.h>
 17 #include <linux/vmalloc.h>
 18 #include <asm/mmu_context.h>
 19 #include <asm/pgalloc.h>
 20 
 21 #include "interrupt.h"
 22 
 23 static gpa_t kvm_trap_emul_gva_to_gpa_cb(gva_t gva)
 24 {
 25         gpa_t gpa;
 26         gva_t kseg = KSEGX(gva);
 27         gva_t gkseg = KVM_GUEST_KSEGX(gva);
 28 
 29         if ((kseg == CKSEG0) || (kseg == CKSEG1))
 30                 gpa = CPHYSADDR(gva);
 31         else if (gkseg == KVM_GUEST_KSEG0)
 32                 gpa = KVM_GUEST_CPHYSADDR(gva);
 33         else {
 34                 kvm_err("%s: cannot find GPA for GVA: %#lx\n", __func__, gva);
 35                 kvm_mips_dump_host_tlbs();
 36                 gpa = KVM_INVALID_ADDR;
 37         }
 38 
 39         kvm_debug("%s: gva %#lx, gpa: %#llx\n", __func__, gva, gpa);
 40 
 41         return gpa;
 42 }
 43 
 44 static int kvm_trap_emul_no_handler(struct kvm_vcpu *vcpu)
 45 {
 46         u32 __user *opc = (u32 __user *) vcpu->arch.pc;
 47         u32 cause = vcpu->arch.host_cp0_cause;
 48         u32 exccode = (cause & CAUSEF_EXCCODE) >> CAUSEB_EXCCODE;
 49         unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr;
 50         u32 inst = 0;
 51 
 52         /*
 53          *  Fetch the instruction.
 54          */
 55         if (cause & CAUSEF_BD)
 56                 opc += 1;
 57         kvm_get_badinstr(opc, vcpu, &inst);
 58 
 59         kvm_err("Exception Code: %d not handled @ PC: %p, inst: 0x%08x BadVaddr: %#lx Status: %#x\n",
 60                 exccode, opc, inst, badvaddr,
 61                 kvm_read_c0_guest_status(vcpu->arch.cop0));
 62         kvm_arch_vcpu_dump_regs(vcpu);
 63         vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 64         return RESUME_HOST;
 65 }
 66 
 67 static int kvm_trap_emul_handle_cop_unusable(struct kvm_vcpu *vcpu)
 68 {
 69         struct mips_coproc *cop0 = vcpu->arch.cop0;
 70         struct kvm_run *run = vcpu->run;
 71         u32 __user *opc = (u32 __user *) vcpu->arch.pc;
 72         u32 cause = vcpu->arch.host_cp0_cause;
 73         enum emulation_result er = EMULATE_DONE;
 74         int ret = RESUME_GUEST;
 75 
 76         if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 1) {
 77                 /* FPU Unusable */
 78                 if (!kvm_mips_guest_has_fpu(&vcpu->arch) ||
 79                     (kvm_read_c0_guest_status(cop0) & ST0_CU1) == 0) {
 80                         /*
 81                          * Unusable/no FPU in guest:
 82                          * deliver guest COP1 Unusable Exception
 83                          */
 84                         er = kvm_mips_emulate_fpu_exc(cause, opc, run, vcpu);
 85                 } else {
 86                         /* Restore FPU state */
 87                         kvm_own_fpu(vcpu);
 88                         er = EMULATE_DONE;
 89                 }
 90         } else {
 91                 er = kvm_mips_emulate_inst(cause, opc, run, vcpu);
 92         }
 93 
 94         switch (er) {
 95         case EMULATE_DONE:
 96                 ret = RESUME_GUEST;
 97                 break;
 98 
 99         case EMULATE_FAIL:
100                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
101                 ret = RESUME_HOST;
102                 break;
103 
104         case EMULATE_WAIT:
105                 run->exit_reason = KVM_EXIT_INTR;
106                 ret = RESUME_HOST;
107                 break;
108 
109         case EMULATE_HYPERCALL:
110                 ret = kvm_mips_handle_hypcall(vcpu);
111                 break;
112 
113         default:
114                 BUG();
115         }
116         return ret;
117 }
118 
119 static int kvm_mips_bad_load(u32 cause, u32 *opc, struct kvm_run *run,
120                              struct kvm_vcpu *vcpu)
121 {
122         enum emulation_result er;
123         union mips_instruction inst;
124         int err;
125 
126         /* A code fetch fault doesn't count as an MMIO */
127         if (kvm_is_ifetch_fault(&vcpu->arch)) {
128                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
129                 return RESUME_HOST;
130         }
131 
132         /* Fetch the instruction. */
133         if (cause & CAUSEF_BD)
134                 opc += 1;
135         err = kvm_get_badinstr(opc, vcpu, &inst.word);
136         if (err) {
137                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
138                 return RESUME_HOST;
139         }
140 
141         /* Emulate the load */
142         er = kvm_mips_emulate_load(inst, cause, run, vcpu);
143         if (er == EMULATE_FAIL) {
144                 kvm_err("Emulate load from MMIO space failed\n");
145                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
146         } else {
147                 run->exit_reason = KVM_EXIT_MMIO;
148         }
149         return RESUME_HOST;
150 }
151 
152 static int kvm_mips_bad_store(u32 cause, u32 *opc, struct kvm_run *run,
153                               struct kvm_vcpu *vcpu)
154 {
155         enum emulation_result er;
156         union mips_instruction inst;
157         int err;
158 
159         /* Fetch the instruction. */
160         if (cause & CAUSEF_BD)
161                 opc += 1;
162         err = kvm_get_badinstr(opc, vcpu, &inst.word);
163         if (err) {
164                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
165                 return RESUME_HOST;
166         }
167 
168         /* Emulate the store */
169         er = kvm_mips_emulate_store(inst, cause, run, vcpu);
170         if (er == EMULATE_FAIL) {
171                 kvm_err("Emulate store to MMIO space failed\n");
172                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
173         } else {
174                 run->exit_reason = KVM_EXIT_MMIO;
175         }
176         return RESUME_HOST;
177 }
178 
179 static int kvm_mips_bad_access(u32 cause, u32 *opc, struct kvm_run *run,
180                                struct kvm_vcpu *vcpu, bool store)
181 {
182         if (store)
183                 return kvm_mips_bad_store(cause, opc, run, vcpu);
184         else
185                 return kvm_mips_bad_load(cause, opc, run, vcpu);
186 }
187 
188 static int kvm_trap_emul_handle_tlb_mod(struct kvm_vcpu *vcpu)
189 {
190         struct mips_coproc *cop0 = vcpu->arch.cop0;
191         struct kvm_run *run = vcpu->run;
192         u32 __user *opc = (u32 __user *) vcpu->arch.pc;
193         unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr;
194         u32 cause = vcpu->arch.host_cp0_cause;
195         struct kvm_mips_tlb *tlb;
196         unsigned long entryhi;
197         int index;
198 
199         if (KVM_GUEST_KSEGX(badvaddr) < KVM_GUEST_KSEG0
200             || KVM_GUEST_KSEGX(badvaddr) == KVM_GUEST_KSEG23) {
201                 /*
202                  * First find the mapping in the guest TLB. If the failure to
203                  * write was due to the guest TLB, it should be up to the guest
204                  * to handle it.
205                  */
206                 entryhi = (badvaddr & VPN2_MASK) |
207                           (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
208                 index = kvm_mips_guest_tlb_lookup(vcpu, entryhi);
209 
210                 /*
211                  * These should never happen.
212                  * They would indicate stale host TLB entries.
213                  */
214                 if (unlikely(index < 0)) {
215                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
216                         return RESUME_HOST;
217                 }
218                 tlb = vcpu->arch.guest_tlb + index;
219                 if (unlikely(!TLB_IS_VALID(*tlb, badvaddr))) {
220                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
221                         return RESUME_HOST;
222                 }
223 
224                 /*
225                  * Guest entry not dirty? That would explain the TLB modified
226                  * exception. Relay that on to the guest so it can handle it.
227                  */
228                 if (!TLB_IS_DIRTY(*tlb, badvaddr)) {
229                         kvm_mips_emulate_tlbmod(cause, opc, run, vcpu);
230                         return RESUME_GUEST;
231                 }
232 
233                 if (kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb, badvaddr,
234                                                          true))
235                         /* Not writable, needs handling as MMIO */
236                         return kvm_mips_bad_store(cause, opc, run, vcpu);
237                 return RESUME_GUEST;
238         } else if (KVM_GUEST_KSEGX(badvaddr) == KVM_GUEST_KSEG0) {
239                 if (kvm_mips_handle_kseg0_tlb_fault(badvaddr, vcpu, true) < 0)
240                         /* Not writable, needs handling as MMIO */
241                         return kvm_mips_bad_store(cause, opc, run, vcpu);
242                 return RESUME_GUEST;
243         } else {
244                 /* host kernel addresses are all handled as MMIO */
245                 return kvm_mips_bad_store(cause, opc, run, vcpu);
246         }
247 }
248 
249 static int kvm_trap_emul_handle_tlb_miss(struct kvm_vcpu *vcpu, bool store)
250 {
251         struct kvm_run *run = vcpu->run;
252         u32 __user *opc = (u32 __user *) vcpu->arch.pc;
253         unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr;
254         u32 cause = vcpu->arch.host_cp0_cause;
255         enum emulation_result er = EMULATE_DONE;
256         int ret = RESUME_GUEST;
257 
258         if (((badvaddr & PAGE_MASK) == KVM_GUEST_COMMPAGE_ADDR)
259             && KVM_GUEST_KERNEL_MODE(vcpu)) {
260                 if (kvm_mips_handle_commpage_tlb_fault(badvaddr, vcpu) < 0) {
261                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
262                         ret = RESUME_HOST;
263                 }
264         } else if (KVM_GUEST_KSEGX(badvaddr) < KVM_GUEST_KSEG0
265                    || KVM_GUEST_KSEGX(badvaddr) == KVM_GUEST_KSEG23) {
266                 kvm_debug("USER ADDR TLB %s fault: cause %#x, PC: %p, BadVaddr: %#lx\n",
267                           store ? "ST" : "LD", cause, opc, badvaddr);
268 
269                 /*
270                  * User Address (UA) fault, this could happen if
271                  * (1) TLB entry not present/valid in both Guest and shadow host
272                  *     TLBs, in this case we pass on the fault to the guest
273                  *     kernel and let it handle it.
274                  * (2) TLB entry is present in the Guest TLB but not in the
275                  *     shadow, in this case we inject the TLB from the Guest TLB
276                  *     into the shadow host TLB
277                  */
278 
279                 er = kvm_mips_handle_tlbmiss(cause, opc, run, vcpu, store);
280                 if (er == EMULATE_DONE)
281                         ret = RESUME_GUEST;
282                 else {
283                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
284                         ret = RESUME_HOST;
285                 }
286         } else if (KVM_GUEST_KSEGX(badvaddr) == KVM_GUEST_KSEG0) {
287                 /*
288                  * All KSEG0 faults are handled by KVM, as the guest kernel does
289                  * not expect to ever get them
290                  */
291                 if (kvm_mips_handle_kseg0_tlb_fault(badvaddr, vcpu, store) < 0)
292                         ret = kvm_mips_bad_access(cause, opc, run, vcpu, store);
293         } else if (KVM_GUEST_KERNEL_MODE(vcpu)
294                    && (KSEGX(badvaddr) == CKSEG0 || KSEGX(badvaddr) == CKSEG1)) {
295                 /*
296                  * With EVA we may get a TLB exception instead of an address
297                  * error when the guest performs MMIO to KSeg1 addresses.
298                  */
299                 ret = kvm_mips_bad_access(cause, opc, run, vcpu, store);
300         } else {
301                 kvm_err("Illegal TLB %s fault address , cause %#x, PC: %p, BadVaddr: %#lx\n",
302                         store ? "ST" : "LD", cause, opc, badvaddr);
303                 kvm_mips_dump_host_tlbs();
304                 kvm_arch_vcpu_dump_regs(vcpu);
305                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
306                 ret = RESUME_HOST;
307         }
308         return ret;
309 }
310 
311 static int kvm_trap_emul_handle_tlb_st_miss(struct kvm_vcpu *vcpu)
312 {
313         return kvm_trap_emul_handle_tlb_miss(vcpu, true);
314 }
315 
316 static int kvm_trap_emul_handle_tlb_ld_miss(struct kvm_vcpu *vcpu)
317 {
318         return kvm_trap_emul_handle_tlb_miss(vcpu, false);
319 }
320 
321 static int kvm_trap_emul_handle_addr_err_st(struct kvm_vcpu *vcpu)
322 {
323         struct kvm_run *run = vcpu->run;
324         u32 __user *opc = (u32 __user *) vcpu->arch.pc;
325         unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr;
326         u32 cause = vcpu->arch.host_cp0_cause;
327         int ret = RESUME_GUEST;
328 
329         if (KVM_GUEST_KERNEL_MODE(vcpu)
330             && (KSEGX(badvaddr) == CKSEG0 || KSEGX(badvaddr) == CKSEG1)) {
331                 ret = kvm_mips_bad_store(cause, opc, run, vcpu);
332         } else {
333                 kvm_err("Address Error (STORE): cause %#x, PC: %p, BadVaddr: %#lx\n",
334                         cause, opc, badvaddr);
335                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
336                 ret = RESUME_HOST;
337         }
338         return ret;
339 }
340 
341 static int kvm_trap_emul_handle_addr_err_ld(struct kvm_vcpu *vcpu)
342 {
343         struct kvm_run *run = vcpu->run;
344         u32 __user *opc = (u32 __user *) vcpu->arch.pc;
345         unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr;
346         u32 cause = vcpu->arch.host_cp0_cause;
347         int ret = RESUME_GUEST;
348 
349         if (KSEGX(badvaddr) == CKSEG0 || KSEGX(badvaddr) == CKSEG1) {
350                 ret = kvm_mips_bad_load(cause, opc, run, vcpu);
351         } else {
352                 kvm_err("Address Error (LOAD): cause %#x, PC: %p, BadVaddr: %#lx\n",
353                         cause, opc, badvaddr);
354                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
355                 ret = RESUME_HOST;
356         }
357         return ret;
358 }
359 
360 static int kvm_trap_emul_handle_syscall(struct kvm_vcpu *vcpu)
361 {
362         struct kvm_run *run = vcpu->run;
363         u32 __user *opc = (u32 __user *) vcpu->arch.pc;
364         u32 cause = vcpu->arch.host_cp0_cause;
365         enum emulation_result er = EMULATE_DONE;
366         int ret = RESUME_GUEST;
367 
368         er = kvm_mips_emulate_syscall(cause, opc, run, vcpu);
369         if (er == EMULATE_DONE)
370                 ret = RESUME_GUEST;
371         else {
372                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
373                 ret = RESUME_HOST;
374         }
375         return ret;
376 }
377 
378 static int kvm_trap_emul_handle_res_inst(struct kvm_vcpu *vcpu)
379 {
380         struct kvm_run *run = vcpu->run;
381         u32 __user *opc = (u32 __user *) vcpu->arch.pc;
382         u32 cause = vcpu->arch.host_cp0_cause;
383         enum emulation_result er = EMULATE_DONE;
384         int ret = RESUME_GUEST;
385 
386         er = kvm_mips_handle_ri(cause, opc, run, vcpu);
387         if (er == EMULATE_DONE)
388                 ret = RESUME_GUEST;
389         else {
390                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
391                 ret = RESUME_HOST;
392         }
393         return ret;
394 }
395 
396 static int kvm_trap_emul_handle_break(struct kvm_vcpu *vcpu)
397 {
398         struct kvm_run *run = vcpu->run;
399         u32 __user *opc = (u32 __user *) vcpu->arch.pc;
400         u32 cause = vcpu->arch.host_cp0_cause;
401         enum emulation_result er = EMULATE_DONE;
402         int ret = RESUME_GUEST;
403 
404         er = kvm_mips_emulate_bp_exc(cause, opc, run, vcpu);
405         if (er == EMULATE_DONE)
406                 ret = RESUME_GUEST;
407         else {
408                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
409                 ret = RESUME_HOST;
410         }
411         return ret;
412 }
413 
414 static int kvm_trap_emul_handle_trap(struct kvm_vcpu *vcpu)
415 {
416         struct kvm_run *run = vcpu->run;
417         u32 __user *opc = (u32 __user *)vcpu->arch.pc;
418         u32 cause = vcpu->arch.host_cp0_cause;
419         enum emulation_result er = EMULATE_DONE;
420         int ret = RESUME_GUEST;
421 
422         er = kvm_mips_emulate_trap_exc(cause, opc, run, vcpu);
423         if (er == EMULATE_DONE) {
424                 ret = RESUME_GUEST;
425         } else {
426                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
427                 ret = RESUME_HOST;
428         }
429         return ret;
430 }
431 
432 static int kvm_trap_emul_handle_msa_fpe(struct kvm_vcpu *vcpu)
433 {
434         struct kvm_run *run = vcpu->run;
435         u32 __user *opc = (u32 __user *)vcpu->arch.pc;
436         u32 cause = vcpu->arch.host_cp0_cause;
437         enum emulation_result er = EMULATE_DONE;
438         int ret = RESUME_GUEST;
439 
440         er = kvm_mips_emulate_msafpe_exc(cause, opc, run, vcpu);
441         if (er == EMULATE_DONE) {
442                 ret = RESUME_GUEST;
443         } else {
444                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
445                 ret = RESUME_HOST;
446         }
447         return ret;
448 }
449 
450 static int kvm_trap_emul_handle_fpe(struct kvm_vcpu *vcpu)
451 {
452         struct kvm_run *run = vcpu->run;
453         u32 __user *opc = (u32 __user *)vcpu->arch.pc;
454         u32 cause = vcpu->arch.host_cp0_cause;
455         enum emulation_result er = EMULATE_DONE;
456         int ret = RESUME_GUEST;
457 
458         er = kvm_mips_emulate_fpe_exc(cause, opc, run, vcpu);
459         if (er == EMULATE_DONE) {
460                 ret = RESUME_GUEST;
461         } else {
462                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
463                 ret = RESUME_HOST;
464         }
465         return ret;
466 }
467 
468 /**
469  * kvm_trap_emul_handle_msa_disabled() - Guest used MSA while disabled in root.
470  * @vcpu:       Virtual CPU context.
471  *
472  * Handle when the guest attempts to use MSA when it is disabled.
473  */
474 static int kvm_trap_emul_handle_msa_disabled(struct kvm_vcpu *vcpu)
475 {
476         struct mips_coproc *cop0 = vcpu->arch.cop0;
477         struct kvm_run *run = vcpu->run;
478         u32 __user *opc = (u32 __user *) vcpu->arch.pc;
479         u32 cause = vcpu->arch.host_cp0_cause;
480         enum emulation_result er = EMULATE_DONE;
481         int ret = RESUME_GUEST;
482 
483         if (!kvm_mips_guest_has_msa(&vcpu->arch) ||
484             (kvm_read_c0_guest_status(cop0) & (ST0_CU1 | ST0_FR)) == ST0_CU1) {
485                 /*
486                  * No MSA in guest, or FPU enabled and not in FR=1 mode,
487                  * guest reserved instruction exception
488                  */
489                 er = kvm_mips_emulate_ri_exc(cause, opc, run, vcpu);
490         } else if (!(kvm_read_c0_guest_config5(cop0) & MIPS_CONF5_MSAEN)) {
491                 /* MSA disabled by guest, guest MSA disabled exception */
492                 er = kvm_mips_emulate_msadis_exc(cause, opc, run, vcpu);
493         } else {
494                 /* Restore MSA/FPU state */
495                 kvm_own_msa(vcpu);
496                 er = EMULATE_DONE;
497         }
498 
499         switch (er) {
500         case EMULATE_DONE:
501                 ret = RESUME_GUEST;
502                 break;
503 
504         case EMULATE_FAIL:
505                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
506                 ret = RESUME_HOST;
507                 break;
508 
509         default:
510                 BUG();
511         }
512         return ret;
513 }
514 
515 static int kvm_trap_emul_hardware_enable(void)
516 {
517         return 0;
518 }
519 
520 static void kvm_trap_emul_hardware_disable(void)
521 {
522 }
523 
524 static int kvm_trap_emul_check_extension(struct kvm *kvm, long ext)
525 {
526         int r;
527 
528         switch (ext) {
529         case KVM_CAP_MIPS_TE:
530                 r = 1;
531                 break;
532         default:
533                 r = 0;
534                 break;
535         }
536 
537         return r;
538 }
539 
540 static int kvm_trap_emul_vcpu_init(struct kvm_vcpu *vcpu)
541 {
542         struct mm_struct *kern_mm = &vcpu->arch.guest_kernel_mm;
543         struct mm_struct *user_mm = &vcpu->arch.guest_user_mm;
544 
545         /*
546          * Allocate GVA -> HPA page tables.
547          * MIPS doesn't use the mm_struct pointer argument.
548          */
549         kern_mm->pgd = pgd_alloc(kern_mm);
550         if (!kern_mm->pgd)
551                 return -ENOMEM;
552 
553         user_mm->pgd = pgd_alloc(user_mm);
554         if (!user_mm->pgd) {
555                 pgd_free(kern_mm, kern_mm->pgd);
556                 return -ENOMEM;
557         }
558 
559         return 0;
560 }
561 
562 static void kvm_mips_emul_free_gva_pt(pgd_t *pgd)
563 {
564         /* Don't free host kernel page tables copied from init_mm.pgd */
565         const unsigned long end = 0x80000000;
566         unsigned long pgd_va, pud_va, pmd_va;
567         pud_t *pud;
568         pmd_t *pmd;
569         pte_t *pte;
570         int i, j, k;
571 
572         for (i = 0; i < USER_PTRS_PER_PGD; i++) {
573                 if (pgd_none(pgd[i]))
574                         continue;
575 
576                 pgd_va = (unsigned long)i << PGDIR_SHIFT;
577                 if (pgd_va >= end)
578                         break;
579                 pud = pud_offset(pgd + i, 0);
580                 for (j = 0; j < PTRS_PER_PUD; j++) {
581                         if (pud_none(pud[j]))
582                                 continue;
583 
584                         pud_va = pgd_va | ((unsigned long)j << PUD_SHIFT);
585                         if (pud_va >= end)
586                                 break;
587                         pmd = pmd_offset(pud + j, 0);
588                         for (k = 0; k < PTRS_PER_PMD; k++) {
589                                 if (pmd_none(pmd[k]))
590                                         continue;
591 
592                                 pmd_va = pud_va | (k << PMD_SHIFT);
593                                 if (pmd_va >= end)
594                                         break;
595                                 pte = pte_offset(pmd + k, 0);
596                                 pte_free_kernel(NULL, pte);
597                         }
598                         pmd_free(NULL, pmd);
599                 }
600                 pud_free(NULL, pud);
601         }
602         pgd_free(NULL, pgd);
603 }
604 
605 static void kvm_trap_emul_vcpu_uninit(struct kvm_vcpu *vcpu)
606 {
607         kvm_mips_emul_free_gva_pt(vcpu->arch.guest_kernel_mm.pgd);
608         kvm_mips_emul_free_gva_pt(vcpu->arch.guest_user_mm.pgd);
609 }
610 
611 static int kvm_trap_emul_vcpu_setup(struct kvm_vcpu *vcpu)
612 {
613         struct mips_coproc *cop0 = vcpu->arch.cop0;
614         u32 config, config1;
615         int vcpu_id = vcpu->vcpu_id;
616 
617         /* Start off the timer at 100 MHz */
618         kvm_mips_init_count(vcpu, 100*1000*1000);
619 
620         /*
621          * Arch specific stuff, set up config registers properly so that the
622          * guest will come up as expected
623          */
624 #ifndef CONFIG_CPU_MIPSR6
625         /* r2-r5, simulate a MIPS 24kc */
626         kvm_write_c0_guest_prid(cop0, 0x00019300);
627 #else
628         /* r6+, simulate a generic QEMU machine */
629         kvm_write_c0_guest_prid(cop0, 0x00010000);
630 #endif
631         /*
632          * Have config1, Cacheable, noncoherent, write-back, write allocate.
633          * Endianness, arch revision & virtually tagged icache should match
634          * host.
635          */
636         config = read_c0_config() & MIPS_CONF_AR;
637         config |= MIPS_CONF_M | CONF_CM_CACHABLE_NONCOHERENT | MIPS_CONF_MT_TLB;
638 #ifdef CONFIG_CPU_BIG_ENDIAN
639         config |= CONF_BE;
640 #endif
641         if (cpu_has_vtag_icache)
642                 config |= MIPS_CONF_VI;
643         kvm_write_c0_guest_config(cop0, config);
644 
645         /* Read the cache characteristics from the host Config1 Register */
646         config1 = (read_c0_config1() & ~0x7f);
647 
648         /* DCache line size not correctly reported in Config1 on Octeon CPUs */
649         if (cpu_dcache_line_size()) {
650                 config1 &= ~MIPS_CONF1_DL;
651                 config1 |= ((ilog2(cpu_dcache_line_size()) - 1) <<
652                             MIPS_CONF1_DL_SHF) & MIPS_CONF1_DL;
653         }
654 
655         /* Set up MMU size */
656         config1 &= ~(0x3f << 25);
657         config1 |= ((KVM_MIPS_GUEST_TLB_SIZE - 1) << 25);
658 
659         /* We unset some bits that we aren't emulating */
660         config1 &= ~(MIPS_CONF1_C2 | MIPS_CONF1_MD | MIPS_CONF1_PC |
661                      MIPS_CONF1_WR | MIPS_CONF1_CA);
662         kvm_write_c0_guest_config1(cop0, config1);
663 
664         /* Have config3, no tertiary/secondary caches implemented */
665         kvm_write_c0_guest_config2(cop0, MIPS_CONF_M);
666         /* MIPS_CONF_M | (read_c0_config2() & 0xfff) */
667 
668         /* Have config4, UserLocal */
669         kvm_write_c0_guest_config3(cop0, MIPS_CONF_M | MIPS_CONF3_ULRI);
670 
671         /* Have config5 */
672         kvm_write_c0_guest_config4(cop0, MIPS_CONF_M);
673 
674         /* No config6 */
675         kvm_write_c0_guest_config5(cop0, 0);
676 
677         /* Set Wait IE/IXMT Ignore in Config7, IAR, AR */
678         kvm_write_c0_guest_config7(cop0, (MIPS_CONF7_WII) | (1 << 10));
679 
680         /* Status */
681         kvm_write_c0_guest_status(cop0, ST0_BEV | ST0_ERL);
682 
683         /*
684          * Setup IntCtl defaults, compatibility mode for timer interrupts (HW5)
685          */
686         kvm_write_c0_guest_intctl(cop0, 0xFC000000);
687 
688         /* Put in vcpu id as CPUNum into Ebase Reg to handle SMP Guests */
689         kvm_write_c0_guest_ebase(cop0, KVM_GUEST_KSEG0 |
690                                        (vcpu_id & MIPS_EBASE_CPUNUM));
691 
692         /* Put PC at guest reset vector */
693         vcpu->arch.pc = KVM_GUEST_CKSEG1ADDR(0x1fc00000);
694 
695         return 0;
696 }
697 
698 static void kvm_trap_emul_flush_shadow_all(struct kvm *kvm)
699 {
700         /* Flush GVA page tables and invalidate GVA ASIDs on all VCPUs */
701         kvm_flush_remote_tlbs(kvm);
702 }
703 
704 static void kvm_trap_emul_flush_shadow_memslot(struct kvm *kvm,
705                                         const struct kvm_memory_slot *slot)
706 {
707         kvm_trap_emul_flush_shadow_all(kvm);
708 }
709 
710 static u64 kvm_trap_emul_get_one_regs[] = {
711         KVM_REG_MIPS_CP0_INDEX,
712         KVM_REG_MIPS_CP0_ENTRYLO0,
713         KVM_REG_MIPS_CP0_ENTRYLO1,
714         KVM_REG_MIPS_CP0_CONTEXT,
715         KVM_REG_MIPS_CP0_USERLOCAL,
716         KVM_REG_MIPS_CP0_PAGEMASK,
717         KVM_REG_MIPS_CP0_WIRED,
718         KVM_REG_MIPS_CP0_HWRENA,
719         KVM_REG_MIPS_CP0_BADVADDR,
720         KVM_REG_MIPS_CP0_COUNT,
721         KVM_REG_MIPS_CP0_ENTRYHI,
722         KVM_REG_MIPS_CP0_COMPARE,
723         KVM_REG_MIPS_CP0_STATUS,
724         KVM_REG_MIPS_CP0_INTCTL,
725         KVM_REG_MIPS_CP0_CAUSE,
726         KVM_REG_MIPS_CP0_EPC,
727         KVM_REG_MIPS_CP0_PRID,
728         KVM_REG_MIPS_CP0_EBASE,
729         KVM_REG_MIPS_CP0_CONFIG,
730         KVM_REG_MIPS_CP0_CONFIG1,
731         KVM_REG_MIPS_CP0_CONFIG2,
732         KVM_REG_MIPS_CP0_CONFIG3,
733         KVM_REG_MIPS_CP0_CONFIG4,
734         KVM_REG_MIPS_CP0_CONFIG5,
735         KVM_REG_MIPS_CP0_CONFIG7,
736         KVM_REG_MIPS_CP0_ERROREPC,
737         KVM_REG_MIPS_CP0_KSCRATCH1,
738         KVM_REG_MIPS_CP0_KSCRATCH2,
739         KVM_REG_MIPS_CP0_KSCRATCH3,
740         KVM_REG_MIPS_CP0_KSCRATCH4,
741         KVM_REG_MIPS_CP0_KSCRATCH5,
742         KVM_REG_MIPS_CP0_KSCRATCH6,
743 
744         KVM_REG_MIPS_COUNT_CTL,
745         KVM_REG_MIPS_COUNT_RESUME,
746         KVM_REG_MIPS_COUNT_HZ,
747 };
748 
749 static unsigned long kvm_trap_emul_num_regs(struct kvm_vcpu *vcpu)
750 {
751         return ARRAY_SIZE(kvm_trap_emul_get_one_regs);
752 }
753 
754 static int kvm_trap_emul_copy_reg_indices(struct kvm_vcpu *vcpu,
755                                           u64 __user *indices)
756 {
757         if (copy_to_user(indices, kvm_trap_emul_get_one_regs,
758                          sizeof(kvm_trap_emul_get_one_regs)))
759                 return -EFAULT;
760         indices += ARRAY_SIZE(kvm_trap_emul_get_one_regs);
761 
762         return 0;
763 }
764 
765 static int kvm_trap_emul_get_one_reg(struct kvm_vcpu *vcpu,
766                                      const struct kvm_one_reg *reg,
767                                      s64 *v)
768 {
769         struct mips_coproc *cop0 = vcpu->arch.cop0;
770 
771         switch (reg->id) {
772         case KVM_REG_MIPS_CP0_INDEX:
773                 *v = (long)kvm_read_c0_guest_index(cop0);
774                 break;
775         case KVM_REG_MIPS_CP0_ENTRYLO0:
776                 *v = kvm_read_c0_guest_entrylo0(cop0);
777                 break;
778         case KVM_REG_MIPS_CP0_ENTRYLO1:
779                 *v = kvm_read_c0_guest_entrylo1(cop0);
780                 break;
781         case KVM_REG_MIPS_CP0_CONTEXT:
782                 *v = (long)kvm_read_c0_guest_context(cop0);
783                 break;
784         case KVM_REG_MIPS_CP0_USERLOCAL:
785                 *v = (long)kvm_read_c0_guest_userlocal(cop0);
786                 break;
787         case KVM_REG_MIPS_CP0_PAGEMASK:
788                 *v = (long)kvm_read_c0_guest_pagemask(cop0);
789                 break;
790         case KVM_REG_MIPS_CP0_WIRED:
791                 *v = (long)kvm_read_c0_guest_wired(cop0);
792                 break;
793         case KVM_REG_MIPS_CP0_HWRENA:
794                 *v = (long)kvm_read_c0_guest_hwrena(cop0);
795                 break;
796         case KVM_REG_MIPS_CP0_BADVADDR:
797                 *v = (long)kvm_read_c0_guest_badvaddr(cop0);
798                 break;
799         case KVM_REG_MIPS_CP0_ENTRYHI:
800                 *v = (long)kvm_read_c0_guest_entryhi(cop0);
801                 break;
802         case KVM_REG_MIPS_CP0_COMPARE:
803                 *v = (long)kvm_read_c0_guest_compare(cop0);
804                 break;
805         case KVM_REG_MIPS_CP0_STATUS:
806                 *v = (long)kvm_read_c0_guest_status(cop0);
807                 break;
808         case KVM_REG_MIPS_CP0_INTCTL:
809                 *v = (long)kvm_read_c0_guest_intctl(cop0);
810                 break;
811         case KVM_REG_MIPS_CP0_CAUSE:
812                 *v = (long)kvm_read_c0_guest_cause(cop0);
813                 break;
814         case KVM_REG_MIPS_CP0_EPC:
815                 *v = (long)kvm_read_c0_guest_epc(cop0);
816                 break;
817         case KVM_REG_MIPS_CP0_PRID:
818                 *v = (long)kvm_read_c0_guest_prid(cop0);
819                 break;
820         case KVM_REG_MIPS_CP0_EBASE:
821                 *v = (long)kvm_read_c0_guest_ebase(cop0);
822                 break;
823         case KVM_REG_MIPS_CP0_CONFIG:
824                 *v = (long)kvm_read_c0_guest_config(cop0);
825                 break;
826         case KVM_REG_MIPS_CP0_CONFIG1:
827                 *v = (long)kvm_read_c0_guest_config1(cop0);
828                 break;
829         case KVM_REG_MIPS_CP0_CONFIG2:
830                 *v = (long)kvm_read_c0_guest_config2(cop0);
831                 break;
832         case KVM_REG_MIPS_CP0_CONFIG3:
833                 *v = (long)kvm_read_c0_guest_config3(cop0);
834                 break;
835         case KVM_REG_MIPS_CP0_CONFIG4:
836                 *v = (long)kvm_read_c0_guest_config4(cop0);
837                 break;
838         case KVM_REG_MIPS_CP0_CONFIG5:
839                 *v = (long)kvm_read_c0_guest_config5(cop0);
840                 break;
841         case KVM_REG_MIPS_CP0_CONFIG7:
842                 *v = (long)kvm_read_c0_guest_config7(cop0);
843                 break;
844         case KVM_REG_MIPS_CP0_COUNT:
845                 *v = kvm_mips_read_count(vcpu);
846                 break;
847         case KVM_REG_MIPS_COUNT_CTL:
848                 *v = vcpu->arch.count_ctl;
849                 break;
850         case KVM_REG_MIPS_COUNT_RESUME:
851                 *v = ktime_to_ns(vcpu->arch.count_resume);
852                 break;
853         case KVM_REG_MIPS_COUNT_HZ:
854                 *v = vcpu->arch.count_hz;
855                 break;
856         case KVM_REG_MIPS_CP0_ERROREPC:
857                 *v = (long)kvm_read_c0_guest_errorepc(cop0);
858                 break;
859         case KVM_REG_MIPS_CP0_KSCRATCH1:
860                 *v = (long)kvm_read_c0_guest_kscratch1(cop0);
861                 break;
862         case KVM_REG_MIPS_CP0_KSCRATCH2:
863                 *v = (long)kvm_read_c0_guest_kscratch2(cop0);
864                 break;
865         case KVM_REG_MIPS_CP0_KSCRATCH3:
866                 *v = (long)kvm_read_c0_guest_kscratch3(cop0);
867                 break;
868         case KVM_REG_MIPS_CP0_KSCRATCH4:
869                 *v = (long)kvm_read_c0_guest_kscratch4(cop0);
870                 break;
871         case KVM_REG_MIPS_CP0_KSCRATCH5:
872                 *v = (long)kvm_read_c0_guest_kscratch5(cop0);
873                 break;
874         case KVM_REG_MIPS_CP0_KSCRATCH6:
875                 *v = (long)kvm_read_c0_guest_kscratch6(cop0);
876                 break;
877         default:
878                 return -EINVAL;
879         }
880         return 0;
881 }
882 
883 static int kvm_trap_emul_set_one_reg(struct kvm_vcpu *vcpu,
884                                      const struct kvm_one_reg *reg,
885                                      s64 v)
886 {
887         struct mips_coproc *cop0 = vcpu->arch.cop0;
888         int ret = 0;
889         unsigned int cur, change;
890 
891         switch (reg->id) {
892         case KVM_REG_MIPS_CP0_INDEX:
893                 kvm_write_c0_guest_index(cop0, v);
894                 break;
895         case KVM_REG_MIPS_CP0_ENTRYLO0:
896                 kvm_write_c0_guest_entrylo0(cop0, v);
897                 break;
898         case KVM_REG_MIPS_CP0_ENTRYLO1:
899                 kvm_write_c0_guest_entrylo1(cop0, v);
900                 break;
901         case KVM_REG_MIPS_CP0_CONTEXT:
902                 kvm_write_c0_guest_context(cop0, v);
903                 break;
904         case KVM_REG_MIPS_CP0_USERLOCAL:
905                 kvm_write_c0_guest_userlocal(cop0, v);
906                 break;
907         case KVM_REG_MIPS_CP0_PAGEMASK:
908                 kvm_write_c0_guest_pagemask(cop0, v);
909                 break;
910         case KVM_REG_MIPS_CP0_WIRED:
911                 kvm_write_c0_guest_wired(cop0, v);
912                 break;
913         case KVM_REG_MIPS_CP0_HWRENA:
914                 kvm_write_c0_guest_hwrena(cop0, v);
915                 break;
916         case KVM_REG_MIPS_CP0_BADVADDR:
917                 kvm_write_c0_guest_badvaddr(cop0, v);
918                 break;
919         case KVM_REG_MIPS_CP0_ENTRYHI:
920                 kvm_write_c0_guest_entryhi(cop0, v);
921                 break;
922         case KVM_REG_MIPS_CP0_STATUS:
923                 kvm_write_c0_guest_status(cop0, v);
924                 break;
925         case KVM_REG_MIPS_CP0_INTCTL:
926                 /* No VInt, so no VS, read-only for now */
927                 break;
928         case KVM_REG_MIPS_CP0_EPC:
929                 kvm_write_c0_guest_epc(cop0, v);
930                 break;
931         case KVM_REG_MIPS_CP0_PRID:
932                 kvm_write_c0_guest_prid(cop0, v);
933                 break;
934         case KVM_REG_MIPS_CP0_EBASE:
935                 /*
936                  * Allow core number to be written, but the exception base must
937                  * remain in guest KSeg0.
938                  */
939                 kvm_change_c0_guest_ebase(cop0, 0x1ffff000 | MIPS_EBASE_CPUNUM,
940                                           v);
941                 break;
942         case KVM_REG_MIPS_CP0_COUNT:
943                 kvm_mips_write_count(vcpu, v);
944                 break;
945         case KVM_REG_MIPS_CP0_COMPARE:
946                 kvm_mips_write_compare(vcpu, v, false);
947                 break;
948         case KVM_REG_MIPS_CP0_CAUSE:
949                 /*
950                  * If the timer is stopped or started (DC bit) it must look
951                  * atomic with changes to the interrupt pending bits (TI, IRQ5).
952                  * A timer interrupt should not happen in between.
953                  */
954                 if ((kvm_read_c0_guest_cause(cop0) ^ v) & CAUSEF_DC) {
955                         if (v & CAUSEF_DC) {
956                                 /* disable timer first */
957                                 kvm_mips_count_disable_cause(vcpu);
958                                 kvm_change_c0_guest_cause(cop0, (u32)~CAUSEF_DC,
959                                                           v);
960                         } else {
961                                 /* enable timer last */
962                                 kvm_change_c0_guest_cause(cop0, (u32)~CAUSEF_DC,
963                                                           v);
964                                 kvm_mips_count_enable_cause(vcpu);
965                         }
966                 } else {
967                         kvm_write_c0_guest_cause(cop0, v);
968                 }
969                 break;
970         case KVM_REG_MIPS_CP0_CONFIG:
971                 /* read-only for now */
972                 break;
973         case KVM_REG_MIPS_CP0_CONFIG1:
974                 cur = kvm_read_c0_guest_config1(cop0);
975                 change = (cur ^ v) & kvm_mips_config1_wrmask(vcpu);
976                 if (change) {
977                         v = cur ^ change;
978                         kvm_write_c0_guest_config1(cop0, v);
979                 }
980                 break;
981         case KVM_REG_MIPS_CP0_CONFIG2:
982                 /* read-only for now */
983                 break;
984         case KVM_REG_MIPS_CP0_CONFIG3:
985                 cur = kvm_read_c0_guest_config3(cop0);
986                 change = (cur ^ v) & kvm_mips_config3_wrmask(vcpu);
987                 if (change) {
988                         v = cur ^ change;
989                         kvm_write_c0_guest_config3(cop0, v);
990                 }
991                 break;
992         case KVM_REG_MIPS_CP0_CONFIG4:
993                 cur = kvm_read_c0_guest_config4(cop0);
994                 change = (cur ^ v) & kvm_mips_config4_wrmask(vcpu);
995                 if (change) {
996                         v = cur ^ change;
997                         kvm_write_c0_guest_config4(cop0, v);
998                 }
999                 break;
1000         case KVM_REG_MIPS_CP0_CONFIG5:
1001                 cur = kvm_read_c0_guest_config5(cop0);
1002                 change = (cur ^ v) & kvm_mips_config5_wrmask(vcpu);
1003                 if (change) {
1004                         v = cur ^ change;
1005                         kvm_write_c0_guest_config5(cop0, v);
1006                 }
1007                 break;
1008         case KVM_REG_MIPS_CP0_CONFIG7:
1009                 /* writes ignored */
1010                 break;
1011         case KVM_REG_MIPS_COUNT_CTL:
1012                 ret = kvm_mips_set_count_ctl(vcpu, v);
1013                 break;
1014         case KVM_REG_MIPS_COUNT_RESUME:
1015                 ret = kvm_mips_set_count_resume(vcpu, v);
1016                 break;
1017         case KVM_REG_MIPS_COUNT_HZ:
1018                 ret = kvm_mips_set_count_hz(vcpu, v);
1019                 break;
1020         case KVM_REG_MIPS_CP0_ERROREPC:
1021                 kvm_write_c0_guest_errorepc(cop0, v);
1022                 break;
1023         case KVM_REG_MIPS_CP0_KSCRATCH1:
1024                 kvm_write_c0_guest_kscratch1(cop0, v);
1025                 break;
1026         case KVM_REG_MIPS_CP0_KSCRATCH2:
1027                 kvm_write_c0_guest_kscratch2(cop0, v);
1028                 break;
1029         case KVM_REG_MIPS_CP0_KSCRATCH3:
1030                 kvm_write_c0_guest_kscratch3(cop0, v);
1031                 break;
1032         case KVM_REG_MIPS_CP0_KSCRATCH4:
1033                 kvm_write_c0_guest_kscratch4(cop0, v);
1034                 break;
1035         case KVM_REG_MIPS_CP0_KSCRATCH5:
1036                 kvm_write_c0_guest_kscratch5(cop0, v);
1037                 break;
1038         case KVM_REG_MIPS_CP0_KSCRATCH6:
1039                 kvm_write_c0_guest_kscratch6(cop0, v);
1040                 break;
1041         default:
1042                 return -EINVAL;
1043         }
1044         return ret;
1045 }
1046 
1047 static int kvm_trap_emul_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1048 {
1049         struct mm_struct *kern_mm = &vcpu->arch.guest_kernel_mm;
1050         struct mm_struct *user_mm = &vcpu->arch.guest_user_mm;
1051         struct mm_struct *mm;
1052 
1053         /*
1054          * Were we in guest context? If so, restore the appropriate ASID based
1055          * on the mode of the Guest (Kernel/User).
1056          */
1057         if (current->flags & PF_VCPU) {
1058                 mm = KVM_GUEST_KERNEL_MODE(vcpu) ? kern_mm : user_mm;
1059                 if ((cpu_context(cpu, mm) ^ asid_cache(cpu)) &
1060                     asid_version_mask(cpu))
1061                         get_new_mmu_context(mm, cpu);
1062                 write_c0_entryhi(cpu_asid(cpu, mm));
1063                 TLBMISS_HANDLER_SETUP_PGD(mm->pgd);
1064                 kvm_mips_suspend_mm(cpu);
1065                 ehb();
1066         }
1067 
1068         return 0;
1069 }
1070 
1071 static int kvm_trap_emul_vcpu_put(struct kvm_vcpu *vcpu, int cpu)
1072 {
1073         kvm_lose_fpu(vcpu);
1074 
1075         if (current->flags & PF_VCPU) {
1076                 /* Restore normal Linux process memory map */
1077                 if (((cpu_context(cpu, current->mm) ^ asid_cache(cpu)) &
1078                      asid_version_mask(cpu)))
1079                         get_new_mmu_context(current->mm, cpu);
1080                 write_c0_entryhi(cpu_asid(cpu, current->mm));
1081                 TLBMISS_HANDLER_SETUP_PGD(current->mm->pgd);
1082                 kvm_mips_resume_mm(cpu);
1083                 ehb();
1084         }
1085 
1086         return 0;
1087 }
1088 
1089 static void kvm_trap_emul_check_requests(struct kvm_vcpu *vcpu, int cpu,
1090                                          bool reload_asid)
1091 {
1092         struct mm_struct *kern_mm = &vcpu->arch.guest_kernel_mm;
1093         struct mm_struct *user_mm = &vcpu->arch.guest_user_mm;
1094         struct mm_struct *mm;
1095         int i;
1096 
1097         if (likely(!kvm_request_pending(vcpu)))
1098                 return;
1099 
1100         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1101                 /*
1102                  * Both kernel & user GVA mappings must be invalidated. The
1103                  * caller is just about to check whether the ASID is stale
1104                  * anyway so no need to reload it here.
1105                  */
1106                 kvm_mips_flush_gva_pt(kern_mm->pgd, KMF_GPA | KMF_KERN);
1107                 kvm_mips_flush_gva_pt(user_mm->pgd, KMF_GPA | KMF_USER);
1108                 for_each_possible_cpu(i) {
1109                         cpu_context(i, kern_mm) = 0;
1110                         cpu_context(i, user_mm) = 0;
1111                 }
1112 
1113                 /* Generate new ASID for current mode */
1114                 if (reload_asid) {
1115                         mm = KVM_GUEST_KERNEL_MODE(vcpu) ? kern_mm : user_mm;
1116                         get_new_mmu_context(mm, cpu);
1117                         htw_stop();
1118                         write_c0_entryhi(cpu_asid(cpu, mm));
1119                         TLBMISS_HANDLER_SETUP_PGD(mm->pgd);
1120                         htw_start();
1121                 }
1122         }
1123 }
1124 
1125 /**
1126  * kvm_trap_emul_gva_lockless_begin() - Begin lockless access to GVA space.
1127  * @vcpu:       VCPU pointer.
1128  *
1129  * Call before a GVA space access outside of guest mode, to ensure that
1130  * asynchronous TLB flush requests are handled or delayed until completion of
1131  * the GVA access (as indicated by a matching kvm_trap_emul_gva_lockless_end()).
1132  *
1133  * Should be called with IRQs already enabled.
1134  */
1135 void kvm_trap_emul_gva_lockless_begin(struct kvm_vcpu *vcpu)
1136 {
1137         /* We re-enable IRQs in kvm_trap_emul_gva_lockless_end() */
1138         WARN_ON_ONCE(irqs_disabled());
1139 
1140         /*
1141          * The caller is about to access the GVA space, so we set the mode to
1142          * force TLB flush requests to send an IPI, and also disable IRQs to
1143          * delay IPI handling until kvm_trap_emul_gva_lockless_end().
1144          */
1145         local_irq_disable();
1146 
1147         /*
1148          * Make sure the read of VCPU requests is not reordered ahead of the
1149          * write to vcpu->mode, or we could miss a TLB flush request while
1150          * the requester sees the VCPU as outside of guest mode and not needing
1151          * an IPI.
1152          */
1153         smp_store_mb(vcpu->mode, READING_SHADOW_PAGE_TABLES);
1154 
1155         /*
1156          * If a TLB flush has been requested (potentially while
1157          * OUTSIDE_GUEST_MODE and assumed immediately effective), perform it
1158          * before accessing the GVA space, and be sure to reload the ASID if
1159          * necessary as it'll be immediately used.
1160          *
1161          * TLB flush requests after this check will trigger an IPI due to the
1162          * mode change above, which will be delayed due to IRQs disabled.
1163          */
1164         kvm_trap_emul_check_requests(vcpu, smp_processor_id(), true);
1165 }
1166 
1167 /**
1168  * kvm_trap_emul_gva_lockless_end() - End lockless access to GVA space.
1169  * @vcpu:       VCPU pointer.
1170  *
1171  * Called after a GVA space access outside of guest mode. Should have a matching
1172  * call to kvm_trap_emul_gva_lockless_begin().
1173  */
1174 void kvm_trap_emul_gva_lockless_end(struct kvm_vcpu *vcpu)
1175 {
1176         /*
1177          * Make sure the write to vcpu->mode is not reordered in front of GVA
1178          * accesses, or a TLB flush requester may not think it necessary to send
1179          * an IPI.
1180          */
1181         smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE);
1182 
1183         /*
1184          * Now that the access to GVA space is complete, its safe for pending
1185          * TLB flush request IPIs to be handled (which indicates completion).
1186          */
1187         local_irq_enable();
1188 }
1189 
1190 static void kvm_trap_emul_vcpu_reenter(struct kvm_run *run,
1191                                        struct kvm_vcpu *vcpu)
1192 {
1193         struct mm_struct *kern_mm = &vcpu->arch.guest_kernel_mm;
1194         struct mm_struct *user_mm = &vcpu->arch.guest_user_mm;
1195         struct mm_struct *mm;
1196         struct mips_coproc *cop0 = vcpu->arch.cop0;
1197         int i, cpu = smp_processor_id();
1198         unsigned int gasid;
1199 
1200         /*
1201          * No need to reload ASID, IRQs are disabled already so there's no rush,
1202          * and we'll check if we need to regenerate below anyway before
1203          * re-entering the guest.
1204          */
1205         kvm_trap_emul_check_requests(vcpu, cpu, false);
1206 
1207         if (KVM_GUEST_KERNEL_MODE(vcpu)) {
1208                 mm = kern_mm;
1209         } else {
1210                 mm = user_mm;
1211 
1212                 /*
1213                  * Lazy host ASID regeneration / PT flush for guest user mode.
1214                  * If the guest ASID has changed since the last guest usermode
1215                  * execution, invalidate the stale TLB entries and flush GVA PT
1216                  * entries too.
1217                  */
1218                 gasid = kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID;
1219                 if (gasid != vcpu->arch.last_user_gasid) {
1220                         kvm_mips_flush_gva_pt(user_mm->pgd, KMF_USER);
1221                         for_each_possible_cpu(i)
1222                                 cpu_context(i, user_mm) = 0;
1223                         vcpu->arch.last_user_gasid = gasid;
1224                 }
1225         }
1226 
1227         /*
1228          * Check if ASID is stale. This may happen due to a TLB flush request or
1229          * a lazy user MM invalidation.
1230          */
1231         if ((cpu_context(cpu, mm) ^ asid_cache(cpu)) &
1232             asid_version_mask(cpu))
1233                 get_new_mmu_context(mm, cpu);
1234 }
1235 
1236 static int kvm_trap_emul_vcpu_run(struct kvm_run *run, struct kvm_vcpu *vcpu)
1237 {
1238         int cpu = smp_processor_id();
1239         int r;
1240 
1241         /* Check if we have any exceptions/interrupts pending */
1242         kvm_mips_deliver_interrupts(vcpu,
1243                                     kvm_read_c0_guest_cause(vcpu->arch.cop0));
1244 
1245         kvm_trap_emul_vcpu_reenter(run, vcpu);
1246 
1247         /*
1248          * We use user accessors to access guest memory, but we don't want to
1249          * invoke Linux page faulting.
1250          */
1251         pagefault_disable();
1252 
1253         /* Disable hardware page table walking while in guest */
1254         htw_stop();
1255 
1256         /*
1257          * While in guest context we're in the guest's address space, not the
1258          * host process address space, so we need to be careful not to confuse
1259          * e.g. cache management IPIs.
1260          */
1261         kvm_mips_suspend_mm(cpu);
1262 
1263         r = vcpu->arch.vcpu_run(run, vcpu);
1264 
1265         /* We may have migrated while handling guest exits */
1266         cpu = smp_processor_id();
1267 
1268         /* Restore normal Linux process memory map */
1269         if (((cpu_context(cpu, current->mm) ^ asid_cache(cpu)) &
1270              asid_version_mask(cpu)))
1271                 get_new_mmu_context(current->mm, cpu);
1272         write_c0_entryhi(cpu_asid(cpu, current->mm));
1273         TLBMISS_HANDLER_SETUP_PGD(current->mm->pgd);
1274         kvm_mips_resume_mm(cpu);
1275 
1276         htw_start();
1277 
1278         pagefault_enable();
1279 
1280         return r;
1281 }
1282 
1283 static struct kvm_mips_callbacks kvm_trap_emul_callbacks = {
1284         /* exit handlers */
1285         .handle_cop_unusable = kvm_trap_emul_handle_cop_unusable,
1286         .handle_tlb_mod = kvm_trap_emul_handle_tlb_mod,
1287         .handle_tlb_st_miss = kvm_trap_emul_handle_tlb_st_miss,
1288         .handle_tlb_ld_miss = kvm_trap_emul_handle_tlb_ld_miss,
1289         .handle_addr_err_st = kvm_trap_emul_handle_addr_err_st,
1290         .handle_addr_err_ld = kvm_trap_emul_handle_addr_err_ld,
1291         .handle_syscall = kvm_trap_emul_handle_syscall,
1292         .handle_res_inst = kvm_trap_emul_handle_res_inst,
1293         .handle_break = kvm_trap_emul_handle_break,
1294         .handle_trap = kvm_trap_emul_handle_trap,
1295         .handle_msa_fpe = kvm_trap_emul_handle_msa_fpe,
1296         .handle_fpe = kvm_trap_emul_handle_fpe,
1297         .handle_msa_disabled = kvm_trap_emul_handle_msa_disabled,
1298         .handle_guest_exit = kvm_trap_emul_no_handler,
1299 
1300         .hardware_enable = kvm_trap_emul_hardware_enable,
1301         .hardware_disable = kvm_trap_emul_hardware_disable,
1302         .check_extension = kvm_trap_emul_check_extension,
1303         .vcpu_init = kvm_trap_emul_vcpu_init,
1304         .vcpu_uninit = kvm_trap_emul_vcpu_uninit,
1305         .vcpu_setup = kvm_trap_emul_vcpu_setup,
1306         .flush_shadow_all = kvm_trap_emul_flush_shadow_all,
1307         .flush_shadow_memslot = kvm_trap_emul_flush_shadow_memslot,
1308         .gva_to_gpa = kvm_trap_emul_gva_to_gpa_cb,
1309         .queue_timer_int = kvm_mips_queue_timer_int_cb,
1310         .dequeue_timer_int = kvm_mips_dequeue_timer_int_cb,
1311         .queue_io_int = kvm_mips_queue_io_int_cb,
1312         .dequeue_io_int = kvm_mips_dequeue_io_int_cb,
1313         .irq_deliver = kvm_mips_irq_deliver_cb,
1314         .irq_clear = kvm_mips_irq_clear_cb,
1315         .num_regs = kvm_trap_emul_num_regs,
1316         .copy_reg_indices = kvm_trap_emul_copy_reg_indices,
1317         .get_one_reg = kvm_trap_emul_get_one_reg,
1318         .set_one_reg = kvm_trap_emul_set_one_reg,
1319         .vcpu_load = kvm_trap_emul_vcpu_load,
1320         .vcpu_put = kvm_trap_emul_vcpu_put,
1321         .vcpu_run = kvm_trap_emul_vcpu_run,
1322         .vcpu_reenter = kvm_trap_emul_vcpu_reenter,
1323 };
1324 
1325 int kvm_mips_emulation_init(struct kvm_mips_callbacks **install_callbacks)
1326 {
1327         *install_callbacks = &kvm_trap_emul_callbacks;
1328         return 0;
1329 }
1330 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp