~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/mips/kvm/emulate.c

Version: ~ [ linux-5.4-rc3 ] ~ [ linux-5.3.6 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.79 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.149 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.196 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.196 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.75 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * This file is subject to the terms and conditions of the GNU General Public
  3  * License.  See the file "COPYING" in the main directory of this archive
  4  * for more details.
  5  *
  6  * KVM/MIPS: Instruction/Exception emulation
  7  *
  8  * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
  9  * Authors: Sanjay Lal <sanjayl@kymasys.com>
 10  */
 11 
 12 #include <linux/errno.h>
 13 #include <linux/err.h>
 14 #include <linux/ktime.h>
 15 #include <linux/kvm_host.h>
 16 #include <linux/module.h>
 17 #include <linux/vmalloc.h>
 18 #include <linux/fs.h>
 19 #include <linux/bootmem.h>
 20 #include <linux/random.h>
 21 #include <asm/page.h>
 22 #include <asm/cacheflush.h>
 23 #include <asm/cacheops.h>
 24 #include <asm/cpu-info.h>
 25 #include <asm/mmu_context.h>
 26 #include <asm/tlbflush.h>
 27 #include <asm/inst.h>
 28 
 29 #undef CONFIG_MIPS_MT
 30 #include <asm/r4kcache.h>
 31 #define CONFIG_MIPS_MT
 32 
 33 #include "interrupt.h"
 34 #include "commpage.h"
 35 
 36 #include "trace.h"
 37 
 38 /*
 39  * Compute the return address and do emulate branch simulation, if required.
 40  * This function should be called only in branch delay slot active.
 41  */
 42 unsigned long kvm_compute_return_epc(struct kvm_vcpu *vcpu,
 43         unsigned long instpc)
 44 {
 45         unsigned int dspcontrol;
 46         union mips_instruction insn;
 47         struct kvm_vcpu_arch *arch = &vcpu->arch;
 48         long epc = instpc;
 49         long nextpc = KVM_INVALID_INST;
 50 
 51         if (epc & 3)
 52                 goto unaligned;
 53 
 54         /* Read the instruction */
 55         insn.word = kvm_get_inst((u32 *) epc, vcpu);
 56 
 57         if (insn.word == KVM_INVALID_INST)
 58                 return KVM_INVALID_INST;
 59 
 60         switch (insn.i_format.opcode) {
 61                 /* jr and jalr are in r_format format. */
 62         case spec_op:
 63                 switch (insn.r_format.func) {
 64                 case jalr_op:
 65                         arch->gprs[insn.r_format.rd] = epc + 8;
 66                         /* Fall through */
 67                 case jr_op:
 68                         nextpc = arch->gprs[insn.r_format.rs];
 69                         break;
 70                 }
 71                 break;
 72 
 73                 /*
 74                  * This group contains:
 75                  * bltz_op, bgez_op, bltzl_op, bgezl_op,
 76                  * bltzal_op, bgezal_op, bltzall_op, bgezall_op.
 77                  */
 78         case bcond_op:
 79                 switch (insn.i_format.rt) {
 80                 case bltz_op:
 81                 case bltzl_op:
 82                         if ((long)arch->gprs[insn.i_format.rs] < 0)
 83                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
 84                         else
 85                                 epc += 8;
 86                         nextpc = epc;
 87                         break;
 88 
 89                 case bgez_op:
 90                 case bgezl_op:
 91                         if ((long)arch->gprs[insn.i_format.rs] >= 0)
 92                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
 93                         else
 94                                 epc += 8;
 95                         nextpc = epc;
 96                         break;
 97 
 98                 case bltzal_op:
 99                 case bltzall_op:
100                         arch->gprs[31] = epc + 8;
101                         if ((long)arch->gprs[insn.i_format.rs] < 0)
102                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
103                         else
104                                 epc += 8;
105                         nextpc = epc;
106                         break;
107 
108                 case bgezal_op:
109                 case bgezall_op:
110                         arch->gprs[31] = epc + 8;
111                         if ((long)arch->gprs[insn.i_format.rs] >= 0)
112                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
113                         else
114                                 epc += 8;
115                         nextpc = epc;
116                         break;
117                 case bposge32_op:
118                         if (!cpu_has_dsp)
119                                 goto sigill;
120 
121                         dspcontrol = rddsp(0x01);
122 
123                         if (dspcontrol >= 32)
124                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
125                         else
126                                 epc += 8;
127                         nextpc = epc;
128                         break;
129                 }
130                 break;
131 
132                 /* These are unconditional and in j_format. */
133         case jal_op:
134                 arch->gprs[31] = instpc + 8;
135         case j_op:
136                 epc += 4;
137                 epc >>= 28;
138                 epc <<= 28;
139                 epc |= (insn.j_format.target << 2);
140                 nextpc = epc;
141                 break;
142 
143                 /* These are conditional and in i_format. */
144         case beq_op:
145         case beql_op:
146                 if (arch->gprs[insn.i_format.rs] ==
147                     arch->gprs[insn.i_format.rt])
148                         epc = epc + 4 + (insn.i_format.simmediate << 2);
149                 else
150                         epc += 8;
151                 nextpc = epc;
152                 break;
153 
154         case bne_op:
155         case bnel_op:
156                 if (arch->gprs[insn.i_format.rs] !=
157                     arch->gprs[insn.i_format.rt])
158                         epc = epc + 4 + (insn.i_format.simmediate << 2);
159                 else
160                         epc += 8;
161                 nextpc = epc;
162                 break;
163 
164         case blez_op:   /* POP06 */
165 #ifndef CONFIG_CPU_MIPSR6
166         case blezl_op:  /* removed in R6 */
167 #endif
168                 if (insn.i_format.rt != 0)
169                         goto compact_branch;
170                 if ((long)arch->gprs[insn.i_format.rs] <= 0)
171                         epc = epc + 4 + (insn.i_format.simmediate << 2);
172                 else
173                         epc += 8;
174                 nextpc = epc;
175                 break;
176 
177         case bgtz_op:   /* POP07 */
178 #ifndef CONFIG_CPU_MIPSR6
179         case bgtzl_op:  /* removed in R6 */
180 #endif
181                 if (insn.i_format.rt != 0)
182                         goto compact_branch;
183                 if ((long)arch->gprs[insn.i_format.rs] > 0)
184                         epc = epc + 4 + (insn.i_format.simmediate << 2);
185                 else
186                         epc += 8;
187                 nextpc = epc;
188                 break;
189 
190                 /* And now the FPA/cp1 branch instructions. */
191         case cop1_op:
192                 kvm_err("%s: unsupported cop1_op\n", __func__);
193                 break;
194 
195 #ifdef CONFIG_CPU_MIPSR6
196         /* R6 added the following compact branches with forbidden slots */
197         case blezl_op:  /* POP26 */
198         case bgtzl_op:  /* POP27 */
199                 /* only rt == 0 isn't compact branch */
200                 if (insn.i_format.rt != 0)
201                         goto compact_branch;
202                 break;
203         case pop10_op:
204         case pop30_op:
205                 /* only rs == rt == 0 is reserved, rest are compact branches */
206                 if (insn.i_format.rs != 0 || insn.i_format.rt != 0)
207                         goto compact_branch;
208                 break;
209         case pop66_op:
210         case pop76_op:
211                 /* only rs == 0 isn't compact branch */
212                 if (insn.i_format.rs != 0)
213                         goto compact_branch;
214                 break;
215 compact_branch:
216                 /*
217                  * If we've hit an exception on the forbidden slot, then
218                  * the branch must not have been taken.
219                  */
220                 epc += 8;
221                 nextpc = epc;
222                 break;
223 #else
224 compact_branch:
225                 /* Compact branches not supported before R6 */
226                 break;
227 #endif
228         }
229 
230         return nextpc;
231 
232 unaligned:
233         kvm_err("%s: unaligned epc\n", __func__);
234         return nextpc;
235 
236 sigill:
237         kvm_err("%s: DSP branch but not DSP ASE\n", __func__);
238         return nextpc;
239 }
240 
241 enum emulation_result update_pc(struct kvm_vcpu *vcpu, u32 cause)
242 {
243         unsigned long branch_pc;
244         enum emulation_result er = EMULATE_DONE;
245 
246         if (cause & CAUSEF_BD) {
247                 branch_pc = kvm_compute_return_epc(vcpu, vcpu->arch.pc);
248                 if (branch_pc == KVM_INVALID_INST) {
249                         er = EMULATE_FAIL;
250                 } else {
251                         vcpu->arch.pc = branch_pc;
252                         kvm_debug("BD update_pc(): New PC: %#lx\n",
253                                   vcpu->arch.pc);
254                 }
255         } else
256                 vcpu->arch.pc += 4;
257 
258         kvm_debug("update_pc(): New PC: %#lx\n", vcpu->arch.pc);
259 
260         return er;
261 }
262 
263 /**
264  * kvm_mips_count_disabled() - Find whether the CP0_Count timer is disabled.
265  * @vcpu:       Virtual CPU.
266  *
267  * Returns:     1 if the CP0_Count timer is disabled by either the guest
268  *              CP0_Cause.DC bit or the count_ctl.DC bit.
269  *              0 otherwise (in which case CP0_Count timer is running).
270  */
271 static inline int kvm_mips_count_disabled(struct kvm_vcpu *vcpu)
272 {
273         struct mips_coproc *cop0 = vcpu->arch.cop0;
274 
275         return  (vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC) ||
276                 (kvm_read_c0_guest_cause(cop0) & CAUSEF_DC);
277 }
278 
279 /**
280  * kvm_mips_ktime_to_count() - Scale ktime_t to a 32-bit count.
281  *
282  * Caches the dynamic nanosecond bias in vcpu->arch.count_dyn_bias.
283  *
284  * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running).
285  */
286 static u32 kvm_mips_ktime_to_count(struct kvm_vcpu *vcpu, ktime_t now)
287 {
288         s64 now_ns, periods;
289         u64 delta;
290 
291         now_ns = ktime_to_ns(now);
292         delta = now_ns + vcpu->arch.count_dyn_bias;
293 
294         if (delta >= vcpu->arch.count_period) {
295                 /* If delta is out of safe range the bias needs adjusting */
296                 periods = div64_s64(now_ns, vcpu->arch.count_period);
297                 vcpu->arch.count_dyn_bias = -periods * vcpu->arch.count_period;
298                 /* Recalculate delta with new bias */
299                 delta = now_ns + vcpu->arch.count_dyn_bias;
300         }
301 
302         /*
303          * We've ensured that:
304          *   delta < count_period
305          *
306          * Therefore the intermediate delta*count_hz will never overflow since
307          * at the boundary condition:
308          *   delta = count_period
309          *   delta = NSEC_PER_SEC * 2^32 / count_hz
310          *   delta * count_hz = NSEC_PER_SEC * 2^32
311          */
312         return div_u64(delta * vcpu->arch.count_hz, NSEC_PER_SEC);
313 }
314 
315 /**
316  * kvm_mips_count_time() - Get effective current time.
317  * @vcpu:       Virtual CPU.
318  *
319  * Get effective monotonic ktime. This is usually a straightforward ktime_get(),
320  * except when the master disable bit is set in count_ctl, in which case it is
321  * count_resume, i.e. the time that the count was disabled.
322  *
323  * Returns:     Effective monotonic ktime for CP0_Count.
324  */
325 static inline ktime_t kvm_mips_count_time(struct kvm_vcpu *vcpu)
326 {
327         if (unlikely(vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC))
328                 return vcpu->arch.count_resume;
329 
330         return ktime_get();
331 }
332 
333 /**
334  * kvm_mips_read_count_running() - Read the current count value as if running.
335  * @vcpu:       Virtual CPU.
336  * @now:        Kernel time to read CP0_Count at.
337  *
338  * Returns the current guest CP0_Count register at time @now and handles if the
339  * timer interrupt is pending and hasn't been handled yet.
340  *
341  * Returns:     The current value of the guest CP0_Count register.
342  */
343 static u32 kvm_mips_read_count_running(struct kvm_vcpu *vcpu, ktime_t now)
344 {
345         struct mips_coproc *cop0 = vcpu->arch.cop0;
346         ktime_t expires, threshold;
347         u32 count, compare;
348         int running;
349 
350         /* Calculate the biased and scaled guest CP0_Count */
351         count = vcpu->arch.count_bias + kvm_mips_ktime_to_count(vcpu, now);
352         compare = kvm_read_c0_guest_compare(cop0);
353 
354         /*
355          * Find whether CP0_Count has reached the closest timer interrupt. If
356          * not, we shouldn't inject it.
357          */
358         if ((s32)(count - compare) < 0)
359                 return count;
360 
361         /*
362          * The CP0_Count we're going to return has already reached the closest
363          * timer interrupt. Quickly check if it really is a new interrupt by
364          * looking at whether the interval until the hrtimer expiry time is
365          * less than 1/4 of the timer period.
366          */
367         expires = hrtimer_get_expires(&vcpu->arch.comparecount_timer);
368         threshold = ktime_add_ns(now, vcpu->arch.count_period / 4);
369         if (ktime_before(expires, threshold)) {
370                 /*
371                  * Cancel it while we handle it so there's no chance of
372                  * interference with the timeout handler.
373                  */
374                 running = hrtimer_cancel(&vcpu->arch.comparecount_timer);
375 
376                 /* Nothing should be waiting on the timeout */
377                 kvm_mips_callbacks->queue_timer_int(vcpu);
378 
379                 /*
380                  * Restart the timer if it was running based on the expiry time
381                  * we read, so that we don't push it back 2 periods.
382                  */
383                 if (running) {
384                         expires = ktime_add_ns(expires,
385                                                vcpu->arch.count_period);
386                         hrtimer_start(&vcpu->arch.comparecount_timer, expires,
387                                       HRTIMER_MODE_ABS);
388                 }
389         }
390 
391         return count;
392 }
393 
394 /**
395  * kvm_mips_read_count() - Read the current count value.
396  * @vcpu:       Virtual CPU.
397  *
398  * Read the current guest CP0_Count value, taking into account whether the timer
399  * is stopped.
400  *
401  * Returns:     The current guest CP0_Count value.
402  */
403 u32 kvm_mips_read_count(struct kvm_vcpu *vcpu)
404 {
405         struct mips_coproc *cop0 = vcpu->arch.cop0;
406 
407         /* If count disabled just read static copy of count */
408         if (kvm_mips_count_disabled(vcpu))
409                 return kvm_read_c0_guest_count(cop0);
410 
411         return kvm_mips_read_count_running(vcpu, ktime_get());
412 }
413 
414 /**
415  * kvm_mips_freeze_hrtimer() - Safely stop the hrtimer.
416  * @vcpu:       Virtual CPU.
417  * @count:      Output pointer for CP0_Count value at point of freeze.
418  *
419  * Freeze the hrtimer safely and return both the ktime and the CP0_Count value
420  * at the point it was frozen. It is guaranteed that any pending interrupts at
421  * the point it was frozen are handled, and none after that point.
422  *
423  * This is useful where the time/CP0_Count is needed in the calculation of the
424  * new parameters.
425  *
426  * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running).
427  *
428  * Returns:     The ktime at the point of freeze.
429  */
430 static ktime_t kvm_mips_freeze_hrtimer(struct kvm_vcpu *vcpu, u32 *count)
431 {
432         ktime_t now;
433 
434         /* stop hrtimer before finding time */
435         hrtimer_cancel(&vcpu->arch.comparecount_timer);
436         now = ktime_get();
437 
438         /* find count at this point and handle pending hrtimer */
439         *count = kvm_mips_read_count_running(vcpu, now);
440 
441         return now;
442 }
443 
444 /**
445  * kvm_mips_resume_hrtimer() - Resume hrtimer, updating expiry.
446  * @vcpu:       Virtual CPU.
447  * @now:        ktime at point of resume.
448  * @count:      CP0_Count at point of resume.
449  *
450  * Resumes the timer and updates the timer expiry based on @now and @count.
451  * This can be used in conjunction with kvm_mips_freeze_timer() when timer
452  * parameters need to be changed.
453  *
454  * It is guaranteed that a timer interrupt immediately after resume will be
455  * handled, but not if CP_Compare is exactly at @count. That case is already
456  * handled by kvm_mips_freeze_timer().
457  *
458  * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running).
459  */
460 static void kvm_mips_resume_hrtimer(struct kvm_vcpu *vcpu,
461                                     ktime_t now, u32 count)
462 {
463         struct mips_coproc *cop0 = vcpu->arch.cop0;
464         u32 compare;
465         u64 delta;
466         ktime_t expire;
467 
468         /* Calculate timeout (wrap 0 to 2^32) */
469         compare = kvm_read_c0_guest_compare(cop0);
470         delta = (u64)(u32)(compare - count - 1) + 1;
471         delta = div_u64(delta * NSEC_PER_SEC, vcpu->arch.count_hz);
472         expire = ktime_add_ns(now, delta);
473 
474         /* Update hrtimer to use new timeout */
475         hrtimer_cancel(&vcpu->arch.comparecount_timer);
476         hrtimer_start(&vcpu->arch.comparecount_timer, expire, HRTIMER_MODE_ABS);
477 }
478 
479 /**
480  * kvm_mips_write_count() - Modify the count and update timer.
481  * @vcpu:       Virtual CPU.
482  * @count:      Guest CP0_Count value to set.
483  *
484  * Sets the CP0_Count value and updates the timer accordingly.
485  */
486 void kvm_mips_write_count(struct kvm_vcpu *vcpu, u32 count)
487 {
488         struct mips_coproc *cop0 = vcpu->arch.cop0;
489         ktime_t now;
490 
491         /* Calculate bias */
492         now = kvm_mips_count_time(vcpu);
493         vcpu->arch.count_bias = count - kvm_mips_ktime_to_count(vcpu, now);
494 
495         if (kvm_mips_count_disabled(vcpu))
496                 /* The timer's disabled, adjust the static count */
497                 kvm_write_c0_guest_count(cop0, count);
498         else
499                 /* Update timeout */
500                 kvm_mips_resume_hrtimer(vcpu, now, count);
501 }
502 
503 /**
504  * kvm_mips_init_count() - Initialise timer.
505  * @vcpu:       Virtual CPU.
506  *
507  * Initialise the timer to a sensible frequency, namely 100MHz, zero it, and set
508  * it going if it's enabled.
509  */
510 void kvm_mips_init_count(struct kvm_vcpu *vcpu)
511 {
512         /* 100 MHz */
513         vcpu->arch.count_hz = 100*1000*1000;
514         vcpu->arch.count_period = div_u64((u64)NSEC_PER_SEC << 32,
515                                           vcpu->arch.count_hz);
516         vcpu->arch.count_dyn_bias = 0;
517 
518         /* Starting at 0 */
519         kvm_mips_write_count(vcpu, 0);
520 }
521 
522 /**
523  * kvm_mips_set_count_hz() - Update the frequency of the timer.
524  * @vcpu:       Virtual CPU.
525  * @count_hz:   Frequency of CP0_Count timer in Hz.
526  *
527  * Change the frequency of the CP0_Count timer. This is done atomically so that
528  * CP0_Count is continuous and no timer interrupt is lost.
529  *
530  * Returns:     -EINVAL if @count_hz is out of range.
531  *              0 on success.
532  */
533 int kvm_mips_set_count_hz(struct kvm_vcpu *vcpu, s64 count_hz)
534 {
535         struct mips_coproc *cop0 = vcpu->arch.cop0;
536         int dc;
537         ktime_t now;
538         u32 count;
539 
540         /* ensure the frequency is in a sensible range... */
541         if (count_hz <= 0 || count_hz > NSEC_PER_SEC)
542                 return -EINVAL;
543         /* ... and has actually changed */
544         if (vcpu->arch.count_hz == count_hz)
545                 return 0;
546 
547         /* Safely freeze timer so we can keep it continuous */
548         dc = kvm_mips_count_disabled(vcpu);
549         if (dc) {
550                 now = kvm_mips_count_time(vcpu);
551                 count = kvm_read_c0_guest_count(cop0);
552         } else {
553                 now = kvm_mips_freeze_hrtimer(vcpu, &count);
554         }
555 
556         /* Update the frequency */
557         vcpu->arch.count_hz = count_hz;
558         vcpu->arch.count_period = div_u64((u64)NSEC_PER_SEC << 32, count_hz);
559         vcpu->arch.count_dyn_bias = 0;
560 
561         /* Calculate adjusted bias so dynamic count is unchanged */
562         vcpu->arch.count_bias = count - kvm_mips_ktime_to_count(vcpu, now);
563 
564         /* Update and resume hrtimer */
565         if (!dc)
566                 kvm_mips_resume_hrtimer(vcpu, now, count);
567         return 0;
568 }
569 
570 /**
571  * kvm_mips_write_compare() - Modify compare and update timer.
572  * @vcpu:       Virtual CPU.
573  * @compare:    New CP0_Compare value.
574  * @ack:        Whether to acknowledge timer interrupt.
575  *
576  * Update CP0_Compare to a new value and update the timeout.
577  * If @ack, atomically acknowledge any pending timer interrupt, otherwise ensure
578  * any pending timer interrupt is preserved.
579  */
580 void kvm_mips_write_compare(struct kvm_vcpu *vcpu, u32 compare, bool ack)
581 {
582         struct mips_coproc *cop0 = vcpu->arch.cop0;
583         int dc;
584         u32 old_compare = kvm_read_c0_guest_compare(cop0);
585         ktime_t now;
586         u32 count;
587 
588         /* if unchanged, must just be an ack */
589         if (old_compare == compare) {
590                 if (!ack)
591                         return;
592                 kvm_mips_callbacks->dequeue_timer_int(vcpu);
593                 kvm_write_c0_guest_compare(cop0, compare);
594                 return;
595         }
596 
597         /* freeze_hrtimer() takes care of timer interrupts <= count */
598         dc = kvm_mips_count_disabled(vcpu);
599         if (!dc)
600                 now = kvm_mips_freeze_hrtimer(vcpu, &count);
601 
602         if (ack)
603                 kvm_mips_callbacks->dequeue_timer_int(vcpu);
604 
605         kvm_write_c0_guest_compare(cop0, compare);
606 
607         /* resume_hrtimer() takes care of timer interrupts > count */
608         if (!dc)
609                 kvm_mips_resume_hrtimer(vcpu, now, count);
610 }
611 
612 /**
613  * kvm_mips_count_disable() - Disable count.
614  * @vcpu:       Virtual CPU.
615  *
616  * Disable the CP0_Count timer. A timer interrupt on or before the final stop
617  * time will be handled but not after.
618  *
619  * Assumes CP0_Count was previously enabled but now Guest.CP0_Cause.DC or
620  * count_ctl.DC has been set (count disabled).
621  *
622  * Returns:     The time that the timer was stopped.
623  */
624 static ktime_t kvm_mips_count_disable(struct kvm_vcpu *vcpu)
625 {
626         struct mips_coproc *cop0 = vcpu->arch.cop0;
627         u32 count;
628         ktime_t now;
629 
630         /* Stop hrtimer */
631         hrtimer_cancel(&vcpu->arch.comparecount_timer);
632 
633         /* Set the static count from the dynamic count, handling pending TI */
634         now = ktime_get();
635         count = kvm_mips_read_count_running(vcpu, now);
636         kvm_write_c0_guest_count(cop0, count);
637 
638         return now;
639 }
640 
641 /**
642  * kvm_mips_count_disable_cause() - Disable count using CP0_Cause.DC.
643  * @vcpu:       Virtual CPU.
644  *
645  * Disable the CP0_Count timer and set CP0_Cause.DC. A timer interrupt on or
646  * before the final stop time will be handled if the timer isn't disabled by
647  * count_ctl.DC, but not after.
648  *
649  * Assumes CP0_Cause.DC is clear (count enabled).
650  */
651 void kvm_mips_count_disable_cause(struct kvm_vcpu *vcpu)
652 {
653         struct mips_coproc *cop0 = vcpu->arch.cop0;
654 
655         kvm_set_c0_guest_cause(cop0, CAUSEF_DC);
656         if (!(vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC))
657                 kvm_mips_count_disable(vcpu);
658 }
659 
660 /**
661  * kvm_mips_count_enable_cause() - Enable count using CP0_Cause.DC.
662  * @vcpu:       Virtual CPU.
663  *
664  * Enable the CP0_Count timer and clear CP0_Cause.DC. A timer interrupt after
665  * the start time will be handled if the timer isn't disabled by count_ctl.DC,
666  * potentially before even returning, so the caller should be careful with
667  * ordering of CP0_Cause modifications so as not to lose it.
668  *
669  * Assumes CP0_Cause.DC is set (count disabled).
670  */
671 void kvm_mips_count_enable_cause(struct kvm_vcpu *vcpu)
672 {
673         struct mips_coproc *cop0 = vcpu->arch.cop0;
674         u32 count;
675 
676         kvm_clear_c0_guest_cause(cop0, CAUSEF_DC);
677 
678         /*
679          * Set the dynamic count to match the static count.
680          * This starts the hrtimer if count_ctl.DC allows it.
681          * Otherwise it conveniently updates the biases.
682          */
683         count = kvm_read_c0_guest_count(cop0);
684         kvm_mips_write_count(vcpu, count);
685 }
686 
687 /**
688  * kvm_mips_set_count_ctl() - Update the count control KVM register.
689  * @vcpu:       Virtual CPU.
690  * @count_ctl:  Count control register new value.
691  *
692  * Set the count control KVM register. The timer is updated accordingly.
693  *
694  * Returns:     -EINVAL if reserved bits are set.
695  *              0 on success.
696  */
697 int kvm_mips_set_count_ctl(struct kvm_vcpu *vcpu, s64 count_ctl)
698 {
699         struct mips_coproc *cop0 = vcpu->arch.cop0;
700         s64 changed = count_ctl ^ vcpu->arch.count_ctl;
701         s64 delta;
702         ktime_t expire, now;
703         u32 count, compare;
704 
705         /* Only allow defined bits to be changed */
706         if (changed & ~(s64)(KVM_REG_MIPS_COUNT_CTL_DC))
707                 return -EINVAL;
708 
709         /* Apply new value */
710         vcpu->arch.count_ctl = count_ctl;
711 
712         /* Master CP0_Count disable */
713         if (changed & KVM_REG_MIPS_COUNT_CTL_DC) {
714                 /* Is CP0_Cause.DC already disabling CP0_Count? */
715                 if (kvm_read_c0_guest_cause(cop0) & CAUSEF_DC) {
716                         if (count_ctl & KVM_REG_MIPS_COUNT_CTL_DC)
717                                 /* Just record the current time */
718                                 vcpu->arch.count_resume = ktime_get();
719                 } else if (count_ctl & KVM_REG_MIPS_COUNT_CTL_DC) {
720                         /* disable timer and record current time */
721                         vcpu->arch.count_resume = kvm_mips_count_disable(vcpu);
722                 } else {
723                         /*
724                          * Calculate timeout relative to static count at resume
725                          * time (wrap 0 to 2^32).
726                          */
727                         count = kvm_read_c0_guest_count(cop0);
728                         compare = kvm_read_c0_guest_compare(cop0);
729                         delta = (u64)(u32)(compare - count - 1) + 1;
730                         delta = div_u64(delta * NSEC_PER_SEC,
731                                         vcpu->arch.count_hz);
732                         expire = ktime_add_ns(vcpu->arch.count_resume, delta);
733 
734                         /* Handle pending interrupt */
735                         now = ktime_get();
736                         if (ktime_compare(now, expire) >= 0)
737                                 /* Nothing should be waiting on the timeout */
738                                 kvm_mips_callbacks->queue_timer_int(vcpu);
739 
740                         /* Resume hrtimer without changing bias */
741                         count = kvm_mips_read_count_running(vcpu, now);
742                         kvm_mips_resume_hrtimer(vcpu, now, count);
743                 }
744         }
745 
746         return 0;
747 }
748 
749 /**
750  * kvm_mips_set_count_resume() - Update the count resume KVM register.
751  * @vcpu:               Virtual CPU.
752  * @count_resume:       Count resume register new value.
753  *
754  * Set the count resume KVM register.
755  *
756  * Returns:     -EINVAL if out of valid range (0..now).
757  *              0 on success.
758  */
759 int kvm_mips_set_count_resume(struct kvm_vcpu *vcpu, s64 count_resume)
760 {
761         /*
762          * It doesn't make sense for the resume time to be in the future, as it
763          * would be possible for the next interrupt to be more than a full
764          * period in the future.
765          */
766         if (count_resume < 0 || count_resume > ktime_to_ns(ktime_get()))
767                 return -EINVAL;
768 
769         vcpu->arch.count_resume = ns_to_ktime(count_resume);
770         return 0;
771 }
772 
773 /**
774  * kvm_mips_count_timeout() - Push timer forward on timeout.
775  * @vcpu:       Virtual CPU.
776  *
777  * Handle an hrtimer event by push the hrtimer forward a period.
778  *
779  * Returns:     The hrtimer_restart value to return to the hrtimer subsystem.
780  */
781 enum hrtimer_restart kvm_mips_count_timeout(struct kvm_vcpu *vcpu)
782 {
783         /* Add the Count period to the current expiry time */
784         hrtimer_add_expires_ns(&vcpu->arch.comparecount_timer,
785                                vcpu->arch.count_period);
786         return HRTIMER_RESTART;
787 }
788 
789 enum emulation_result kvm_mips_emul_eret(struct kvm_vcpu *vcpu)
790 {
791         struct mips_coproc *cop0 = vcpu->arch.cop0;
792         enum emulation_result er = EMULATE_DONE;
793 
794         if (kvm_read_c0_guest_status(cop0) & ST0_ERL) {
795                 kvm_clear_c0_guest_status(cop0, ST0_ERL);
796                 vcpu->arch.pc = kvm_read_c0_guest_errorepc(cop0);
797         } else if (kvm_read_c0_guest_status(cop0) & ST0_EXL) {
798                 kvm_debug("[%#lx] ERET to %#lx\n", vcpu->arch.pc,
799                           kvm_read_c0_guest_epc(cop0));
800                 kvm_clear_c0_guest_status(cop0, ST0_EXL);
801                 vcpu->arch.pc = kvm_read_c0_guest_epc(cop0);
802 
803         } else {
804                 kvm_err("[%#lx] ERET when MIPS_SR_EXL|MIPS_SR_ERL == 0\n",
805                         vcpu->arch.pc);
806                 er = EMULATE_FAIL;
807         }
808 
809         return er;
810 }
811 
812 enum emulation_result kvm_mips_emul_wait(struct kvm_vcpu *vcpu)
813 {
814         kvm_debug("[%#lx] !!!WAIT!!! (%#lx)\n", vcpu->arch.pc,
815                   vcpu->arch.pending_exceptions);
816 
817         ++vcpu->stat.wait_exits;
818         trace_kvm_exit(vcpu, KVM_TRACE_EXIT_WAIT);
819         if (!vcpu->arch.pending_exceptions) {
820                 vcpu->arch.wait = 1;
821                 kvm_vcpu_block(vcpu);
822 
823                 /*
824                  * We we are runnable, then definitely go off to user space to
825                  * check if any I/O interrupts are pending.
826                  */
827                 if (kvm_check_request(KVM_REQ_UNHALT, vcpu)) {
828                         clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
829                         vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
830                 }
831         }
832 
833         return EMULATE_DONE;
834 }
835 
836 /*
837  * XXXKYMA: Linux doesn't seem to use TLBR, return EMULATE_FAIL for now so that
838  * we can catch this, if things ever change
839  */
840 enum emulation_result kvm_mips_emul_tlbr(struct kvm_vcpu *vcpu)
841 {
842         struct mips_coproc *cop0 = vcpu->arch.cop0;
843         unsigned long pc = vcpu->arch.pc;
844 
845         kvm_err("[%#lx] COP0_TLBR [%ld]\n", pc, kvm_read_c0_guest_index(cop0));
846         return EMULATE_FAIL;
847 }
848 
849 /**
850  * kvm_mips_invalidate_guest_tlb() - Indicates a change in guest MMU map.
851  * @vcpu:       VCPU with changed mappings.
852  * @tlb:        TLB entry being removed.
853  *
854  * This is called to indicate a single change in guest MMU mappings, so that we
855  * can arrange TLB flushes on this and other CPUs.
856  */
857 static void kvm_mips_invalidate_guest_tlb(struct kvm_vcpu *vcpu,
858                                           struct kvm_mips_tlb *tlb)
859 {
860         int cpu, i;
861         bool user;
862 
863         /* No need to flush for entries which are already invalid */
864         if (!((tlb->tlb_lo[0] | tlb->tlb_lo[1]) & ENTRYLO_V))
865                 return;
866         /* User address space doesn't need flushing for KSeg2/3 changes */
867         user = tlb->tlb_hi < KVM_GUEST_KSEG0;
868 
869         preempt_disable();
870 
871         /*
872          * Probe the shadow host TLB for the entry being overwritten, if one
873          * matches, invalidate it
874          */
875         kvm_mips_host_tlb_inv(vcpu, tlb->tlb_hi);
876 
877         /* Invalidate the whole ASID on other CPUs */
878         cpu = smp_processor_id();
879         for_each_possible_cpu(i) {
880                 if (i == cpu)
881                         continue;
882                 if (user)
883                         vcpu->arch.guest_user_asid[i] = 0;
884                 vcpu->arch.guest_kernel_asid[i] = 0;
885         }
886 
887         preempt_enable();
888 }
889 
890 /* Write Guest TLB Entry @ Index */
891 enum emulation_result kvm_mips_emul_tlbwi(struct kvm_vcpu *vcpu)
892 {
893         struct mips_coproc *cop0 = vcpu->arch.cop0;
894         int index = kvm_read_c0_guest_index(cop0);
895         struct kvm_mips_tlb *tlb = NULL;
896         unsigned long pc = vcpu->arch.pc;
897 
898         if (index < 0 || index >= KVM_MIPS_GUEST_TLB_SIZE) {
899                 kvm_debug("%s: illegal index: %d\n", __func__, index);
900                 kvm_debug("[%#lx] COP0_TLBWI [%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx, mask: %#lx)\n",
901                           pc, index, kvm_read_c0_guest_entryhi(cop0),
902                           kvm_read_c0_guest_entrylo0(cop0),
903                           kvm_read_c0_guest_entrylo1(cop0),
904                           kvm_read_c0_guest_pagemask(cop0));
905                 index = (index & ~0x80000000) % KVM_MIPS_GUEST_TLB_SIZE;
906         }
907 
908         tlb = &vcpu->arch.guest_tlb[index];
909 
910         kvm_mips_invalidate_guest_tlb(vcpu, tlb);
911 
912         tlb->tlb_mask = kvm_read_c0_guest_pagemask(cop0);
913         tlb->tlb_hi = kvm_read_c0_guest_entryhi(cop0);
914         tlb->tlb_lo[0] = kvm_read_c0_guest_entrylo0(cop0);
915         tlb->tlb_lo[1] = kvm_read_c0_guest_entrylo1(cop0);
916 
917         kvm_debug("[%#lx] COP0_TLBWI [%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx, mask: %#lx)\n",
918                   pc, index, kvm_read_c0_guest_entryhi(cop0),
919                   kvm_read_c0_guest_entrylo0(cop0),
920                   kvm_read_c0_guest_entrylo1(cop0),
921                   kvm_read_c0_guest_pagemask(cop0));
922 
923         return EMULATE_DONE;
924 }
925 
926 /* Write Guest TLB Entry @ Random Index */
927 enum emulation_result kvm_mips_emul_tlbwr(struct kvm_vcpu *vcpu)
928 {
929         struct mips_coproc *cop0 = vcpu->arch.cop0;
930         struct kvm_mips_tlb *tlb = NULL;
931         unsigned long pc = vcpu->arch.pc;
932         int index;
933 
934         get_random_bytes(&index, sizeof(index));
935         index &= (KVM_MIPS_GUEST_TLB_SIZE - 1);
936 
937         tlb = &vcpu->arch.guest_tlb[index];
938 
939         kvm_mips_invalidate_guest_tlb(vcpu, tlb);
940 
941         tlb->tlb_mask = kvm_read_c0_guest_pagemask(cop0);
942         tlb->tlb_hi = kvm_read_c0_guest_entryhi(cop0);
943         tlb->tlb_lo[0] = kvm_read_c0_guest_entrylo0(cop0);
944         tlb->tlb_lo[1] = kvm_read_c0_guest_entrylo1(cop0);
945 
946         kvm_debug("[%#lx] COP0_TLBWR[%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx)\n",
947                   pc, index, kvm_read_c0_guest_entryhi(cop0),
948                   kvm_read_c0_guest_entrylo0(cop0),
949                   kvm_read_c0_guest_entrylo1(cop0));
950 
951         return EMULATE_DONE;
952 }
953 
954 enum emulation_result kvm_mips_emul_tlbp(struct kvm_vcpu *vcpu)
955 {
956         struct mips_coproc *cop0 = vcpu->arch.cop0;
957         long entryhi = kvm_read_c0_guest_entryhi(cop0);
958         unsigned long pc = vcpu->arch.pc;
959         int index = -1;
960 
961         index = kvm_mips_guest_tlb_lookup(vcpu, entryhi);
962 
963         kvm_write_c0_guest_index(cop0, index);
964 
965         kvm_debug("[%#lx] COP0_TLBP (entryhi: %#lx), index: %d\n", pc, entryhi,
966                   index);
967 
968         return EMULATE_DONE;
969 }
970 
971 /**
972  * kvm_mips_config1_wrmask() - Find mask of writable bits in guest Config1
973  * @vcpu:       Virtual CPU.
974  *
975  * Finds the mask of bits which are writable in the guest's Config1 CP0
976  * register, by userland (currently read-only to the guest).
977  */
978 unsigned int kvm_mips_config1_wrmask(struct kvm_vcpu *vcpu)
979 {
980         unsigned int mask = 0;
981 
982         /* Permit FPU to be present if FPU is supported */
983         if (kvm_mips_guest_can_have_fpu(&vcpu->arch))
984                 mask |= MIPS_CONF1_FP;
985 
986         return mask;
987 }
988 
989 /**
990  * kvm_mips_config3_wrmask() - Find mask of writable bits in guest Config3
991  * @vcpu:       Virtual CPU.
992  *
993  * Finds the mask of bits which are writable in the guest's Config3 CP0
994  * register, by userland (currently read-only to the guest).
995  */
996 unsigned int kvm_mips_config3_wrmask(struct kvm_vcpu *vcpu)
997 {
998         /* Config4 and ULRI are optional */
999         unsigned int mask = MIPS_CONF_M | MIPS_CONF3_ULRI;
1000 
1001         /* Permit MSA to be present if MSA is supported */
1002         if (kvm_mips_guest_can_have_msa(&vcpu->arch))
1003                 mask |= MIPS_CONF3_MSA;
1004 
1005         return mask;
1006 }
1007 
1008 /**
1009  * kvm_mips_config4_wrmask() - Find mask of writable bits in guest Config4
1010  * @vcpu:       Virtual CPU.
1011  *
1012  * Finds the mask of bits which are writable in the guest's Config4 CP0
1013  * register, by userland (currently read-only to the guest).
1014  */
1015 unsigned int kvm_mips_config4_wrmask(struct kvm_vcpu *vcpu)
1016 {
1017         /* Config5 is optional */
1018         unsigned int mask = MIPS_CONF_M;
1019 
1020         /* KScrExist */
1021         mask |= (unsigned int)vcpu->arch.kscratch_enabled << 16;
1022 
1023         return mask;
1024 }
1025 
1026 /**
1027  * kvm_mips_config5_wrmask() - Find mask of writable bits in guest Config5
1028  * @vcpu:       Virtual CPU.
1029  *
1030  * Finds the mask of bits which are writable in the guest's Config5 CP0
1031  * register, by the guest itself.
1032  */
1033 unsigned int kvm_mips_config5_wrmask(struct kvm_vcpu *vcpu)
1034 {
1035         unsigned int mask = 0;
1036 
1037         /* Permit MSAEn changes if MSA supported and enabled */
1038         if (kvm_mips_guest_has_msa(&vcpu->arch))
1039                 mask |= MIPS_CONF5_MSAEN;
1040 
1041         /*
1042          * Permit guest FPU mode changes if FPU is enabled and the relevant
1043          * feature exists according to FIR register.
1044          */
1045         if (kvm_mips_guest_has_fpu(&vcpu->arch)) {
1046                 if (cpu_has_fre)
1047                         mask |= MIPS_CONF5_FRE;
1048                 /* We don't support UFR or UFE */
1049         }
1050 
1051         return mask;
1052 }
1053 
1054 enum emulation_result kvm_mips_emulate_CP0(union mips_instruction inst,
1055                                            u32 *opc, u32 cause,
1056                                            struct kvm_run *run,
1057                                            struct kvm_vcpu *vcpu)
1058 {
1059         struct mips_coproc *cop0 = vcpu->arch.cop0;
1060         enum emulation_result er = EMULATE_DONE;
1061         u32 rt, rd, sel;
1062         unsigned long curr_pc;
1063         int cpu, i;
1064 
1065         /*
1066          * Update PC and hold onto current PC in case there is
1067          * an error and we want to rollback the PC
1068          */
1069         curr_pc = vcpu->arch.pc;
1070         er = update_pc(vcpu, cause);
1071         if (er == EMULATE_FAIL)
1072                 return er;
1073 
1074         if (inst.co_format.co) {
1075                 switch (inst.co_format.func) {
1076                 case tlbr_op:   /*  Read indexed TLB entry  */
1077                         er = kvm_mips_emul_tlbr(vcpu);
1078                         break;
1079                 case tlbwi_op:  /*  Write indexed  */
1080                         er = kvm_mips_emul_tlbwi(vcpu);
1081                         break;
1082                 case tlbwr_op:  /*  Write random  */
1083                         er = kvm_mips_emul_tlbwr(vcpu);
1084                         break;
1085                 case tlbp_op:   /* TLB Probe */
1086                         er = kvm_mips_emul_tlbp(vcpu);
1087                         break;
1088                 case rfe_op:
1089                         kvm_err("!!!COP0_RFE!!!\n");
1090                         break;
1091                 case eret_op:
1092                         er = kvm_mips_emul_eret(vcpu);
1093                         goto dont_update_pc;
1094                 case wait_op:
1095                         er = kvm_mips_emul_wait(vcpu);
1096                         break;
1097                 }
1098         } else {
1099                 rt = inst.c0r_format.rt;
1100                 rd = inst.c0r_format.rd;
1101                 sel = inst.c0r_format.sel;
1102 
1103                 switch (inst.c0r_format.rs) {
1104                 case mfc_op:
1105 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS
1106                         cop0->stat[rd][sel]++;
1107 #endif
1108                         /* Get reg */
1109                         if ((rd == MIPS_CP0_COUNT) && (sel == 0)) {
1110                                 vcpu->arch.gprs[rt] =
1111                                     (s32)kvm_mips_read_count(vcpu);
1112                         } else if ((rd == MIPS_CP0_ERRCTL) && (sel == 0)) {
1113                                 vcpu->arch.gprs[rt] = 0x0;
1114 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1115                                 kvm_mips_trans_mfc0(inst, opc, vcpu);
1116 #endif
1117                         } else {
1118                                 vcpu->arch.gprs[rt] = (s32)cop0->reg[rd][sel];
1119 
1120 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1121                                 kvm_mips_trans_mfc0(inst, opc, vcpu);
1122 #endif
1123                         }
1124 
1125                         trace_kvm_hwr(vcpu, KVM_TRACE_MFC0,
1126                                       KVM_TRACE_COP0(rd, sel),
1127                                       vcpu->arch.gprs[rt]);
1128                         break;
1129 
1130                 case dmfc_op:
1131                         vcpu->arch.gprs[rt] = cop0->reg[rd][sel];
1132 
1133                         trace_kvm_hwr(vcpu, KVM_TRACE_DMFC0,
1134                                       KVM_TRACE_COP0(rd, sel),
1135                                       vcpu->arch.gprs[rt]);
1136                         break;
1137 
1138                 case mtc_op:
1139 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS
1140                         cop0->stat[rd][sel]++;
1141 #endif
1142                         trace_kvm_hwr(vcpu, KVM_TRACE_MTC0,
1143                                       KVM_TRACE_COP0(rd, sel),
1144                                       vcpu->arch.gprs[rt]);
1145 
1146                         if ((rd == MIPS_CP0_TLB_INDEX)
1147                             && (vcpu->arch.gprs[rt] >=
1148                                 KVM_MIPS_GUEST_TLB_SIZE)) {
1149                                 kvm_err("Invalid TLB Index: %ld",
1150                                         vcpu->arch.gprs[rt]);
1151                                 er = EMULATE_FAIL;
1152                                 break;
1153                         }
1154 #define C0_EBASE_CORE_MASK 0xff
1155                         if ((rd == MIPS_CP0_PRID) && (sel == 1)) {
1156                                 /* Preserve CORE number */
1157                                 kvm_change_c0_guest_ebase(cop0,
1158                                                           ~(C0_EBASE_CORE_MASK),
1159                                                           vcpu->arch.gprs[rt]);
1160                                 kvm_err("MTCz, cop0->reg[EBASE]: %#lx\n",
1161                                         kvm_read_c0_guest_ebase(cop0));
1162                         } else if (rd == MIPS_CP0_TLB_HI && sel == 0) {
1163                                 u32 nasid =
1164                                         vcpu->arch.gprs[rt] & KVM_ENTRYHI_ASID;
1165                                 if ((KSEGX(vcpu->arch.gprs[rt]) != CKSEG0) &&
1166                                     ((kvm_read_c0_guest_entryhi(cop0) &
1167                                       KVM_ENTRYHI_ASID) != nasid)) {
1168                                         trace_kvm_asid_change(vcpu,
1169                                                 kvm_read_c0_guest_entryhi(cop0)
1170                                                         & KVM_ENTRYHI_ASID,
1171                                                 nasid);
1172 
1173                                         preempt_disable();
1174                                         /* Blow away the shadow host TLBs */
1175                                         kvm_mips_flush_host_tlb(1);
1176                                         cpu = smp_processor_id();
1177                                         for_each_possible_cpu(i)
1178                                                 if (i != cpu) {
1179                                                         vcpu->arch.guest_user_asid[i] = 0;
1180                                                         vcpu->arch.guest_kernel_asid[i] = 0;
1181                                                 }
1182                                         preempt_enable();
1183                                 }
1184                                 kvm_write_c0_guest_entryhi(cop0,
1185                                                            vcpu->arch.gprs[rt]);
1186                         }
1187                         /* Are we writing to COUNT */
1188                         else if ((rd == MIPS_CP0_COUNT) && (sel == 0)) {
1189                                 kvm_mips_write_count(vcpu, vcpu->arch.gprs[rt]);
1190                                 goto done;
1191                         } else if ((rd == MIPS_CP0_COMPARE) && (sel == 0)) {
1192                                 /* If we are writing to COMPARE */
1193                                 /* Clear pending timer interrupt, if any */
1194                                 kvm_mips_write_compare(vcpu,
1195                                                        vcpu->arch.gprs[rt],
1196                                                        true);
1197                         } else if ((rd == MIPS_CP0_STATUS) && (sel == 0)) {
1198                                 unsigned int old_val, val, change;
1199 
1200                                 old_val = kvm_read_c0_guest_status(cop0);
1201                                 val = vcpu->arch.gprs[rt];
1202                                 change = val ^ old_val;
1203 
1204                                 /* Make sure that the NMI bit is never set */
1205                                 val &= ~ST0_NMI;
1206 
1207                                 /*
1208                                  * Don't allow CU1 or FR to be set unless FPU
1209                                  * capability enabled and exists in guest
1210                                  * configuration.
1211                                  */
1212                                 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
1213                                         val &= ~(ST0_CU1 | ST0_FR);
1214 
1215                                 /*
1216                                  * Also don't allow FR to be set if host doesn't
1217                                  * support it.
1218                                  */
1219                                 if (!(current_cpu_data.fpu_id & MIPS_FPIR_F64))
1220                                         val &= ~ST0_FR;
1221 
1222 
1223                                 /* Handle changes in FPU mode */
1224                                 preempt_disable();
1225 
1226                                 /*
1227                                  * FPU and Vector register state is made
1228                                  * UNPREDICTABLE by a change of FR, so don't
1229                                  * even bother saving it.
1230                                  */
1231                                 if (change & ST0_FR)
1232                                         kvm_drop_fpu(vcpu);
1233 
1234                                 /*
1235                                  * If MSA state is already live, it is undefined
1236                                  * how it interacts with FR=0 FPU state, and we
1237                                  * don't want to hit reserved instruction
1238                                  * exceptions trying to save the MSA state later
1239                                  * when CU=1 && FR=1, so play it safe and save
1240                                  * it first.
1241                                  */
1242                                 if (change & ST0_CU1 && !(val & ST0_FR) &&
1243                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA)
1244                                         kvm_lose_fpu(vcpu);
1245 
1246                                 /*
1247                                  * Propagate CU1 (FPU enable) changes
1248                                  * immediately if the FPU context is already
1249                                  * loaded. When disabling we leave the context
1250                                  * loaded so it can be quickly enabled again in
1251                                  * the near future.
1252                                  */
1253                                 if (change & ST0_CU1 &&
1254                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)
1255                                         change_c0_status(ST0_CU1, val);
1256 
1257                                 preempt_enable();
1258 
1259                                 kvm_write_c0_guest_status(cop0, val);
1260 
1261 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1262                                 /*
1263                                  * If FPU present, we need CU1/FR bits to take
1264                                  * effect fairly soon.
1265                                  */
1266                                 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
1267                                         kvm_mips_trans_mtc0(inst, opc, vcpu);
1268 #endif
1269                         } else if ((rd == MIPS_CP0_CONFIG) && (sel == 5)) {
1270                                 unsigned int old_val, val, change, wrmask;
1271 
1272                                 old_val = kvm_read_c0_guest_config5(cop0);
1273                                 val = vcpu->arch.gprs[rt];
1274 
1275                                 /* Only a few bits are writable in Config5 */
1276                                 wrmask = kvm_mips_config5_wrmask(vcpu);
1277                                 change = (val ^ old_val) & wrmask;
1278                                 val = old_val ^ change;
1279 
1280 
1281                                 /* Handle changes in FPU/MSA modes */
1282                                 preempt_disable();
1283 
1284                                 /*
1285                                  * Propagate FRE changes immediately if the FPU
1286                                  * context is already loaded.
1287                                  */
1288                                 if (change & MIPS_CONF5_FRE &&
1289                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)
1290                                         change_c0_config5(MIPS_CONF5_FRE, val);
1291 
1292                                 /*
1293                                  * Propagate MSAEn changes immediately if the
1294                                  * MSA context is already loaded. When disabling
1295                                  * we leave the context loaded so it can be
1296                                  * quickly enabled again in the near future.
1297                                  */
1298                                 if (change & MIPS_CONF5_MSAEN &&
1299                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA)
1300                                         change_c0_config5(MIPS_CONF5_MSAEN,
1301                                                           val);
1302 
1303                                 preempt_enable();
1304 
1305                                 kvm_write_c0_guest_config5(cop0, val);
1306                         } else if ((rd == MIPS_CP0_CAUSE) && (sel == 0)) {
1307                                 u32 old_cause, new_cause;
1308 
1309                                 old_cause = kvm_read_c0_guest_cause(cop0);
1310                                 new_cause = vcpu->arch.gprs[rt];
1311                                 /* Update R/W bits */
1312                                 kvm_change_c0_guest_cause(cop0, 0x08800300,
1313                                                           new_cause);
1314                                 /* DC bit enabling/disabling timer? */
1315                                 if ((old_cause ^ new_cause) & CAUSEF_DC) {
1316                                         if (new_cause & CAUSEF_DC)
1317                                                 kvm_mips_count_disable_cause(vcpu);
1318                                         else
1319                                                 kvm_mips_count_enable_cause(vcpu);
1320                                 }
1321                         } else if ((rd == MIPS_CP0_HWRENA) && (sel == 0)) {
1322                                 u32 mask = MIPS_HWRENA_CPUNUM |
1323                                            MIPS_HWRENA_SYNCISTEP |
1324                                            MIPS_HWRENA_CC |
1325                                            MIPS_HWRENA_CCRES;
1326 
1327                                 if (kvm_read_c0_guest_config3(cop0) &
1328                                     MIPS_CONF3_ULRI)
1329                                         mask |= MIPS_HWRENA_ULR;
1330                                 cop0->reg[rd][sel] = vcpu->arch.gprs[rt] & mask;
1331                         } else {
1332                                 cop0->reg[rd][sel] = vcpu->arch.gprs[rt];
1333 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1334                                 kvm_mips_trans_mtc0(inst, opc, vcpu);
1335 #endif
1336                         }
1337                         break;
1338 
1339                 case dmtc_op:
1340                         kvm_err("!!!!!!![%#lx]dmtc_op: rt: %d, rd: %d, sel: %d!!!!!!\n",
1341                                 vcpu->arch.pc, rt, rd, sel);
1342                         trace_kvm_hwr(vcpu, KVM_TRACE_DMTC0,
1343                                       KVM_TRACE_COP0(rd, sel),
1344                                       vcpu->arch.gprs[rt]);
1345                         er = EMULATE_FAIL;
1346                         break;
1347 
1348                 case mfmc0_op:
1349 #ifdef KVM_MIPS_DEBUG_COP0_COUNTERS
1350                         cop0->stat[MIPS_CP0_STATUS][0]++;
1351 #endif
1352                         if (rt != 0)
1353                                 vcpu->arch.gprs[rt] =
1354                                     kvm_read_c0_guest_status(cop0);
1355                         /* EI */
1356                         if (inst.mfmc0_format.sc) {
1357                                 kvm_debug("[%#lx] mfmc0_op: EI\n",
1358                                           vcpu->arch.pc);
1359                                 kvm_set_c0_guest_status(cop0, ST0_IE);
1360                         } else {
1361                                 kvm_debug("[%#lx] mfmc0_op: DI\n",
1362                                           vcpu->arch.pc);
1363                                 kvm_clear_c0_guest_status(cop0, ST0_IE);
1364                         }
1365 
1366                         break;
1367 
1368                 case wrpgpr_op:
1369                         {
1370                                 u32 css = cop0->reg[MIPS_CP0_STATUS][2] & 0xf;
1371                                 u32 pss =
1372                                     (cop0->reg[MIPS_CP0_STATUS][2] >> 6) & 0xf;
1373                                 /*
1374                                  * We don't support any shadow register sets, so
1375                                  * SRSCtl[PSS] == SRSCtl[CSS] = 0
1376                                  */
1377                                 if (css || pss) {
1378                                         er = EMULATE_FAIL;
1379                                         break;
1380                                 }
1381                                 kvm_debug("WRPGPR[%d][%d] = %#lx\n", pss, rd,
1382                                           vcpu->arch.gprs[rt]);
1383                                 vcpu->arch.gprs[rd] = vcpu->arch.gprs[rt];
1384                         }
1385                         break;
1386                 default:
1387                         kvm_err("[%#lx]MachEmulateCP0: unsupported COP0, copz: 0x%x\n",
1388                                 vcpu->arch.pc, inst.c0r_format.rs);
1389                         er = EMULATE_FAIL;
1390                         break;
1391                 }
1392         }
1393 
1394 done:
1395         /* Rollback PC only if emulation was unsuccessful */
1396         if (er == EMULATE_FAIL)
1397                 vcpu->arch.pc = curr_pc;
1398 
1399 dont_update_pc:
1400         /*
1401          * This is for special instructions whose emulation
1402          * updates the PC, so do not overwrite the PC under
1403          * any circumstances
1404          */
1405 
1406         return er;
1407 }
1408 
1409 enum emulation_result kvm_mips_emulate_store(union mips_instruction inst,
1410                                              u32 cause,
1411                                              struct kvm_run *run,
1412                                              struct kvm_vcpu *vcpu)
1413 {
1414         enum emulation_result er = EMULATE_DO_MMIO;
1415         u32 rt;
1416         u32 bytes;
1417         void *data = run->mmio.data;
1418         unsigned long curr_pc;
1419 
1420         /*
1421          * Update PC and hold onto current PC in case there is
1422          * an error and we want to rollback the PC
1423          */
1424         curr_pc = vcpu->arch.pc;
1425         er = update_pc(vcpu, cause);
1426         if (er == EMULATE_FAIL)
1427                 return er;
1428 
1429         rt = inst.i_format.rt;
1430 
1431         switch (inst.i_format.opcode) {
1432         case sb_op:
1433                 bytes = 1;
1434                 if (bytes > sizeof(run->mmio.data)) {
1435                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1436                                run->mmio.len);
1437                 }
1438                 run->mmio.phys_addr =
1439                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1440                                                    host_cp0_badvaddr);
1441                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1442                         er = EMULATE_FAIL;
1443                         break;
1444                 }
1445                 run->mmio.len = bytes;
1446                 run->mmio.is_write = 1;
1447                 vcpu->mmio_needed = 1;
1448                 vcpu->mmio_is_write = 1;
1449                 *(u8 *) data = vcpu->arch.gprs[rt];
1450                 kvm_debug("OP_SB: eaddr: %#lx, gpr: %#lx, data: %#x\n",
1451                           vcpu->arch.host_cp0_badvaddr, vcpu->arch.gprs[rt],
1452                           *(u8 *) data);
1453 
1454                 break;
1455 
1456         case sw_op:
1457                 bytes = 4;
1458                 if (bytes > sizeof(run->mmio.data)) {
1459                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1460                                run->mmio.len);
1461                 }
1462                 run->mmio.phys_addr =
1463                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1464                                                    host_cp0_badvaddr);
1465                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1466                         er = EMULATE_FAIL;
1467                         break;
1468                 }
1469 
1470                 run->mmio.len = bytes;
1471                 run->mmio.is_write = 1;
1472                 vcpu->mmio_needed = 1;
1473                 vcpu->mmio_is_write = 1;
1474                 *(u32 *) data = vcpu->arch.gprs[rt];
1475 
1476                 kvm_debug("[%#lx] OP_SW: eaddr: %#lx, gpr: %#lx, data: %#x\n",
1477                           vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr,
1478                           vcpu->arch.gprs[rt], *(u32 *) data);
1479                 break;
1480 
1481         case sh_op:
1482                 bytes = 2;
1483                 if (bytes > sizeof(run->mmio.data)) {
1484                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1485                                run->mmio.len);
1486                 }
1487                 run->mmio.phys_addr =
1488                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1489                                                    host_cp0_badvaddr);
1490                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1491                         er = EMULATE_FAIL;
1492                         break;
1493                 }
1494 
1495                 run->mmio.len = bytes;
1496                 run->mmio.is_write = 1;
1497                 vcpu->mmio_needed = 1;
1498                 vcpu->mmio_is_write = 1;
1499                 *(u16 *) data = vcpu->arch.gprs[rt];
1500 
1501                 kvm_debug("[%#lx] OP_SH: eaddr: %#lx, gpr: %#lx, data: %#x\n",
1502                           vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr,
1503                           vcpu->arch.gprs[rt], *(u32 *) data);
1504                 break;
1505 
1506         default:
1507                 kvm_err("Store not yet supported (inst=0x%08x)\n",
1508                         inst.word);
1509                 er = EMULATE_FAIL;
1510                 break;
1511         }
1512 
1513         /* Rollback PC if emulation was unsuccessful */
1514         if (er == EMULATE_FAIL)
1515                 vcpu->arch.pc = curr_pc;
1516 
1517         return er;
1518 }
1519 
1520 enum emulation_result kvm_mips_emulate_load(union mips_instruction inst,
1521                                             u32 cause, struct kvm_run *run,
1522                                             struct kvm_vcpu *vcpu)
1523 {
1524         enum emulation_result er = EMULATE_DO_MMIO;
1525         unsigned long curr_pc;
1526         u32 op, rt;
1527         u32 bytes;
1528 
1529         rt = inst.i_format.rt;
1530         op = inst.i_format.opcode;
1531 
1532         /*
1533          * Find the resume PC now while we have safe and easy access to the
1534          * prior branch instruction, and save it for
1535          * kvm_mips_complete_mmio_load() to restore later.
1536          */
1537         curr_pc = vcpu->arch.pc;
1538         er = update_pc(vcpu, cause);
1539         if (er == EMULATE_FAIL)
1540                 return er;
1541         vcpu->arch.io_pc = vcpu->arch.pc;
1542         vcpu->arch.pc = curr_pc;
1543 
1544         vcpu->arch.io_gpr = rt;
1545 
1546         switch (op) {
1547         case lw_op:
1548                 bytes = 4;
1549                 if (bytes > sizeof(run->mmio.data)) {
1550                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1551                                run->mmio.len);
1552                         er = EMULATE_FAIL;
1553                         break;
1554                 }
1555                 run->mmio.phys_addr =
1556                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1557                                                    host_cp0_badvaddr);
1558                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1559                         er = EMULATE_FAIL;
1560                         break;
1561                 }
1562 
1563                 run->mmio.len = bytes;
1564                 run->mmio.is_write = 0;
1565                 vcpu->mmio_needed = 1;
1566                 vcpu->mmio_is_write = 0;
1567                 break;
1568 
1569         case lh_op:
1570         case lhu_op:
1571                 bytes = 2;
1572                 if (bytes > sizeof(run->mmio.data)) {
1573                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1574                                run->mmio.len);
1575                         er = EMULATE_FAIL;
1576                         break;
1577                 }
1578                 run->mmio.phys_addr =
1579                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1580                                                    host_cp0_badvaddr);
1581                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1582                         er = EMULATE_FAIL;
1583                         break;
1584                 }
1585 
1586                 run->mmio.len = bytes;
1587                 run->mmio.is_write = 0;
1588                 vcpu->mmio_needed = 1;
1589                 vcpu->mmio_is_write = 0;
1590 
1591                 if (op == lh_op)
1592                         vcpu->mmio_needed = 2;
1593                 else
1594                         vcpu->mmio_needed = 1;
1595 
1596                 break;
1597 
1598         case lbu_op:
1599         case lb_op:
1600                 bytes = 1;
1601                 if (bytes > sizeof(run->mmio.data)) {
1602                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1603                                run->mmio.len);
1604                         er = EMULATE_FAIL;
1605                         break;
1606                 }
1607                 run->mmio.phys_addr =
1608                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1609                                                    host_cp0_badvaddr);
1610                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1611                         er = EMULATE_FAIL;
1612                         break;
1613                 }
1614 
1615                 run->mmio.len = bytes;
1616                 run->mmio.is_write = 0;
1617                 vcpu->mmio_is_write = 0;
1618 
1619                 if (op == lb_op)
1620                         vcpu->mmio_needed = 2;
1621                 else
1622                         vcpu->mmio_needed = 1;
1623 
1624                 break;
1625 
1626         default:
1627                 kvm_err("Load not yet supported (inst=0x%08x)\n",
1628                         inst.word);
1629                 er = EMULATE_FAIL;
1630                 break;
1631         }
1632 
1633         return er;
1634 }
1635 
1636 enum emulation_result kvm_mips_emulate_cache(union mips_instruction inst,
1637                                              u32 *opc, u32 cause,
1638                                              struct kvm_run *run,
1639                                              struct kvm_vcpu *vcpu)
1640 {
1641         struct mips_coproc *cop0 = vcpu->arch.cop0;
1642         enum emulation_result er = EMULATE_DONE;
1643         u32 cache, op_inst, op, base;
1644         s16 offset;
1645         struct kvm_vcpu_arch *arch = &vcpu->arch;
1646         unsigned long va;
1647         unsigned long curr_pc;
1648 
1649         /*
1650          * Update PC and hold onto current PC in case there is
1651          * an error and we want to rollback the PC
1652          */
1653         curr_pc = vcpu->arch.pc;
1654         er = update_pc(vcpu, cause);
1655         if (er == EMULATE_FAIL)
1656                 return er;
1657 
1658         base = inst.i_format.rs;
1659         op_inst = inst.i_format.rt;
1660         if (cpu_has_mips_r6)
1661                 offset = inst.spec3_format.simmediate;
1662         else
1663                 offset = inst.i_format.simmediate;
1664         cache = op_inst & CacheOp_Cache;
1665         op = op_inst & CacheOp_Op;
1666 
1667         va = arch->gprs[base] + offset;
1668 
1669         kvm_debug("CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1670                   cache, op, base, arch->gprs[base], offset);
1671 
1672         /*
1673          * Treat INDEX_INV as a nop, basically issued by Linux on startup to
1674          * invalidate the caches entirely by stepping through all the
1675          * ways/indexes
1676          */
1677         if (op == Index_Writeback_Inv) {
1678                 kvm_debug("@ %#lx/%#lx CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1679                           vcpu->arch.pc, vcpu->arch.gprs[31], cache, op, base,
1680                           arch->gprs[base], offset);
1681 
1682                 if (cache == Cache_D)
1683                         r4k_blast_dcache();
1684                 else if (cache == Cache_I)
1685                         r4k_blast_icache();
1686                 else {
1687                         kvm_err("%s: unsupported CACHE INDEX operation\n",
1688                                 __func__);
1689                         return EMULATE_FAIL;
1690                 }
1691 
1692 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1693                 kvm_mips_trans_cache_index(inst, opc, vcpu);
1694 #endif
1695                 goto done;
1696         }
1697 
1698         preempt_disable();
1699         if (KVM_GUEST_KSEGX(va) == KVM_GUEST_KSEG0) {
1700                 if (kvm_mips_host_tlb_lookup(vcpu, va) < 0 &&
1701                     kvm_mips_handle_kseg0_tlb_fault(va, vcpu)) {
1702                         kvm_err("%s: handling mapped kseg0 tlb fault for %lx, vcpu: %p, ASID: %#lx\n",
1703                                 __func__, va, vcpu, read_c0_entryhi());
1704                         er = EMULATE_FAIL;
1705                         preempt_enable();
1706                         goto done;
1707                 }
1708         } else if ((KVM_GUEST_KSEGX(va) < KVM_GUEST_KSEG0) ||
1709                    KVM_GUEST_KSEGX(va) == KVM_GUEST_KSEG23) {
1710                 int index;
1711 
1712                 /* If an entry already exists then skip */
1713                 if (kvm_mips_host_tlb_lookup(vcpu, va) >= 0)
1714                         goto skip_fault;
1715 
1716                 /*
1717                  * If address not in the guest TLB, then give the guest a fault,
1718                  * the resulting handler will do the right thing
1719                  */
1720                 index = kvm_mips_guest_tlb_lookup(vcpu, (va & VPN2_MASK) |
1721                                                   (kvm_read_c0_guest_entryhi
1722                                                    (cop0) & KVM_ENTRYHI_ASID));
1723 
1724                 if (index < 0) {
1725                         vcpu->arch.host_cp0_badvaddr = va;
1726                         vcpu->arch.pc = curr_pc;
1727                         er = kvm_mips_emulate_tlbmiss_ld(cause, NULL, run,
1728                                                          vcpu);
1729                         preempt_enable();
1730                         goto dont_update_pc;
1731                 } else {
1732                         struct kvm_mips_tlb *tlb = &vcpu->arch.guest_tlb[index];
1733                         /*
1734                          * Check if the entry is valid, if not then setup a TLB
1735                          * invalid exception to the guest
1736                          */
1737                         if (!TLB_IS_VALID(*tlb, va)) {
1738                                 vcpu->arch.host_cp0_badvaddr = va;
1739                                 vcpu->arch.pc = curr_pc;
1740                                 er = kvm_mips_emulate_tlbinv_ld(cause, NULL,
1741                                                                 run, vcpu);
1742                                 preempt_enable();
1743                                 goto dont_update_pc;
1744                         }
1745                         /*
1746                          * We fault an entry from the guest tlb to the
1747                          * shadow host TLB
1748                          */
1749                         if (kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb)) {
1750                                 kvm_err("%s: handling mapped seg tlb fault for %lx, index: %u, vcpu: %p, ASID: %#lx\n",
1751                                         __func__, va, index, vcpu,
1752                                         read_c0_entryhi());
1753                                 er = EMULATE_FAIL;
1754                                 preempt_enable();
1755                                 goto done;
1756                         }
1757                 }
1758         } else {
1759                 kvm_err("INVALID CACHE INDEX/ADDRESS (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1760                         cache, op, base, arch->gprs[base], offset);
1761                 er = EMULATE_FAIL;
1762                 preempt_enable();
1763                 goto done;
1764 
1765         }
1766 
1767 skip_fault:
1768         /* XXXKYMA: Only a subset of cache ops are supported, used by Linux */
1769         if (op_inst == Hit_Writeback_Inv_D || op_inst == Hit_Invalidate_D) {
1770                 flush_dcache_line(va);
1771 
1772 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1773                 /*
1774                  * Replace the CACHE instruction, with a SYNCI, not the same,
1775                  * but avoids a trap
1776                  */
1777                 kvm_mips_trans_cache_va(inst, opc, vcpu);
1778 #endif
1779         } else if (op_inst == Hit_Invalidate_I) {
1780                 flush_dcache_line(va);
1781                 flush_icache_line(va);
1782 
1783 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1784                 /* Replace the CACHE instruction, with a SYNCI */
1785                 kvm_mips_trans_cache_va(inst, opc, vcpu);
1786 #endif
1787         } else {
1788                 kvm_err("NO-OP CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1789                         cache, op, base, arch->gprs[base], offset);
1790                 er = EMULATE_FAIL;
1791         }
1792 
1793         preempt_enable();
1794 done:
1795         /* Rollback PC only if emulation was unsuccessful */
1796         if (er == EMULATE_FAIL)
1797                 vcpu->arch.pc = curr_pc;
1798 
1799 dont_update_pc:
1800         /*
1801          * This is for exceptions whose emulation updates the PC, so do not
1802          * overwrite the PC under any circumstances
1803          */
1804 
1805         return er;
1806 }
1807 
1808 enum emulation_result kvm_mips_emulate_inst(u32 cause, u32 *opc,
1809                                             struct kvm_run *run,
1810                                             struct kvm_vcpu *vcpu)
1811 {
1812         union mips_instruction inst;
1813         enum emulation_result er = EMULATE_DONE;
1814 
1815         /* Fetch the instruction. */
1816         if (cause & CAUSEF_BD)
1817                 opc += 1;
1818 
1819         inst.word = kvm_get_inst(opc, vcpu);
1820 
1821         switch (inst.r_format.opcode) {
1822         case cop0_op:
1823                 er = kvm_mips_emulate_CP0(inst, opc, cause, run, vcpu);
1824                 break;
1825         case sb_op:
1826         case sh_op:
1827         case sw_op:
1828                 er = kvm_mips_emulate_store(inst, cause, run, vcpu);
1829                 break;
1830         case lb_op:
1831         case lbu_op:
1832         case lhu_op:
1833         case lh_op:
1834         case lw_op:
1835                 er = kvm_mips_emulate_load(inst, cause, run, vcpu);
1836                 break;
1837 
1838 #ifndef CONFIG_CPU_MIPSR6
1839         case cache_op:
1840                 ++vcpu->stat.cache_exits;
1841                 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE);
1842                 er = kvm_mips_emulate_cache(inst, opc, cause, run, vcpu);
1843                 break;
1844 #else
1845         case spec3_op:
1846                 switch (inst.spec3_format.func) {
1847                 case cache6_op:
1848                         ++vcpu->stat.cache_exits;
1849                         trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE);
1850                         er = kvm_mips_emulate_cache(inst, opc, cause, run,
1851                                                     vcpu);
1852                         break;
1853                 default:
1854                         goto unknown;
1855                 };
1856                 break;
1857 unknown:
1858 #endif
1859 
1860         default:
1861                 kvm_err("Instruction emulation not supported (%p/%#x)\n", opc,
1862                         inst.word);
1863                 kvm_arch_vcpu_dump_regs(vcpu);
1864                 er = EMULATE_FAIL;
1865                 break;
1866         }
1867 
1868         return er;
1869 }
1870 
1871 enum emulation_result kvm_mips_emulate_syscall(u32 cause,
1872                                                u32 *opc,
1873                                                struct kvm_run *run,
1874                                                struct kvm_vcpu *vcpu)
1875 {
1876         struct mips_coproc *cop0 = vcpu->arch.cop0;
1877         struct kvm_vcpu_arch *arch = &vcpu->arch;
1878         enum emulation_result er = EMULATE_DONE;
1879 
1880         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1881                 /* save old pc */
1882                 kvm_write_c0_guest_epc(cop0, arch->pc);
1883                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1884 
1885                 if (cause & CAUSEF_BD)
1886                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1887                 else
1888                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1889 
1890                 kvm_debug("Delivering SYSCALL @ pc %#lx\n", arch->pc);
1891 
1892                 kvm_change_c0_guest_cause(cop0, (0xff),
1893                                           (EXCCODE_SYS << CAUSEB_EXCCODE));
1894 
1895                 /* Set PC to the exception entry point */
1896                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1897 
1898         } else {
1899                 kvm_err("Trying to deliver SYSCALL when EXL is already set\n");
1900                 er = EMULATE_FAIL;
1901         }
1902 
1903         return er;
1904 }
1905 
1906 enum emulation_result kvm_mips_emulate_tlbmiss_ld(u32 cause,
1907                                                   u32 *opc,
1908                                                   struct kvm_run *run,
1909                                                   struct kvm_vcpu *vcpu)
1910 {
1911         struct mips_coproc *cop0 = vcpu->arch.cop0;
1912         struct kvm_vcpu_arch *arch = &vcpu->arch;
1913         unsigned long entryhi = (vcpu->arch.  host_cp0_badvaddr & VPN2_MASK) |
1914                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
1915 
1916         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1917                 /* save old pc */
1918                 kvm_write_c0_guest_epc(cop0, arch->pc);
1919                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1920 
1921                 if (cause & CAUSEF_BD)
1922                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1923                 else
1924                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1925 
1926                 kvm_debug("[EXL == 0] delivering TLB MISS @ pc %#lx\n",
1927                           arch->pc);
1928 
1929                 /* set pc to the exception entry point */
1930                 arch->pc = KVM_GUEST_KSEG0 + 0x0;
1931 
1932         } else {
1933                 kvm_debug("[EXL == 1] delivering TLB MISS @ pc %#lx\n",
1934                           arch->pc);
1935 
1936                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1937         }
1938 
1939         kvm_change_c0_guest_cause(cop0, (0xff),
1940                                   (EXCCODE_TLBL << CAUSEB_EXCCODE));
1941 
1942         /* setup badvaddr, context and entryhi registers for the guest */
1943         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
1944         /* XXXKYMA: is the context register used by linux??? */
1945         kvm_write_c0_guest_entryhi(cop0, entryhi);
1946         /* Blow away the shadow host TLBs */
1947         kvm_mips_flush_host_tlb(1);
1948 
1949         return EMULATE_DONE;
1950 }
1951 
1952 enum emulation_result kvm_mips_emulate_tlbinv_ld(u32 cause,
1953                                                  u32 *opc,
1954                                                  struct kvm_run *run,
1955                                                  struct kvm_vcpu *vcpu)
1956 {
1957         struct mips_coproc *cop0 = vcpu->arch.cop0;
1958         struct kvm_vcpu_arch *arch = &vcpu->arch;
1959         unsigned long entryhi =
1960                 (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
1961                 (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
1962 
1963         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1964                 /* save old pc */
1965                 kvm_write_c0_guest_epc(cop0, arch->pc);
1966                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1967 
1968                 if (cause & CAUSEF_BD)
1969                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1970                 else
1971                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1972 
1973                 kvm_debug("[EXL == 0] delivering TLB INV @ pc %#lx\n",
1974                           arch->pc);
1975 
1976                 /* set pc to the exception entry point */
1977                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1978 
1979         } else {
1980                 kvm_debug("[EXL == 1] delivering TLB MISS @ pc %#lx\n",
1981                           arch->pc);
1982                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1983         }
1984 
1985         kvm_change_c0_guest_cause(cop0, (0xff),
1986                                   (EXCCODE_TLBL << CAUSEB_EXCCODE));
1987 
1988         /* setup badvaddr, context and entryhi registers for the guest */
1989         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
1990         /* XXXKYMA: is the context register used by linux??? */
1991         kvm_write_c0_guest_entryhi(cop0, entryhi);
1992         /* Blow away the shadow host TLBs */
1993         kvm_mips_flush_host_tlb(1);
1994 
1995         return EMULATE_DONE;
1996 }
1997 
1998 enum emulation_result kvm_mips_emulate_tlbmiss_st(u32 cause,
1999                                                   u32 *opc,
2000                                                   struct kvm_run *run,
2001                                                   struct kvm_vcpu *vcpu)
2002 {
2003         struct mips_coproc *cop0 = vcpu->arch.cop0;
2004         struct kvm_vcpu_arch *arch = &vcpu->arch;
2005         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
2006                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
2007 
2008         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2009                 /* save old pc */
2010                 kvm_write_c0_guest_epc(cop0, arch->pc);
2011                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2012 
2013                 if (cause & CAUSEF_BD)
2014                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2015                 else
2016                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2017 
2018                 kvm_debug("[EXL == 0] Delivering TLB MISS @ pc %#lx\n",
2019                           arch->pc);
2020 
2021                 /* Set PC to the exception entry point */
2022                 arch->pc = KVM_GUEST_KSEG0 + 0x0;
2023         } else {
2024                 kvm_debug("[EXL == 1] Delivering TLB MISS @ pc %#lx\n",
2025                           arch->pc);
2026                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2027         }
2028 
2029         kvm_change_c0_guest_cause(cop0, (0xff),
2030                                   (EXCCODE_TLBS << CAUSEB_EXCCODE));
2031 
2032         /* setup badvaddr, context and entryhi registers for the guest */
2033         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2034         /* XXXKYMA: is the context register used by linux??? */
2035         kvm_write_c0_guest_entryhi(cop0, entryhi);
2036         /* Blow away the shadow host TLBs */
2037         kvm_mips_flush_host_tlb(1);
2038 
2039         return EMULATE_DONE;
2040 }
2041 
2042 enum emulation_result kvm_mips_emulate_tlbinv_st(u32 cause,
2043                                                  u32 *opc,
2044                                                  struct kvm_run *run,
2045                                                  struct kvm_vcpu *vcpu)
2046 {
2047         struct mips_coproc *cop0 = vcpu->arch.cop0;
2048         struct kvm_vcpu_arch *arch = &vcpu->arch;
2049         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
2050                 (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
2051 
2052         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2053                 /* save old pc */
2054                 kvm_write_c0_guest_epc(cop0, arch->pc);
2055                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2056 
2057                 if (cause & CAUSEF_BD)
2058                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2059                 else
2060                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2061 
2062                 kvm_debug("[EXL == 0] Delivering TLB MISS @ pc %#lx\n",
2063                           arch->pc);
2064 
2065                 /* Set PC to the exception entry point */
2066                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2067         } else {
2068                 kvm_debug("[EXL == 1] Delivering TLB MISS @ pc %#lx\n",
2069                           arch->pc);
2070                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2071         }
2072 
2073         kvm_change_c0_guest_cause(cop0, (0xff),
2074                                   (EXCCODE_TLBS << CAUSEB_EXCCODE));
2075 
2076         /* setup badvaddr, context and entryhi registers for the guest */
2077         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2078         /* XXXKYMA: is the context register used by linux??? */
2079         kvm_write_c0_guest_entryhi(cop0, entryhi);
2080         /* Blow away the shadow host TLBs */
2081         kvm_mips_flush_host_tlb(1);
2082 
2083         return EMULATE_DONE;
2084 }
2085 
2086 /* TLBMOD: store into address matching TLB with Dirty bit off */
2087 enum emulation_result kvm_mips_handle_tlbmod(u32 cause, u32 *opc,
2088                                              struct kvm_run *run,
2089                                              struct kvm_vcpu *vcpu)
2090 {
2091         enum emulation_result er = EMULATE_DONE;
2092 #ifdef DEBUG
2093         struct mips_coproc *cop0 = vcpu->arch.cop0;
2094         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
2095                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
2096         int index;
2097 
2098         /* If address not in the guest TLB, then we are in trouble */
2099         index = kvm_mips_guest_tlb_lookup(vcpu, entryhi);
2100         if (index < 0) {
2101                 /* XXXKYMA Invalidate and retry */
2102                 kvm_mips_host_tlb_inv(vcpu, vcpu->arch.host_cp0_badvaddr);
2103                 kvm_err("%s: host got TLBMOD for %#lx but entry not present in Guest TLB\n",
2104                      __func__, entryhi);
2105                 kvm_mips_dump_guest_tlbs(vcpu);
2106                 kvm_mips_dump_host_tlbs();
2107                 return EMULATE_FAIL;
2108         }
2109 #endif
2110 
2111         er = kvm_mips_emulate_tlbmod(cause, opc, run, vcpu);
2112         return er;
2113 }
2114 
2115 enum emulation_result kvm_mips_emulate_tlbmod(u32 cause,
2116                                               u32 *opc,
2117                                               struct kvm_run *run,
2118                                               struct kvm_vcpu *vcpu)
2119 {
2120         struct mips_coproc *cop0 = vcpu->arch.cop0;
2121         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
2122                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
2123         struct kvm_vcpu_arch *arch = &vcpu->arch;
2124 
2125         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2126                 /* save old pc */
2127                 kvm_write_c0_guest_epc(cop0, arch->pc);
2128                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2129 
2130                 if (cause & CAUSEF_BD)
2131                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2132                 else
2133                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2134 
2135                 kvm_debug("[EXL == 0] Delivering TLB MOD @ pc %#lx\n",
2136                           arch->pc);
2137 
2138                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2139         } else {
2140                 kvm_debug("[EXL == 1] Delivering TLB MOD @ pc %#lx\n",
2141                           arch->pc);
2142                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2143         }
2144 
2145         kvm_change_c0_guest_cause(cop0, (0xff),
2146                                   (EXCCODE_MOD << CAUSEB_EXCCODE));
2147 
2148         /* setup badvaddr, context and entryhi registers for the guest */
2149         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2150         /* XXXKYMA: is the context register used by linux??? */
2151         kvm_write_c0_guest_entryhi(cop0, entryhi);
2152         /* Blow away the shadow host TLBs */
2153         kvm_mips_flush_host_tlb(1);
2154 
2155         return EMULATE_DONE;
2156 }
2157 
2158 enum emulation_result kvm_mips_emulate_fpu_exc(u32 cause,
2159                                                u32 *opc,
2160                                                struct kvm_run *run,
2161                                                struct kvm_vcpu *vcpu)
2162 {
2163         struct mips_coproc *cop0 = vcpu->arch.cop0;
2164         struct kvm_vcpu_arch *arch = &vcpu->arch;
2165 
2166         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2167                 /* save old pc */
2168                 kvm_write_c0_guest_epc(cop0, arch->pc);
2169                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2170 
2171                 if (cause & CAUSEF_BD)
2172                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2173                 else
2174                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2175 
2176         }
2177 
2178         arch->pc = KVM_GUEST_KSEG0 + 0x180;
2179 
2180         kvm_change_c0_guest_cause(cop0, (0xff),
2181                                   (EXCCODE_CPU << CAUSEB_EXCCODE));
2182         kvm_change_c0_guest_cause(cop0, (CAUSEF_CE), (0x1 << CAUSEB_CE));
2183 
2184         return EMULATE_DONE;
2185 }
2186 
2187 enum emulation_result kvm_mips_emulate_ri_exc(u32 cause,
2188                                               u32 *opc,
2189                                               struct kvm_run *run,
2190                                               struct kvm_vcpu *vcpu)
2191 {
2192         struct mips_coproc *cop0 = vcpu->arch.cop0;
2193         struct kvm_vcpu_arch *arch = &vcpu->arch;
2194         enum emulation_result er = EMULATE_DONE;
2195 
2196         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2197                 /* save old pc */
2198                 kvm_write_c0_guest_epc(cop0, arch->pc);
2199                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2200 
2201                 if (cause & CAUSEF_BD)
2202                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2203                 else
2204                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2205 
2206                 kvm_debug("Delivering RI @ pc %#lx\n", arch->pc);
2207 
2208                 kvm_change_c0_guest_cause(cop0, (0xff),
2209                                           (EXCCODE_RI << CAUSEB_EXCCODE));
2210 
2211                 /* Set PC to the exception entry point */
2212                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2213 
2214         } else {
2215                 kvm_err("Trying to deliver RI when EXL is already set\n");
2216                 er = EMULATE_FAIL;
2217         }
2218 
2219         return er;
2220 }
2221 
2222 enum emulation_result kvm_mips_emulate_bp_exc(u32 cause,
2223                                               u32 *opc,
2224                                               struct kvm_run *run,
2225                                               struct kvm_vcpu *vcpu)
2226 {
2227         struct mips_coproc *cop0 = vcpu->arch.cop0;
2228         struct kvm_vcpu_arch *arch = &vcpu->arch;
2229         enum emulation_result er = EMULATE_DONE;
2230 
2231         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2232                 /* save old pc */
2233                 kvm_write_c0_guest_epc(cop0, arch->pc);
2234                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2235 
2236                 if (cause & CAUSEF_BD)
2237                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2238                 else
2239                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2240 
2241                 kvm_debug("Delivering BP @ pc %#lx\n", arch->pc);
2242 
2243                 kvm_change_c0_guest_cause(cop0, (0xff),
2244                                           (EXCCODE_BP << CAUSEB_EXCCODE));
2245 
2246                 /* Set PC to the exception entry point */
2247                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2248 
2249         } else {
2250                 kvm_err("Trying to deliver BP when EXL is already set\n");
2251                 er = EMULATE_FAIL;
2252         }
2253 
2254         return er;
2255 }
2256 
2257 enum emulation_result kvm_mips_emulate_trap_exc(u32 cause,
2258                                                 u32 *opc,
2259                                                 struct kvm_run *run,
2260                                                 struct kvm_vcpu *vcpu)
2261 {
2262         struct mips_coproc *cop0 = vcpu->arch.cop0;
2263         struct kvm_vcpu_arch *arch = &vcpu->arch;
2264         enum emulation_result er = EMULATE_DONE;
2265 
2266         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2267                 /* save old pc */
2268                 kvm_write_c0_guest_epc(cop0, arch->pc);
2269                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2270 
2271                 if (cause & CAUSEF_BD)
2272                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2273                 else
2274                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2275 
2276                 kvm_debug("Delivering TRAP @ pc %#lx\n", arch->pc);
2277 
2278                 kvm_change_c0_guest_cause(cop0, (0xff),
2279                                           (EXCCODE_TR << CAUSEB_EXCCODE));
2280 
2281                 /* Set PC to the exception entry point */
2282                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2283 
2284         } else {
2285                 kvm_err("Trying to deliver TRAP when EXL is already set\n");
2286                 er = EMULATE_FAIL;
2287         }
2288 
2289         return er;
2290 }
2291 
2292 enum emulation_result kvm_mips_emulate_msafpe_exc(u32 cause,
2293                                                   u32 *opc,
2294                                                   struct kvm_run *run,
2295                                                   struct kvm_vcpu *vcpu)
2296 {
2297         struct mips_coproc *cop0 = vcpu->arch.cop0;
2298         struct kvm_vcpu_arch *arch = &vcpu->arch;
2299         enum emulation_result er = EMULATE_DONE;
2300 
2301         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2302                 /* save old pc */
2303                 kvm_write_c0_guest_epc(cop0, arch->pc);
2304                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2305 
2306                 if (cause & CAUSEF_BD)
2307                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2308                 else
2309                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2310 
2311                 kvm_debug("Delivering MSAFPE @ pc %#lx\n", arch->pc);
2312 
2313                 kvm_change_c0_guest_cause(cop0, (0xff),
2314                                           (EXCCODE_MSAFPE << CAUSEB_EXCCODE));
2315 
2316                 /* Set PC to the exception entry point */
2317                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2318 
2319         } else {
2320                 kvm_err("Trying to deliver MSAFPE when EXL is already set\n");
2321                 er = EMULATE_FAIL;
2322         }
2323 
2324         return er;
2325 }
2326 
2327 enum emulation_result kvm_mips_emulate_fpe_exc(u32 cause,
2328                                                u32 *opc,
2329                                                struct kvm_run *run,
2330                                                struct kvm_vcpu *vcpu)
2331 {
2332         struct mips_coproc *cop0 = vcpu->arch.cop0;
2333         struct kvm_vcpu_arch *arch = &vcpu->arch;
2334         enum emulation_result er = EMULATE_DONE;
2335 
2336         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2337                 /* save old pc */
2338                 kvm_write_c0_guest_epc(cop0, arch->pc);
2339                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2340 
2341                 if (cause & CAUSEF_BD)
2342                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2343                 else
2344                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2345 
2346                 kvm_debug("Delivering FPE @ pc %#lx\n", arch->pc);
2347 
2348                 kvm_change_c0_guest_cause(cop0, (0xff),
2349                                           (EXCCODE_FPE << CAUSEB_EXCCODE));
2350 
2351                 /* Set PC to the exception entry point */
2352                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2353 
2354         } else {
2355                 kvm_err("Trying to deliver FPE when EXL is already set\n");
2356                 er = EMULATE_FAIL;
2357         }
2358 
2359         return er;
2360 }
2361 
2362 enum emulation_result kvm_mips_emulate_msadis_exc(u32 cause,
2363                                                   u32 *opc,
2364                                                   struct kvm_run *run,
2365                                                   struct kvm_vcpu *vcpu)
2366 {
2367         struct mips_coproc *cop0 = vcpu->arch.cop0;
2368         struct kvm_vcpu_arch *arch = &vcpu->arch;
2369         enum emulation_result er = EMULATE_DONE;
2370 
2371         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2372                 /* save old pc */
2373                 kvm_write_c0_guest_epc(cop0, arch->pc);
2374                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2375 
2376                 if (cause & CAUSEF_BD)
2377                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2378                 else
2379                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2380 
2381                 kvm_debug("Delivering MSADIS @ pc %#lx\n", arch->pc);
2382 
2383                 kvm_change_c0_guest_cause(cop0, (0xff),
2384                                           (EXCCODE_MSADIS << CAUSEB_EXCCODE));
2385 
2386                 /* Set PC to the exception entry point */
2387                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2388 
2389         } else {
2390                 kvm_err("Trying to deliver MSADIS when EXL is already set\n");
2391                 er = EMULATE_FAIL;
2392         }
2393 
2394         return er;
2395 }
2396 
2397 enum emulation_result kvm_mips_handle_ri(u32 cause, u32 *opc,
2398                                          struct kvm_run *run,
2399                                          struct kvm_vcpu *vcpu)
2400 {
2401         struct mips_coproc *cop0 = vcpu->arch.cop0;
2402         struct kvm_vcpu_arch *arch = &vcpu->arch;
2403         enum emulation_result er = EMULATE_DONE;
2404         unsigned long curr_pc;
2405         union mips_instruction inst;
2406 
2407         /*
2408          * Update PC and hold onto current PC in case there is
2409          * an error and we want to rollback the PC
2410          */
2411         curr_pc = vcpu->arch.pc;
2412         er = update_pc(vcpu, cause);
2413         if (er == EMULATE_FAIL)
2414                 return er;
2415 
2416         /* Fetch the instruction. */
2417         if (cause & CAUSEF_BD)
2418                 opc += 1;
2419 
2420         inst.word = kvm_get_inst(opc, vcpu);
2421 
2422         if (inst.word == KVM_INVALID_INST) {
2423                 kvm_err("%s: Cannot get inst @ %p\n", __func__, opc);
2424                 return EMULATE_FAIL;
2425         }
2426 
2427         if (inst.r_format.opcode == spec3_op &&
2428             inst.r_format.func == rdhwr_op &&
2429             inst.r_format.rs == 0 &&
2430             (inst.r_format.re >> 3) == 0) {
2431                 int usermode = !KVM_GUEST_KERNEL_MODE(vcpu);
2432                 int rd = inst.r_format.rd;
2433                 int rt = inst.r_format.rt;
2434                 int sel = inst.r_format.re & 0x7;
2435 
2436                 /* If usermode, check RDHWR rd is allowed by guest HWREna */
2437                 if (usermode && !(kvm_read_c0_guest_hwrena(cop0) & BIT(rd))) {
2438                         kvm_debug("RDHWR %#x disallowed by HWREna @ %p\n",
2439                                   rd, opc);
2440                         goto emulate_ri;
2441                 }
2442                 switch (rd) {
2443                 case MIPS_HWR_CPUNUM:           /* CPU number */
2444                         arch->gprs[rt] = vcpu->vcpu_id;
2445                         break;
2446                 case MIPS_HWR_SYNCISTEP:        /* SYNCI length */
2447                         arch->gprs[rt] = min(current_cpu_data.dcache.linesz,
2448                                              current_cpu_data.icache.linesz);
2449                         break;
2450                 case MIPS_HWR_CC:               /* Read count register */
2451                         arch->gprs[rt] = (s32)kvm_mips_read_count(vcpu);
2452                         break;
2453                 case MIPS_HWR_CCRES:            /* Count register resolution */
2454                         switch (current_cpu_data.cputype) {
2455                         case CPU_20KC:
2456                         case CPU_25KF:
2457                                 arch->gprs[rt] = 1;
2458                                 break;
2459                         default:
2460                                 arch->gprs[rt] = 2;
2461                         }
2462                         break;
2463                 case MIPS_HWR_ULR:              /* Read UserLocal register */
2464                         arch->gprs[rt] = kvm_read_c0_guest_userlocal(cop0);
2465                         break;
2466 
2467                 default:
2468                         kvm_debug("RDHWR %#x not supported @ %p\n", rd, opc);
2469                         goto emulate_ri;
2470                 }
2471 
2472                 trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR, KVM_TRACE_HWR(rd, sel),
2473                               vcpu->arch.gprs[rt]);
2474         } else {
2475                 kvm_debug("Emulate RI not supported @ %p: %#x\n",
2476                           opc, inst.word);
2477                 goto emulate_ri;
2478         }
2479 
2480         return EMULATE_DONE;
2481 
2482 emulate_ri:
2483         /*
2484          * Rollback PC (if in branch delay slot then the PC already points to
2485          * branch target), and pass the RI exception to the guest OS.
2486          */
2487         vcpu->arch.pc = curr_pc;
2488         return kvm_mips_emulate_ri_exc(cause, opc, run, vcpu);
2489 }
2490 
2491 enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu,
2492                                                   struct kvm_run *run)
2493 {
2494         unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr];
2495         enum emulation_result er = EMULATE_DONE;
2496 
2497         if (run->mmio.len > sizeof(*gpr)) {
2498                 kvm_err("Bad MMIO length: %d", run->mmio.len);
2499                 er = EMULATE_FAIL;
2500                 goto done;
2501         }
2502 
2503         /* Restore saved resume PC */
2504         vcpu->arch.pc = vcpu->arch.io_pc;
2505 
2506         switch (run->mmio.len) {
2507         case 4:
2508                 *gpr = *(s32 *) run->mmio.data;
2509                 break;
2510 
2511         case 2:
2512                 if (vcpu->mmio_needed == 2)
2513                         *gpr = *(s16 *) run->mmio.data;
2514                 else
2515                         *gpr = *(u16 *)run->mmio.data;
2516 
2517                 break;
2518         case 1:
2519                 if (vcpu->mmio_needed == 2)
2520                         *gpr = *(s8 *) run->mmio.data;
2521                 else
2522                         *gpr = *(u8 *) run->mmio.data;
2523                 break;
2524         }
2525 
2526 done:
2527         return er;
2528 }
2529 
2530 static enum emulation_result kvm_mips_emulate_exc(u32 cause,
2531                                                   u32 *opc,
2532                                                   struct kvm_run *run,
2533                                                   struct kvm_vcpu *vcpu)
2534 {
2535         u32 exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
2536         struct mips_coproc *cop0 = vcpu->arch.cop0;
2537         struct kvm_vcpu_arch *arch = &vcpu->arch;
2538         enum emulation_result er = EMULATE_DONE;
2539 
2540         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2541                 /* save old pc */
2542                 kvm_write_c0_guest_epc(cop0, arch->pc);
2543                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2544 
2545                 if (cause & CAUSEF_BD)
2546                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2547                 else
2548                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2549 
2550                 kvm_change_c0_guest_cause(cop0, (0xff),
2551                                           (exccode << CAUSEB_EXCCODE));
2552 
2553                 /* Set PC to the exception entry point */
2554                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2555                 kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2556 
2557                 kvm_debug("Delivering EXC %d @ pc %#lx, badVaddr: %#lx\n",
2558                           exccode, kvm_read_c0_guest_epc(cop0),
2559                           kvm_read_c0_guest_badvaddr(cop0));
2560         } else {
2561                 kvm_err("Trying to deliver EXC when EXL is already set\n");
2562                 er = EMULATE_FAIL;
2563         }
2564 
2565         return er;
2566 }
2567 
2568 enum emulation_result kvm_mips_check_privilege(u32 cause,
2569                                                u32 *opc,
2570                                                struct kvm_run *run,
2571                                                struct kvm_vcpu *vcpu)
2572 {
2573         enum emulation_result er = EMULATE_DONE;
2574         u32 exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
2575         unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr;
2576 
2577         int usermode = !KVM_GUEST_KERNEL_MODE(vcpu);
2578 
2579         if (usermode) {
2580                 switch (exccode) {
2581                 case EXCCODE_INT:
2582                 case EXCCODE_SYS:
2583                 case EXCCODE_BP:
2584                 case EXCCODE_RI:
2585                 case EXCCODE_TR:
2586                 case EXCCODE_MSAFPE:
2587                 case EXCCODE_FPE:
2588                 case EXCCODE_MSADIS:
2589                         break;
2590 
2591                 case EXCCODE_CPU:
2592                         if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 0)
2593                                 er = EMULATE_PRIV_FAIL;
2594                         break;
2595 
2596                 case EXCCODE_MOD:
2597                         break;
2598 
2599                 case EXCCODE_TLBL:
2600                         /*
2601                          * We we are accessing Guest kernel space, then send an
2602                          * address error exception to the guest
2603                          */
2604                         if (badvaddr >= (unsigned long) KVM_GUEST_KSEG0) {
2605                                 kvm_debug("%s: LD MISS @ %#lx\n", __func__,
2606                                           badvaddr);
2607                                 cause &= ~0xff;
2608                                 cause |= (EXCCODE_ADEL << CAUSEB_EXCCODE);
2609                                 er = EMULATE_PRIV_FAIL;
2610                         }
2611                         break;
2612 
2613                 case EXCCODE_TLBS:
2614                         /*
2615                          * We we are accessing Guest kernel space, then send an
2616                          * address error exception to the guest
2617                          */
2618                         if (badvaddr >= (unsigned long) KVM_GUEST_KSEG0) {
2619                                 kvm_debug("%s: ST MISS @ %#lx\n", __func__,
2620                                           badvaddr);
2621                                 cause &= ~0xff;
2622                                 cause |= (EXCCODE_ADES << CAUSEB_EXCCODE);
2623                                 er = EMULATE_PRIV_FAIL;
2624                         }
2625                         break;
2626 
2627                 case EXCCODE_ADES:
2628                         kvm_debug("%s: address error ST @ %#lx\n", __func__,
2629                                   badvaddr);
2630                         if ((badvaddr & PAGE_MASK) == KVM_GUEST_COMMPAGE_ADDR) {
2631                                 cause &= ~0xff;
2632                                 cause |= (EXCCODE_TLBS << CAUSEB_EXCCODE);
2633                         }
2634                         er = EMULATE_PRIV_FAIL;
2635                         break;
2636                 case EXCCODE_ADEL:
2637                         kvm_debug("%s: address error LD @ %#lx\n", __func__,
2638                                   badvaddr);
2639                         if ((badvaddr & PAGE_MASK) == KVM_GUEST_COMMPAGE_ADDR) {
2640                                 cause &= ~0xff;
2641                                 cause |= (EXCCODE_TLBL << CAUSEB_EXCCODE);
2642                         }
2643                         er = EMULATE_PRIV_FAIL;
2644                         break;
2645                 default:
2646                         er = EMULATE_PRIV_FAIL;
2647                         break;
2648                 }
2649         }
2650 
2651         if (er == EMULATE_PRIV_FAIL)
2652                 kvm_mips_emulate_exc(cause, opc, run, vcpu);
2653 
2654         return er;
2655 }
2656 
2657 /*
2658  * User Address (UA) fault, this could happen if
2659  * (1) TLB entry not present/valid in both Guest and shadow host TLBs, in this
2660  *     case we pass on the fault to the guest kernel and let it handle it.
2661  * (2) TLB entry is present in the Guest TLB but not in the shadow, in this
2662  *     case we inject the TLB from the Guest TLB into the shadow host TLB
2663  */
2664 enum emulation_result kvm_mips_handle_tlbmiss(u32 cause,
2665                                               u32 *opc,
2666                                               struct kvm_run *run,
2667                                               struct kvm_vcpu *vcpu)
2668 {
2669         enum emulation_result er = EMULATE_DONE;
2670         u32 exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
2671         unsigned long va = vcpu->arch.host_cp0_badvaddr;
2672         int index;
2673 
2674         kvm_debug("kvm_mips_handle_tlbmiss: badvaddr: %#lx\n",
2675                   vcpu->arch.host_cp0_badvaddr);
2676 
2677         /*
2678          * KVM would not have got the exception if this entry was valid in the
2679          * shadow host TLB. Check the Guest TLB, if the entry is not there then
2680          * send the guest an exception. The guest exc handler should then inject
2681          * an entry into the guest TLB.
2682          */
2683         index = kvm_mips_guest_tlb_lookup(vcpu,
2684                       (va & VPN2_MASK) |
2685                       (kvm_read_c0_guest_entryhi(vcpu->arch.cop0) &
2686                        KVM_ENTRYHI_ASID));
2687         if (index < 0) {
2688                 if (exccode == EXCCODE_TLBL) {
2689                         er = kvm_mips_emulate_tlbmiss_ld(cause, opc, run, vcpu);
2690                 } else if (exccode == EXCCODE_TLBS) {
2691                         er = kvm_mips_emulate_tlbmiss_st(cause, opc, run, vcpu);
2692                 } else {
2693                         kvm_err("%s: invalid exc code: %d\n", __func__,
2694                                 exccode);
2695                         er = EMULATE_FAIL;
2696                 }
2697         } else {
2698                 struct kvm_mips_tlb *tlb = &vcpu->arch.guest_tlb[index];
2699 
2700                 /*
2701                  * Check if the entry is valid, if not then setup a TLB invalid
2702                  * exception to the guest
2703                  */
2704                 if (!TLB_IS_VALID(*tlb, va)) {
2705                         if (exccode == EXCCODE_TLBL) {
2706                                 er = kvm_mips_emulate_tlbinv_ld(cause, opc, run,
2707                                                                 vcpu);
2708                         } else if (exccode == EXCCODE_TLBS) {
2709                                 er = kvm_mips_emulate_tlbinv_st(cause, opc, run,
2710                                                                 vcpu);
2711                         } else {
2712                                 kvm_err("%s: invalid exc code: %d\n", __func__,
2713                                         exccode);
2714                                 er = EMULATE_FAIL;
2715                         }
2716                 } else {
2717                         kvm_debug("Injecting hi: %#lx, lo0: %#lx, lo1: %#lx into shadow host TLB\n",
2718                                   tlb->tlb_hi, tlb->tlb_lo[0], tlb->tlb_lo[1]);
2719                         /*
2720                          * OK we have a Guest TLB entry, now inject it into the
2721                          * shadow host TLB
2722                          */
2723                         if (kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb)) {
2724                                 kvm_err("%s: handling mapped seg tlb fault for %lx, index: %u, vcpu: %p, ASID: %#lx\n",
2725                                         __func__, va, index, vcpu,
2726                                         read_c0_entryhi());
2727                                 er = EMULATE_FAIL;
2728                         }
2729                 }
2730         }
2731 
2732         return er;
2733 }
2734 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp