~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/powerpc/kvm/book3s_hv_rm_xics.c

Version: ~ [ linux-5.11-rc3 ] ~ [ linux-5.10.7 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.89 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.167 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.215 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.251 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.251 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Copyright 2012 Michael Ellerman, IBM Corporation.
  3  * Copyright 2012 Benjamin Herrenschmidt, IBM Corporation
  4  *
  5  * This program is free software; you can redistribute it and/or modify
  6  * it under the terms of the GNU General Public License, version 2, as
  7  * published by the Free Software Foundation.
  8  */
  9 
 10 #include <linux/kernel.h>
 11 #include <linux/kvm_host.h>
 12 #include <linux/err.h>
 13 #include <linux/kernel_stat.h>
 14 
 15 #include <asm/kvm_book3s.h>
 16 #include <asm/kvm_ppc.h>
 17 #include <asm/hvcall.h>
 18 #include <asm/xics.h>
 19 #include <asm/synch.h>
 20 #include <asm/cputhreads.h>
 21 #include <asm/pgtable.h>
 22 #include <asm/ppc-opcode.h>
 23 #include <asm/pnv-pci.h>
 24 #include <asm/opal.h>
 25 #include <asm/smp.h>
 26 
 27 #include "book3s_xics.h"
 28 
 29 #define DEBUG_PASSUP
 30 
 31 int h_ipi_redirect = 1;
 32 EXPORT_SYMBOL(h_ipi_redirect);
 33 int kvm_irq_bypass = 1;
 34 EXPORT_SYMBOL(kvm_irq_bypass);
 35 
 36 static void icp_rm_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
 37                             u32 new_irq, bool check_resend);
 38 static int xics_opal_set_server(unsigned int hw_irq, int server_cpu);
 39 
 40 /* -- ICS routines -- */
 41 static void ics_rm_check_resend(struct kvmppc_xics *xics,
 42                                 struct kvmppc_ics *ics, struct kvmppc_icp *icp)
 43 {
 44         int i;
 45 
 46         for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
 47                 struct ics_irq_state *state = &ics->irq_state[i];
 48                 if (state->resend)
 49                         icp_rm_deliver_irq(xics, icp, state->number, true);
 50         }
 51 
 52 }
 53 
 54 /* -- ICP routines -- */
 55 
 56 #ifdef CONFIG_SMP
 57 static inline void icp_send_hcore_msg(int hcore, struct kvm_vcpu *vcpu)
 58 {
 59         int hcpu;
 60 
 61         hcpu = hcore << threads_shift;
 62         kvmppc_host_rm_ops_hv->rm_core[hcore].rm_data = vcpu;
 63         smp_muxed_ipi_set_message(hcpu, PPC_MSG_RM_HOST_ACTION);
 64         kvmppc_set_host_ipi(hcpu, 1);
 65         smp_mb();
 66         kvmhv_rm_send_ipi(hcpu);
 67 }
 68 #else
 69 static inline void icp_send_hcore_msg(int hcore, struct kvm_vcpu *vcpu) { }
 70 #endif
 71 
 72 /*
 73  * We start the search from our current CPU Id in the core map
 74  * and go in a circle until we get back to our ID looking for a
 75  * core that is running in host context and that hasn't already
 76  * been targeted for another rm_host_ops.
 77  *
 78  * In the future, could consider using a fairer algorithm (one
 79  * that distributes the IPIs better)
 80  *
 81  * Returns -1, if no CPU could be found in the host
 82  * Else, returns a CPU Id which has been reserved for use
 83  */
 84 static inline int grab_next_hostcore(int start,
 85                 struct kvmppc_host_rm_core *rm_core, int max, int action)
 86 {
 87         bool success;
 88         int core;
 89         union kvmppc_rm_state old, new;
 90 
 91         for (core = start + 1; core < max; core++)  {
 92                 old = new = READ_ONCE(rm_core[core].rm_state);
 93 
 94                 if (!old.in_host || old.rm_action)
 95                         continue;
 96 
 97                 /* Try to grab this host core if not taken already. */
 98                 new.rm_action = action;
 99 
100                 success = cmpxchg64(&rm_core[core].rm_state.raw,
101                                                 old.raw, new.raw) == old.raw;
102                 if (success) {
103                         /*
104                          * Make sure that the store to the rm_action is made
105                          * visible before we return to caller (and the
106                          * subsequent store to rm_data) to synchronize with
107                          * the IPI handler.
108                          */
109                         smp_wmb();
110                         return core;
111                 }
112         }
113 
114         return -1;
115 }
116 
117 static inline int find_available_hostcore(int action)
118 {
119         int core;
120         int my_core = smp_processor_id() >> threads_shift;
121         struct kvmppc_host_rm_core *rm_core = kvmppc_host_rm_ops_hv->rm_core;
122 
123         core = grab_next_hostcore(my_core, rm_core, cpu_nr_cores(), action);
124         if (core == -1)
125                 core = grab_next_hostcore(core, rm_core, my_core, action);
126 
127         return core;
128 }
129 
130 static void icp_rm_set_vcpu_irq(struct kvm_vcpu *vcpu,
131                                 struct kvm_vcpu *this_vcpu)
132 {
133         struct kvmppc_icp *this_icp = this_vcpu->arch.icp;
134         int cpu;
135         int hcore;
136 
137         /* Mark the target VCPU as having an interrupt pending */
138         vcpu->stat.queue_intr++;
139         set_bit(BOOK3S_IRQPRIO_EXTERNAL_LEVEL, &vcpu->arch.pending_exceptions);
140 
141         /* Kick self ? Just set MER and return */
142         if (vcpu == this_vcpu) {
143                 mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) | LPCR_MER);
144                 return;
145         }
146 
147         /*
148          * Check if the core is loaded,
149          * if not, find an available host core to post to wake the VCPU,
150          * if we can't find one, set up state to eventually return too hard.
151          */
152         cpu = vcpu->arch.thread_cpu;
153         if (cpu < 0 || cpu >= nr_cpu_ids) {
154                 hcore = -1;
155                 if (kvmppc_host_rm_ops_hv && h_ipi_redirect)
156                         hcore = find_available_hostcore(XICS_RM_KICK_VCPU);
157                 if (hcore != -1) {
158                         icp_send_hcore_msg(hcore, vcpu);
159                 } else {
160                         this_icp->rm_action |= XICS_RM_KICK_VCPU;
161                         this_icp->rm_kick_target = vcpu;
162                 }
163                 return;
164         }
165 
166         smp_mb();
167         kvmhv_rm_send_ipi(cpu);
168 }
169 
170 static void icp_rm_clr_vcpu_irq(struct kvm_vcpu *vcpu)
171 {
172         /* Note: Only called on self ! */
173         clear_bit(BOOK3S_IRQPRIO_EXTERNAL_LEVEL,
174                   &vcpu->arch.pending_exceptions);
175         mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) & ~LPCR_MER);
176 }
177 
178 static inline bool icp_rm_try_update(struct kvmppc_icp *icp,
179                                      union kvmppc_icp_state old,
180                                      union kvmppc_icp_state new)
181 {
182         struct kvm_vcpu *this_vcpu = local_paca->kvm_hstate.kvm_vcpu;
183         bool success;
184 
185         /* Calculate new output value */
186         new.out_ee = (new.xisr && (new.pending_pri < new.cppr));
187 
188         /* Attempt atomic update */
189         success = cmpxchg64(&icp->state.raw, old.raw, new.raw) == old.raw;
190         if (!success)
191                 goto bail;
192 
193         /*
194          * Check for output state update
195          *
196          * Note that this is racy since another processor could be updating
197          * the state already. This is why we never clear the interrupt output
198          * here, we only ever set it. The clear only happens prior to doing
199          * an update and only by the processor itself. Currently we do it
200          * in Accept (H_XIRR) and Up_Cppr (H_XPPR).
201          *
202          * We also do not try to figure out whether the EE state has changed,
203          * we unconditionally set it if the new state calls for it. The reason
204          * for that is that we opportunistically remove the pending interrupt
205          * flag when raising CPPR, so we need to set it back here if an
206          * interrupt is still pending.
207          */
208         if (new.out_ee)
209                 icp_rm_set_vcpu_irq(icp->vcpu, this_vcpu);
210 
211         /* Expose the state change for debug purposes */
212         this_vcpu->arch.icp->rm_dbgstate = new;
213         this_vcpu->arch.icp->rm_dbgtgt = icp->vcpu;
214 
215  bail:
216         return success;
217 }
218 
219 static inline int check_too_hard(struct kvmppc_xics *xics,
220                                  struct kvmppc_icp *icp)
221 {
222         return (xics->real_mode_dbg || icp->rm_action) ? H_TOO_HARD : H_SUCCESS;
223 }
224 
225 static void icp_rm_check_resend(struct kvmppc_xics *xics,
226                              struct kvmppc_icp *icp)
227 {
228         u32 icsid;
229 
230         /* Order this load with the test for need_resend in the caller */
231         smp_rmb();
232         for_each_set_bit(icsid, icp->resend_map, xics->max_icsid + 1) {
233                 struct kvmppc_ics *ics = xics->ics[icsid];
234 
235                 if (!test_and_clear_bit(icsid, icp->resend_map))
236                         continue;
237                 if (!ics)
238                         continue;
239                 ics_rm_check_resend(xics, ics, icp);
240         }
241 }
242 
243 static bool icp_rm_try_to_deliver(struct kvmppc_icp *icp, u32 irq, u8 priority,
244                                u32 *reject)
245 {
246         union kvmppc_icp_state old_state, new_state;
247         bool success;
248 
249         do {
250                 old_state = new_state = READ_ONCE(icp->state);
251 
252                 *reject = 0;
253 
254                 /* See if we can deliver */
255                 success = new_state.cppr > priority &&
256                         new_state.mfrr > priority &&
257                         new_state.pending_pri > priority;
258 
259                 /*
260                  * If we can, check for a rejection and perform the
261                  * delivery
262                  */
263                 if (success) {
264                         *reject = new_state.xisr;
265                         new_state.xisr = irq;
266                         new_state.pending_pri = priority;
267                 } else {
268                         /*
269                          * If we failed to deliver we set need_resend
270                          * so a subsequent CPPR state change causes us
271                          * to try a new delivery.
272                          */
273                         new_state.need_resend = true;
274                 }
275 
276         } while (!icp_rm_try_update(icp, old_state, new_state));
277 
278         return success;
279 }
280 
281 static void icp_rm_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
282                             u32 new_irq, bool check_resend)
283 {
284         struct ics_irq_state *state;
285         struct kvmppc_ics *ics;
286         u32 reject;
287         u16 src;
288 
289         /*
290          * This is used both for initial delivery of an interrupt and
291          * for subsequent rejection.
292          *
293          * Rejection can be racy vs. resends. We have evaluated the
294          * rejection in an atomic ICP transaction which is now complete,
295          * so potentially the ICP can already accept the interrupt again.
296          *
297          * So we need to retry the delivery. Essentially the reject path
298          * boils down to a failed delivery. Always.
299          *
300          * Now the interrupt could also have moved to a different target,
301          * thus we may need to re-do the ICP lookup as well
302          */
303 
304  again:
305         /* Get the ICS state and lock it */
306         ics = kvmppc_xics_find_ics(xics, new_irq, &src);
307         if (!ics) {
308                 /* Unsafe increment, but this does not need to be accurate */
309                 xics->err_noics++;
310                 return;
311         }
312         state = &ics->irq_state[src];
313 
314         /* Get a lock on the ICS */
315         arch_spin_lock(&ics->lock);
316 
317         /* Get our server */
318         if (!icp || state->server != icp->server_num) {
319                 icp = kvmppc_xics_find_server(xics->kvm, state->server);
320                 if (!icp) {
321                         /* Unsafe increment again*/
322                         xics->err_noicp++;
323                         goto out;
324                 }
325         }
326 
327         if (check_resend)
328                 if (!state->resend)
329                         goto out;
330 
331         /* Clear the resend bit of that interrupt */
332         state->resend = 0;
333 
334         /*
335          * If masked, bail out
336          *
337          * Note: PAPR doesn't mention anything about masked pending
338          * when doing a resend, only when doing a delivery.
339          *
340          * However that would have the effect of losing a masked
341          * interrupt that was rejected and isn't consistent with
342          * the whole masked_pending business which is about not
343          * losing interrupts that occur while masked.
344          *
345          * I don't differentiate normal deliveries and resends, this
346          * implementation will differ from PAPR and not lose such
347          * interrupts.
348          */
349         if (state->priority == MASKED) {
350                 state->masked_pending = 1;
351                 goto out;
352         }
353 
354         /*
355          * Try the delivery, this will set the need_resend flag
356          * in the ICP as part of the atomic transaction if the
357          * delivery is not possible.
358          *
359          * Note that if successful, the new delivery might have itself
360          * rejected an interrupt that was "delivered" before we took the
361          * ics spin lock.
362          *
363          * In this case we do the whole sequence all over again for the
364          * new guy. We cannot assume that the rejected interrupt is less
365          * favored than the new one, and thus doesn't need to be delivered,
366          * because by the time we exit icp_rm_try_to_deliver() the target
367          * processor may well have already consumed & completed it, and thus
368          * the rejected interrupt might actually be already acceptable.
369          */
370         if (icp_rm_try_to_deliver(icp, new_irq, state->priority, &reject)) {
371                 /*
372                  * Delivery was successful, did we reject somebody else ?
373                  */
374                 if (reject && reject != XICS_IPI) {
375                         arch_spin_unlock(&ics->lock);
376                         icp->n_reject++;
377                         new_irq = reject;
378                         check_resend = 0;
379                         goto again;
380                 }
381         } else {
382                 /*
383                  * We failed to deliver the interrupt we need to set the
384                  * resend map bit and mark the ICS state as needing a resend
385                  */
386                 state->resend = 1;
387 
388                 /*
389                  * Make sure when checking resend, we don't miss the resend
390                  * if resend_map bit is seen and cleared.
391                  */
392                 smp_wmb();
393                 set_bit(ics->icsid, icp->resend_map);
394 
395                 /*
396                  * If the need_resend flag got cleared in the ICP some time
397                  * between icp_rm_try_to_deliver() atomic update and now, then
398                  * we know it might have missed the resend_map bit. So we
399                  * retry
400                  */
401                 smp_mb();
402                 if (!icp->state.need_resend) {
403                         state->resend = 0;
404                         arch_spin_unlock(&ics->lock);
405                         check_resend = 0;
406                         goto again;
407                 }
408         }
409  out:
410         arch_spin_unlock(&ics->lock);
411 }
412 
413 static void icp_rm_down_cppr(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
414                              u8 new_cppr)
415 {
416         union kvmppc_icp_state old_state, new_state;
417         bool resend;
418 
419         /*
420          * This handles several related states in one operation:
421          *
422          * ICP State: Down_CPPR
423          *
424          * Load CPPR with new value and if the XISR is 0
425          * then check for resends:
426          *
427          * ICP State: Resend
428          *
429          * If MFRR is more favored than CPPR, check for IPIs
430          * and notify ICS of a potential resend. This is done
431          * asynchronously (when used in real mode, we will have
432          * to exit here).
433          *
434          * We do not handle the complete Check_IPI as documented
435          * here. In the PAPR, this state will be used for both
436          * Set_MFRR and Down_CPPR. However, we know that we aren't
437          * changing the MFRR state here so we don't need to handle
438          * the case of an MFRR causing a reject of a pending irq,
439          * this will have been handled when the MFRR was set in the
440          * first place.
441          *
442          * Thus we don't have to handle rejects, only resends.
443          *
444          * When implementing real mode for HV KVM, resend will lead to
445          * a H_TOO_HARD return and the whole transaction will be handled
446          * in virtual mode.
447          */
448         do {
449                 old_state = new_state = READ_ONCE(icp->state);
450 
451                 /* Down_CPPR */
452                 new_state.cppr = new_cppr;
453 
454                 /*
455                  * Cut down Resend / Check_IPI / IPI
456                  *
457                  * The logic is that we cannot have a pending interrupt
458                  * trumped by an IPI at this point (see above), so we
459                  * know that either the pending interrupt is already an
460                  * IPI (in which case we don't care to override it) or
461                  * it's either more favored than us or non existent
462                  */
463                 if (new_state.mfrr < new_cppr &&
464                     new_state.mfrr <= new_state.pending_pri) {
465                         new_state.pending_pri = new_state.mfrr;
466                         new_state.xisr = XICS_IPI;
467                 }
468 
469                 /* Latch/clear resend bit */
470                 resend = new_state.need_resend;
471                 new_state.need_resend = 0;
472 
473         } while (!icp_rm_try_update(icp, old_state, new_state));
474 
475         /*
476          * Now handle resend checks. Those are asynchronous to the ICP
477          * state update in HW (ie bus transactions) so we can handle them
478          * separately here as well.
479          */
480         if (resend) {
481                 icp->n_check_resend++;
482                 icp_rm_check_resend(xics, icp);
483         }
484 }
485 
486 
487 unsigned long xics_rm_h_xirr(struct kvm_vcpu *vcpu)
488 {
489         union kvmppc_icp_state old_state, new_state;
490         struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
491         struct kvmppc_icp *icp = vcpu->arch.icp;
492         u32 xirr;
493 
494         if (!xics || !xics->real_mode)
495                 return H_TOO_HARD;
496 
497         /* First clear the interrupt */
498         icp_rm_clr_vcpu_irq(icp->vcpu);
499 
500         /*
501          * ICP State: Accept_Interrupt
502          *
503          * Return the pending interrupt (if any) along with the
504          * current CPPR, then clear the XISR & set CPPR to the
505          * pending priority
506          */
507         do {
508                 old_state = new_state = READ_ONCE(icp->state);
509 
510                 xirr = old_state.xisr | (((u32)old_state.cppr) << 24);
511                 if (!old_state.xisr)
512                         break;
513                 new_state.cppr = new_state.pending_pri;
514                 new_state.pending_pri = 0xff;
515                 new_state.xisr = 0;
516 
517         } while (!icp_rm_try_update(icp, old_state, new_state));
518 
519         /* Return the result in GPR4 */
520         vcpu->arch.gpr[4] = xirr;
521 
522         return check_too_hard(xics, icp);
523 }
524 
525 int xics_rm_h_ipi(struct kvm_vcpu *vcpu, unsigned long server,
526                   unsigned long mfrr)
527 {
528         union kvmppc_icp_state old_state, new_state;
529         struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
530         struct kvmppc_icp *icp, *this_icp = vcpu->arch.icp;
531         u32 reject;
532         bool resend;
533         bool local;
534 
535         if (!xics || !xics->real_mode)
536                 return H_TOO_HARD;
537 
538         local = this_icp->server_num == server;
539         if (local)
540                 icp = this_icp;
541         else
542                 icp = kvmppc_xics_find_server(vcpu->kvm, server);
543         if (!icp)
544                 return H_PARAMETER;
545 
546         /*
547          * ICP state: Set_MFRR
548          *
549          * If the CPPR is more favored than the new MFRR, then
550          * nothing needs to be done as there can be no XISR to
551          * reject.
552          *
553          * ICP state: Check_IPI
554          *
555          * If the CPPR is less favored, then we might be replacing
556          * an interrupt, and thus need to possibly reject it.
557          *
558          * ICP State: IPI
559          *
560          * Besides rejecting any pending interrupts, we also
561          * update XISR and pending_pri to mark IPI as pending.
562          *
563          * PAPR does not describe this state, but if the MFRR is being
564          * made less favored than its earlier value, there might be
565          * a previously-rejected interrupt needing to be resent.
566          * Ideally, we would want to resend only if
567          *      prio(pending_interrupt) < mfrr &&
568          *      prio(pending_interrupt) < cppr
569          * where pending interrupt is the one that was rejected. But
570          * we don't have that state, so we simply trigger a resend
571          * whenever the MFRR is made less favored.
572          */
573         do {
574                 old_state = new_state = READ_ONCE(icp->state);
575 
576                 /* Set_MFRR */
577                 new_state.mfrr = mfrr;
578 
579                 /* Check_IPI */
580                 reject = 0;
581                 resend = false;
582                 if (mfrr < new_state.cppr) {
583                         /* Reject a pending interrupt if not an IPI */
584                         if (mfrr <= new_state.pending_pri) {
585                                 reject = new_state.xisr;
586                                 new_state.pending_pri = mfrr;
587                                 new_state.xisr = XICS_IPI;
588                         }
589                 }
590 
591                 if (mfrr > old_state.mfrr) {
592                         resend = new_state.need_resend;
593                         new_state.need_resend = 0;
594                 }
595         } while (!icp_rm_try_update(icp, old_state, new_state));
596 
597         /* Handle reject in real mode */
598         if (reject && reject != XICS_IPI) {
599                 this_icp->n_reject++;
600                 icp_rm_deliver_irq(xics, icp, reject, false);
601         }
602 
603         /* Handle resends in real mode */
604         if (resend) {
605                 this_icp->n_check_resend++;
606                 icp_rm_check_resend(xics, icp);
607         }
608 
609         return check_too_hard(xics, this_icp);
610 }
611 
612 int xics_rm_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr)
613 {
614         union kvmppc_icp_state old_state, new_state;
615         struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
616         struct kvmppc_icp *icp = vcpu->arch.icp;
617         u32 reject;
618 
619         if (!xics || !xics->real_mode)
620                 return H_TOO_HARD;
621 
622         /*
623          * ICP State: Set_CPPR
624          *
625          * We can safely compare the new value with the current
626          * value outside of the transaction as the CPPR is only
627          * ever changed by the processor on itself
628          */
629         if (cppr > icp->state.cppr) {
630                 icp_rm_down_cppr(xics, icp, cppr);
631                 goto bail;
632         } else if (cppr == icp->state.cppr)
633                 return H_SUCCESS;
634 
635         /*
636          * ICP State: Up_CPPR
637          *
638          * The processor is raising its priority, this can result
639          * in a rejection of a pending interrupt:
640          *
641          * ICP State: Reject_Current
642          *
643          * We can remove EE from the current processor, the update
644          * transaction will set it again if needed
645          */
646         icp_rm_clr_vcpu_irq(icp->vcpu);
647 
648         do {
649                 old_state = new_state = READ_ONCE(icp->state);
650 
651                 reject = 0;
652                 new_state.cppr = cppr;
653 
654                 if (cppr <= new_state.pending_pri) {
655                         reject = new_state.xisr;
656                         new_state.xisr = 0;
657                         new_state.pending_pri = 0xff;
658                 }
659 
660         } while (!icp_rm_try_update(icp, old_state, new_state));
661 
662         /*
663          * Check for rejects. They are handled by doing a new delivery
664          * attempt (see comments in icp_rm_deliver_irq).
665          */
666         if (reject && reject != XICS_IPI) {
667                 icp->n_reject++;
668                 icp_rm_deliver_irq(xics, icp, reject, false);
669         }
670  bail:
671         return check_too_hard(xics, icp);
672 }
673 
674 static int ics_rm_eoi(struct kvm_vcpu *vcpu, u32 irq)
675 {
676         struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
677         struct kvmppc_icp *icp = vcpu->arch.icp;
678         struct kvmppc_ics *ics;
679         struct ics_irq_state *state;
680         u16 src;
681         u32 pq_old, pq_new;
682 
683         /*
684          * ICS EOI handling: For LSI, if P bit is still set, we need to
685          * resend it.
686          *
687          * For MSI, we move Q bit into P (and clear Q). If it is set,
688          * resend it.
689          */
690 
691         ics = kvmppc_xics_find_ics(xics, irq, &src);
692         if (!ics)
693                 goto bail;
694 
695         state = &ics->irq_state[src];
696 
697         if (state->lsi)
698                 pq_new = state->pq_state;
699         else
700                 do {
701                         pq_old = state->pq_state;
702                         pq_new = pq_old >> 1;
703                 } while (cmpxchg(&state->pq_state, pq_old, pq_new) != pq_old);
704 
705         if (pq_new & PQ_PRESENTED)
706                 icp_rm_deliver_irq(xics, NULL, irq, false);
707 
708         if (!hlist_empty(&vcpu->kvm->irq_ack_notifier_list)) {
709                 icp->rm_action |= XICS_RM_NOTIFY_EOI;
710                 icp->rm_eoied_irq = irq;
711         }
712 
713         if (state->host_irq) {
714                 ++vcpu->stat.pthru_all;
715                 if (state->intr_cpu != -1) {
716                         int pcpu = raw_smp_processor_id();
717 
718                         pcpu = cpu_first_thread_sibling(pcpu);
719                         ++vcpu->stat.pthru_host;
720                         if (state->intr_cpu != pcpu) {
721                                 ++vcpu->stat.pthru_bad_aff;
722                                 xics_opal_set_server(state->host_irq, pcpu);
723                         }
724                         state->intr_cpu = -1;
725                 }
726         }
727 
728  bail:
729         return check_too_hard(xics, icp);
730 }
731 
732 int xics_rm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr)
733 {
734         struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
735         struct kvmppc_icp *icp = vcpu->arch.icp;
736         u32 irq = xirr & 0x00ffffff;
737 
738         if (!xics || !xics->real_mode)
739                 return H_TOO_HARD;
740 
741         /*
742          * ICP State: EOI
743          *
744          * Note: If EOI is incorrectly used by SW to lower the CPPR
745          * value (ie more favored), we do not check for rejection of
746          * a pending interrupt, this is a SW error and PAPR specifies
747          * that we don't have to deal with it.
748          *
749          * The sending of an EOI to the ICS is handled after the
750          * CPPR update
751          *
752          * ICP State: Down_CPPR which we handle
753          * in a separate function as it's shared with H_CPPR.
754          */
755         icp_rm_down_cppr(xics, icp, xirr >> 24);
756 
757         /* IPIs have no EOI */
758         if (irq == XICS_IPI)
759                 return check_too_hard(xics, icp);
760 
761         return ics_rm_eoi(vcpu, irq);
762 }
763 
764 unsigned long eoi_rc;
765 
766 static void icp_eoi(struct irq_chip *c, u32 hwirq, __be32 xirr, bool *again)
767 {
768         void __iomem *xics_phys;
769         int64_t rc;
770 
771         rc = pnv_opal_pci_msi_eoi(c, hwirq);
772 
773         if (rc)
774                 eoi_rc = rc;
775 
776         iosync();
777 
778         /* EOI it */
779         xics_phys = local_paca->kvm_hstate.xics_phys;
780         if (xics_phys) {
781                 __raw_rm_writel(xirr, xics_phys + XICS_XIRR);
782         } else {
783                 rc = opal_int_eoi(be32_to_cpu(xirr));
784                 *again = rc > 0;
785         }
786 }
787 
788 static int xics_opal_set_server(unsigned int hw_irq, int server_cpu)
789 {
790         unsigned int mangle_cpu = get_hard_smp_processor_id(server_cpu) << 2;
791 
792         return opal_set_xive(hw_irq, mangle_cpu, DEFAULT_PRIORITY);
793 }
794 
795 /*
796  * Increment a per-CPU 32-bit unsigned integer variable.
797  * Safe to call in real-mode. Handles vmalloc'ed addresses
798  *
799  * ToDo: Make this work for any integral type
800  */
801 
802 static inline void this_cpu_inc_rm(unsigned int __percpu *addr)
803 {
804         unsigned long l;
805         unsigned int *raddr;
806         int cpu = smp_processor_id();
807 
808         raddr = per_cpu_ptr(addr, cpu);
809         l = (unsigned long)raddr;
810 
811         if (REGION_ID(l) == VMALLOC_REGION_ID) {
812                 l = vmalloc_to_phys(raddr);
813                 raddr = (unsigned int *)l;
814         }
815         ++*raddr;
816 }
817 
818 /*
819  * We don't try to update the flags in the irq_desc 'istate' field in
820  * here as would happen in the normal IRQ handling path for several reasons:
821  *  - state flags represent internal IRQ state and are not expected to be
822  *    updated outside the IRQ subsystem
823  *  - more importantly, these are useful for edge triggered interrupts,
824  *    IRQ probing, etc., but we are only handling MSI/MSIx interrupts here
825  *    and these states shouldn't apply to us.
826  *
827  * However, we do update irq_stats - we somewhat duplicate the code in
828  * kstat_incr_irqs_this_cpu() for this since this function is defined
829  * in irq/internal.h which we don't want to include here.
830  * The only difference is that desc->kstat_irqs is an allocated per CPU
831  * variable and could have been vmalloc'ed, so we can't directly
832  * call __this_cpu_inc() on it. The kstat structure is a static
833  * per CPU variable and it should be accessible by real-mode KVM.
834  *
835  */
836 static void kvmppc_rm_handle_irq_desc(struct irq_desc *desc)
837 {
838         this_cpu_inc_rm(desc->kstat_irqs);
839         __this_cpu_inc(kstat.irqs_sum);
840 }
841 
842 long kvmppc_deliver_irq_passthru(struct kvm_vcpu *vcpu,
843                                  __be32 xirr,
844                                  struct kvmppc_irq_map *irq_map,
845                                  struct kvmppc_passthru_irqmap *pimap,
846                                  bool *again)
847 {
848         struct kvmppc_xics *xics;
849         struct kvmppc_icp *icp;
850         struct kvmppc_ics *ics;
851         struct ics_irq_state *state;
852         u32 irq;
853         u16 src;
854         u32 pq_old, pq_new;
855 
856         irq = irq_map->v_hwirq;
857         xics = vcpu->kvm->arch.xics;
858         icp = vcpu->arch.icp;
859 
860         kvmppc_rm_handle_irq_desc(irq_map->desc);
861 
862         ics = kvmppc_xics_find_ics(xics, irq, &src);
863         if (!ics)
864                 return 2;
865 
866         state = &ics->irq_state[src];
867 
868         /* only MSIs register bypass producers, so it must be MSI here */
869         do {
870                 pq_old = state->pq_state;
871                 pq_new = ((pq_old << 1) & 3) | PQ_PRESENTED;
872         } while (cmpxchg(&state->pq_state, pq_old, pq_new) != pq_old);
873 
874         /* Test P=1, Q=0, this is the only case where we present */
875         if (pq_new == PQ_PRESENTED)
876                 icp_rm_deliver_irq(xics, icp, irq, false);
877 
878         /* EOI the interrupt */
879         icp_eoi(irq_desc_get_chip(irq_map->desc), irq_map->r_hwirq, xirr,
880                 again);
881 
882         if (check_too_hard(xics, icp) == H_TOO_HARD)
883                 return 2;
884         else
885                 return -2;
886 }
887 
888 /*  --- Non-real mode XICS-related built-in routines ---  */
889 
890 /**
891  * Host Operations poked by RM KVM
892  */
893 static void rm_host_ipi_action(int action, void *data)
894 {
895         switch (action) {
896         case XICS_RM_KICK_VCPU:
897                 kvmppc_host_rm_ops_hv->vcpu_kick(data);
898                 break;
899         default:
900                 WARN(1, "Unexpected rm_action=%d data=%p\n", action, data);
901                 break;
902         }
903 
904 }
905 
906 void kvmppc_xics_ipi_action(void)
907 {
908         int core;
909         unsigned int cpu = smp_processor_id();
910         struct kvmppc_host_rm_core *rm_corep;
911 
912         core = cpu >> threads_shift;
913         rm_corep = &kvmppc_host_rm_ops_hv->rm_core[core];
914 
915         if (rm_corep->rm_data) {
916                 rm_host_ipi_action(rm_corep->rm_state.rm_action,
917                                                         rm_corep->rm_data);
918                 /* Order these stores against the real mode KVM */
919                 rm_corep->rm_data = NULL;
920                 smp_wmb();
921                 rm_corep->rm_state.rm_action = 0;
922         }
923 }
924 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp