~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/powerpc/kernel/irq.c

Version: ~ [ linux-5.16 ] ~ [ linux-5.15.13 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.90 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.170 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.224 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.261 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.296 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.298 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-or-later
  2 /*
  3  *  Derived from arch/i386/kernel/irq.c
  4  *    Copyright (C) 1992 Linus Torvalds
  5  *  Adapted from arch/i386 by Gary Thomas
  6  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
  7  *  Updated and modified by Cort Dougan <cort@fsmlabs.com>
  8  *    Copyright (C) 1996-2001 Cort Dougan
  9  *  Adapted for Power Macintosh by Paul Mackerras
 10  *    Copyright (C) 1996 Paul Mackerras (paulus@cs.anu.edu.au)
 11  *
 12  * This file contains the code used by various IRQ handling routines:
 13  * asking for different IRQ's should be done through these routines
 14  * instead of just grabbing them. Thus setups with different IRQ numbers
 15  * shouldn't result in any weird surprises, and installing new handlers
 16  * should be easier.
 17  *
 18  * The MPC8xx has an interrupt mask in the SIU.  If a bit is set, the
 19  * interrupt is _enabled_.  As expected, IRQ0 is bit 0 in the 32-bit
 20  * mask register (of which only 16 are defined), hence the weird shifting
 21  * and complement of the cached_irq_mask.  I want to be able to stuff
 22  * this right into the SIU SMASK register.
 23  * Many of the prep/chrp functions are conditional compiled on CONFIG_PPC_8xx
 24  * to reduce code space and undefined function references.
 25  */
 26 
 27 #undef DEBUG
 28 
 29 #include <linux/export.h>
 30 #include <linux/threads.h>
 31 #include <linux/kernel_stat.h>
 32 #include <linux/signal.h>
 33 #include <linux/sched.h>
 34 #include <linux/ptrace.h>
 35 #include <linux/ioport.h>
 36 #include <linux/interrupt.h>
 37 #include <linux/timex.h>
 38 #include <linux/init.h>
 39 #include <linux/slab.h>
 40 #include <linux/delay.h>
 41 #include <linux/irq.h>
 42 #include <linux/seq_file.h>
 43 #include <linux/cpumask.h>
 44 #include <linux/profile.h>
 45 #include <linux/bitops.h>
 46 #include <linux/list.h>
 47 #include <linux/radix-tree.h>
 48 #include <linux/mutex.h>
 49 #include <linux/pci.h>
 50 #include <linux/debugfs.h>
 51 #include <linux/of.h>
 52 #include <linux/of_irq.h>
 53 #include <linux/vmalloc.h>
 54 
 55 #include <linux/uaccess.h>
 56 #include <asm/io.h>
 57 #include <asm/pgtable.h>
 58 #include <asm/irq.h>
 59 #include <asm/cache.h>
 60 #include <asm/prom.h>
 61 #include <asm/ptrace.h>
 62 #include <asm/machdep.h>
 63 #include <asm/udbg.h>
 64 #include <asm/smp.h>
 65 #include <asm/livepatch.h>
 66 #include <asm/asm-prototypes.h>
 67 #include <asm/hw_irq.h>
 68 
 69 #ifdef CONFIG_PPC64
 70 #include <asm/paca.h>
 71 #include <asm/firmware.h>
 72 #include <asm/lv1call.h>
 73 #endif
 74 #define CREATE_TRACE_POINTS
 75 #include <asm/trace.h>
 76 #include <asm/cpu_has_feature.h>
 77 
 78 DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
 79 EXPORT_PER_CPU_SYMBOL(irq_stat);
 80 
 81 #ifdef CONFIG_PPC32
 82 atomic_t ppc_n_lost_interrupts;
 83 
 84 #ifdef CONFIG_TAU_INT
 85 extern int tau_initialized;
 86 u32 tau_interrupts(unsigned long cpu);
 87 #endif
 88 #endif /* CONFIG_PPC32 */
 89 
 90 #ifdef CONFIG_PPC64
 91 
 92 int distribute_irqs = 1;
 93 
 94 static inline notrace unsigned long get_irq_happened(void)
 95 {
 96         unsigned long happened;
 97 
 98         __asm__ __volatile__("lbz %0,%1(13)"
 99         : "=r" (happened) : "i" (offsetof(struct paca_struct, irq_happened)));
100 
101         return happened;
102 }
103 
104 static inline notrace int decrementer_check_overflow(void)
105 {
106         u64 now = get_tb_or_rtc();
107         u64 *next_tb = this_cpu_ptr(&decrementers_next_tb);
108  
109         return now >= *next_tb;
110 }
111 
112 /* This is called whenever we are re-enabling interrupts
113  * and returns either 0 (nothing to do) or 500/900/280/a00/e80 if
114  * there's an EE, DEC or DBELL to generate.
115  *
116  * This is called in two contexts: From arch_local_irq_restore()
117  * before soft-enabling interrupts, and from the exception exit
118  * path when returning from an interrupt from a soft-disabled to
119  * a soft enabled context. In both case we have interrupts hard
120  * disabled.
121  *
122  * We take care of only clearing the bits we handled in the
123  * PACA irq_happened field since we can only re-emit one at a
124  * time and we don't want to "lose" one.
125  */
126 notrace unsigned int __check_irq_replay(void)
127 {
128         /*
129          * We use local_paca rather than get_paca() to avoid all
130          * the debug_smp_processor_id() business in this low level
131          * function
132          */
133         unsigned char happened = local_paca->irq_happened;
134 
135         /*
136          * We are responding to the next interrupt, so interrupt-off
137          * latencies should be reset here.
138          */
139         trace_hardirqs_on();
140         trace_hardirqs_off();
141 
142         /*
143          * We are always hard disabled here, but PACA_IRQ_HARD_DIS may
144          * not be set, which means interrupts have only just been hard
145          * disabled as part of the local_irq_restore or interrupt return
146          * code. In that case, skip the decrementr check becaus it's
147          * expensive to read the TB.
148          *
149          * HARD_DIS then gets cleared here, but it's reconciled later.
150          * Either local_irq_disable will replay the interrupt and that
151          * will reconcile state like other hard interrupts. Or interrupt
152          * retur will replay the interrupt and in that case it sets
153          * PACA_IRQ_HARD_DIS by hand (see comments in entry_64.S).
154          */
155         if (happened & PACA_IRQ_HARD_DIS) {
156                 local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
157 
158                 /*
159                  * We may have missed a decrementer interrupt if hard disabled.
160                  * Check the decrementer register in case we had a rollover
161                  * while hard disabled.
162                  */
163                 if (!(happened & PACA_IRQ_DEC)) {
164                         if (decrementer_check_overflow()) {
165                                 local_paca->irq_happened |= PACA_IRQ_DEC;
166                                 happened |= PACA_IRQ_DEC;
167                         }
168                 }
169         }
170 
171         /*
172          * Force the delivery of pending soft-disabled interrupts on PS3.
173          * Any HV call will have this side effect.
174          */
175         if (firmware_has_feature(FW_FEATURE_PS3_LV1)) {
176                 u64 tmp, tmp2;
177                 lv1_get_version_info(&tmp, &tmp2);
178         }
179 
180         /*
181          * Check if an hypervisor Maintenance interrupt happened.
182          * This is a higher priority interrupt than the others, so
183          * replay it first.
184          */
185         if (happened & PACA_IRQ_HMI) {
186                 local_paca->irq_happened &= ~PACA_IRQ_HMI;
187                 return 0xe60;
188         }
189 
190         if (happened & PACA_IRQ_DEC) {
191                 local_paca->irq_happened &= ~PACA_IRQ_DEC;
192                 return 0x900;
193         }
194 
195         if (happened & PACA_IRQ_PMI) {
196                 local_paca->irq_happened &= ~PACA_IRQ_PMI;
197                 return 0xf00;
198         }
199 
200         if (happened & PACA_IRQ_EE) {
201                 local_paca->irq_happened &= ~PACA_IRQ_EE;
202                 return 0x500;
203         }
204 
205 #ifdef CONFIG_PPC_BOOK3E
206         /*
207          * Check if an EPR external interrupt happened this bit is typically
208          * set if we need to handle another "edge" interrupt from within the
209          * MPIC "EPR" handler.
210          */
211         if (happened & PACA_IRQ_EE_EDGE) {
212                 local_paca->irq_happened &= ~PACA_IRQ_EE_EDGE;
213                 return 0x500;
214         }
215 
216         if (happened & PACA_IRQ_DBELL) {
217                 local_paca->irq_happened &= ~PACA_IRQ_DBELL;
218                 return 0x280;
219         }
220 #else
221         if (happened & PACA_IRQ_DBELL) {
222                 local_paca->irq_happened &= ~PACA_IRQ_DBELL;
223                 return 0xa00;
224         }
225 #endif /* CONFIG_PPC_BOOK3E */
226 
227         /* There should be nothing left ! */
228         BUG_ON(local_paca->irq_happened != 0);
229 
230         return 0;
231 }
232 
233 notrace void arch_local_irq_restore(unsigned long mask)
234 {
235         unsigned char irq_happened;
236         unsigned int replay;
237 
238         /* Write the new soft-enabled value */
239         irq_soft_mask_set(mask);
240         if (mask)
241                 return;
242 
243         /*
244          * From this point onward, we can take interrupts, preempt,
245          * etc... unless we got hard-disabled. We check if an event
246          * happened. If none happened, we know we can just return.
247          *
248          * We may have preempted before the check below, in which case
249          * we are checking the "new" CPU instead of the old one. This
250          * is only a problem if an event happened on the "old" CPU.
251          *
252          * External interrupt events will have caused interrupts to
253          * be hard-disabled, so there is no problem, we
254          * cannot have preempted.
255          */
256         irq_happened = get_irq_happened();
257         if (!irq_happened) {
258 #ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
259                 WARN_ON_ONCE(!(mfmsr() & MSR_EE));
260 #endif
261                 return;
262         }
263 
264         /*
265          * We need to hard disable to get a trusted value from
266          * __check_irq_replay(). We also need to soft-disable
267          * again to avoid warnings in there due to the use of
268          * per-cpu variables.
269          */
270         if (!(irq_happened & PACA_IRQ_HARD_DIS)) {
271 #ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
272                 WARN_ON_ONCE(!(mfmsr() & MSR_EE));
273 #endif
274                 __hard_irq_disable();
275 #ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
276         } else {
277                 /*
278                  * We should already be hard disabled here. We had bugs
279                  * where that wasn't the case so let's dbl check it and
280                  * warn if we are wrong. Only do that when IRQ tracing
281                  * is enabled as mfmsr() can be costly.
282                  */
283                 if (WARN_ON_ONCE(mfmsr() & MSR_EE))
284                         __hard_irq_disable();
285 #endif
286         }
287 
288         irq_soft_mask_set(IRQS_ALL_DISABLED);
289         trace_hardirqs_off();
290 
291         /*
292          * Check if anything needs to be re-emitted. We haven't
293          * soft-enabled yet to avoid warnings in decrementer_check_overflow
294          * accessing per-cpu variables
295          */
296         replay = __check_irq_replay();
297 
298         /* We can soft-enable now */
299         trace_hardirqs_on();
300         irq_soft_mask_set(IRQS_ENABLED);
301 
302         /*
303          * And replay if we have to. This will return with interrupts
304          * hard-enabled.
305          */
306         if (replay) {
307                 __replay_interrupt(replay);
308                 return;
309         }
310 
311         /* Finally, let's ensure we are hard enabled */
312         __hard_irq_enable();
313 }
314 EXPORT_SYMBOL(arch_local_irq_restore);
315 
316 /*
317  * This is specifically called by assembly code to re-enable interrupts
318  * if they are currently disabled. This is typically called before
319  * schedule() or do_signal() when returning to userspace. We do it
320  * in C to avoid the burden of dealing with lockdep etc...
321  *
322  * NOTE: This is called with interrupts hard disabled but not marked
323  * as such in paca->irq_happened, so we need to resync this.
324  */
325 void notrace restore_interrupts(void)
326 {
327         if (irqs_disabled()) {
328                 local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
329                 local_irq_enable();
330         } else
331                 __hard_irq_enable();
332 }
333 
334 /*
335  * This is a helper to use when about to go into idle low-power
336  * when the latter has the side effect of re-enabling interrupts
337  * (such as calling H_CEDE under pHyp).
338  *
339  * You call this function with interrupts soft-disabled (this is
340  * already the case when ppc_md.power_save is called). The function
341  * will return whether to enter power save or just return.
342  *
343  * In the former case, it will have notified lockdep of interrupts
344  * being re-enabled and generally sanitized the lazy irq state,
345  * and in the latter case it will leave with interrupts hard
346  * disabled and marked as such, so the local_irq_enable() call
347  * in arch_cpu_idle() will properly re-enable everything.
348  */
349 bool prep_irq_for_idle(void)
350 {
351         /*
352          * First we need to hard disable to ensure no interrupt
353          * occurs before we effectively enter the low power state
354          */
355         __hard_irq_disable();
356         local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
357 
358         /*
359          * If anything happened while we were soft-disabled,
360          * we return now and do not enter the low power state.
361          */
362         if (lazy_irq_pending())
363                 return false;
364 
365         /* Tell lockdep we are about to re-enable */
366         trace_hardirqs_on();
367 
368         /*
369          * Mark interrupts as soft-enabled and clear the
370          * PACA_IRQ_HARD_DIS from the pending mask since we
371          * are about to hard enable as well as a side effect
372          * of entering the low power state.
373          */
374         local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
375         irq_soft_mask_set(IRQS_ENABLED);
376 
377         /* Tell the caller to enter the low power state */
378         return true;
379 }
380 
381 #ifdef CONFIG_PPC_BOOK3S
382 /*
383  * This is for idle sequences that return with IRQs off, but the
384  * idle state itself wakes on interrupt. Tell the irq tracer that
385  * IRQs are enabled for the duration of idle so it does not get long
386  * off times. Must be paired with fini_irq_for_idle_irqsoff.
387  */
388 bool prep_irq_for_idle_irqsoff(void)
389 {
390         WARN_ON(!irqs_disabled());
391 
392         /*
393          * First we need to hard disable to ensure no interrupt
394          * occurs before we effectively enter the low power state
395          */
396         __hard_irq_disable();
397         local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
398 
399         /*
400          * If anything happened while we were soft-disabled,
401          * we return now and do not enter the low power state.
402          */
403         if (lazy_irq_pending())
404                 return false;
405 
406         /* Tell lockdep we are about to re-enable */
407         trace_hardirqs_on();
408 
409         return true;
410 }
411 
412 /*
413  * Take the SRR1 wakeup reason, index into this table to find the
414  * appropriate irq_happened bit.
415  *
416  * Sytem reset exceptions taken in idle state also come through here,
417  * but they are NMI interrupts so do not need to wait for IRQs to be
418  * restored, and should be taken as early as practical. These are marked
419  * with 0xff in the table. The Power ISA specifies 0100b as the system
420  * reset interrupt reason.
421  */
422 #define IRQ_SYSTEM_RESET        0xff
423 
424 static const u8 srr1_to_lazyirq[0x10] = {
425         0, 0, 0,
426         PACA_IRQ_DBELL,
427         IRQ_SYSTEM_RESET,
428         PACA_IRQ_DBELL,
429         PACA_IRQ_DEC,
430         0,
431         PACA_IRQ_EE,
432         PACA_IRQ_EE,
433         PACA_IRQ_HMI,
434         0, 0, 0, 0, 0 };
435 
436 void replay_system_reset(void)
437 {
438         struct pt_regs regs;
439 
440         ppc_save_regs(&regs);
441         regs.trap = 0x100;
442         get_paca()->in_nmi = 1;
443         system_reset_exception(&regs);
444         get_paca()->in_nmi = 0;
445 }
446 EXPORT_SYMBOL_GPL(replay_system_reset);
447 
448 void irq_set_pending_from_srr1(unsigned long srr1)
449 {
450         unsigned int idx = (srr1 & SRR1_WAKEMASK_P8) >> 18;
451         u8 reason = srr1_to_lazyirq[idx];
452 
453         /*
454          * Take the system reset now, which is immediately after registers
455          * are restored from idle. It's an NMI, so interrupts need not be
456          * re-enabled before it is taken.
457          */
458         if (unlikely(reason == IRQ_SYSTEM_RESET)) {
459                 replay_system_reset();
460                 return;
461         }
462 
463         /*
464          * The 0 index (SRR1[42:45]=b0000) must always evaluate to 0,
465          * so this can be called unconditionally with the SRR1 wake
466          * reason as returned by the idle code, which uses 0 to mean no
467          * interrupt.
468          *
469          * If a future CPU was to designate this as an interrupt reason,
470          * then a new index for no interrupt must be assigned.
471          */
472         local_paca->irq_happened |= reason;
473 }
474 #endif /* CONFIG_PPC_BOOK3S */
475 
476 /*
477  * Force a replay of the external interrupt handler on this CPU.
478  */
479 void force_external_irq_replay(void)
480 {
481         /*
482          * This must only be called with interrupts soft-disabled,
483          * the replay will happen when re-enabling.
484          */
485         WARN_ON(!arch_irqs_disabled());
486 
487         /*
488          * Interrupts must always be hard disabled before irq_happened is
489          * modified (to prevent lost update in case of interrupt between
490          * load and store).
491          */
492         __hard_irq_disable();
493         local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
494 
495         /* Indicate in the PACA that we have an interrupt to replay */
496         local_paca->irq_happened |= PACA_IRQ_EE;
497 }
498 
499 #endif /* CONFIG_PPC64 */
500 
501 int arch_show_interrupts(struct seq_file *p, int prec)
502 {
503         int j;
504 
505 #if defined(CONFIG_PPC32) && defined(CONFIG_TAU_INT)
506         if (tau_initialized) {
507                 seq_printf(p, "%*s: ", prec, "TAU");
508                 for_each_online_cpu(j)
509                         seq_printf(p, "%10u ", tau_interrupts(j));
510                 seq_puts(p, "  PowerPC             Thermal Assist (cpu temp)\n");
511         }
512 #endif /* CONFIG_PPC32 && CONFIG_TAU_INT */
513 
514         seq_printf(p, "%*s: ", prec, "LOC");
515         for_each_online_cpu(j)
516                 seq_printf(p, "%10u ", per_cpu(irq_stat, j).timer_irqs_event);
517         seq_printf(p, "  Local timer interrupts for timer event device\n");
518 
519         seq_printf(p, "%*s: ", prec, "BCT");
520         for_each_online_cpu(j)
521                 seq_printf(p, "%10u ", per_cpu(irq_stat, j).broadcast_irqs_event);
522         seq_printf(p, "  Broadcast timer interrupts for timer event device\n");
523 
524         seq_printf(p, "%*s: ", prec, "LOC");
525         for_each_online_cpu(j)
526                 seq_printf(p, "%10u ", per_cpu(irq_stat, j).timer_irqs_others);
527         seq_printf(p, "  Local timer interrupts for others\n");
528 
529         seq_printf(p, "%*s: ", prec, "SPU");
530         for_each_online_cpu(j)
531                 seq_printf(p, "%10u ", per_cpu(irq_stat, j).spurious_irqs);
532         seq_printf(p, "  Spurious interrupts\n");
533 
534         seq_printf(p, "%*s: ", prec, "PMI");
535         for_each_online_cpu(j)
536                 seq_printf(p, "%10u ", per_cpu(irq_stat, j).pmu_irqs);
537         seq_printf(p, "  Performance monitoring interrupts\n");
538 
539         seq_printf(p, "%*s: ", prec, "MCE");
540         for_each_online_cpu(j)
541                 seq_printf(p, "%10u ", per_cpu(irq_stat, j).mce_exceptions);
542         seq_printf(p, "  Machine check exceptions\n");
543 
544         if (cpu_has_feature(CPU_FTR_HVMODE)) {
545                 seq_printf(p, "%*s: ", prec, "HMI");
546                 for_each_online_cpu(j)
547                         seq_printf(p, "%10u ",
548                                         per_cpu(irq_stat, j).hmi_exceptions);
549                 seq_printf(p, "  Hypervisor Maintenance Interrupts\n");
550         }
551 
552         seq_printf(p, "%*s: ", prec, "NMI");
553         for_each_online_cpu(j)
554                 seq_printf(p, "%10u ", per_cpu(irq_stat, j).sreset_irqs);
555         seq_printf(p, "  System Reset interrupts\n");
556 
557 #ifdef CONFIG_PPC_WATCHDOG
558         seq_printf(p, "%*s: ", prec, "WDG");
559         for_each_online_cpu(j)
560                 seq_printf(p, "%10u ", per_cpu(irq_stat, j).soft_nmi_irqs);
561         seq_printf(p, "  Watchdog soft-NMI interrupts\n");
562 #endif
563 
564 #ifdef CONFIG_PPC_DOORBELL
565         if (cpu_has_feature(CPU_FTR_DBELL)) {
566                 seq_printf(p, "%*s: ", prec, "DBL");
567                 for_each_online_cpu(j)
568                         seq_printf(p, "%10u ", per_cpu(irq_stat, j).doorbell_irqs);
569                 seq_printf(p, "  Doorbell interrupts\n");
570         }
571 #endif
572 
573         return 0;
574 }
575 
576 /*
577  * /proc/stat helpers
578  */
579 u64 arch_irq_stat_cpu(unsigned int cpu)
580 {
581         u64 sum = per_cpu(irq_stat, cpu).timer_irqs_event;
582 
583         sum += per_cpu(irq_stat, cpu).broadcast_irqs_event;
584         sum += per_cpu(irq_stat, cpu).pmu_irqs;
585         sum += per_cpu(irq_stat, cpu).mce_exceptions;
586         sum += per_cpu(irq_stat, cpu).spurious_irqs;
587         sum += per_cpu(irq_stat, cpu).timer_irqs_others;
588         sum += per_cpu(irq_stat, cpu).hmi_exceptions;
589         sum += per_cpu(irq_stat, cpu).sreset_irqs;
590 #ifdef CONFIG_PPC_WATCHDOG
591         sum += per_cpu(irq_stat, cpu).soft_nmi_irqs;
592 #endif
593 #ifdef CONFIG_PPC_DOORBELL
594         sum += per_cpu(irq_stat, cpu).doorbell_irqs;
595 #endif
596 
597         return sum;
598 }
599 
600 static inline void check_stack_overflow(void)
601 {
602 #ifdef CONFIG_DEBUG_STACKOVERFLOW
603         long sp;
604 
605         sp = current_stack_pointer() & (THREAD_SIZE-1);
606 
607         /* check for stack overflow: is there less than 2KB free? */
608         if (unlikely(sp < 2048)) {
609                 pr_err("do_IRQ: stack overflow: %ld\n", sp);
610                 dump_stack();
611         }
612 #endif
613 }
614 
615 void __do_irq(struct pt_regs *regs)
616 {
617         unsigned int irq;
618 
619         irq_enter();
620 
621         trace_irq_entry(regs);
622 
623         /*
624          * Query the platform PIC for the interrupt & ack it.
625          *
626          * This will typically lower the interrupt line to the CPU
627          */
628         irq = ppc_md.get_irq();
629 
630         /* We can hard enable interrupts now to allow perf interrupts */
631         may_hard_irq_enable();
632 
633         /* And finally process it */
634         if (unlikely(!irq))
635                 __this_cpu_inc(irq_stat.spurious_irqs);
636         else
637                 generic_handle_irq(irq);
638 
639         trace_irq_exit(regs);
640 
641         irq_exit();
642 }
643 
644 void do_IRQ(struct pt_regs *regs)
645 {
646         struct pt_regs *old_regs = set_irq_regs(regs);
647         void *cursp, *irqsp, *sirqsp;
648 
649         /* Switch to the irq stack to handle this */
650         cursp = (void *)(current_stack_pointer() & ~(THREAD_SIZE - 1));
651         irqsp = hardirq_ctx[raw_smp_processor_id()];
652         sirqsp = softirq_ctx[raw_smp_processor_id()];
653 
654         check_stack_overflow();
655 
656         /* Already there ? */
657         if (unlikely(cursp == irqsp || cursp == sirqsp)) {
658                 __do_irq(regs);
659                 set_irq_regs(old_regs);
660                 return;
661         }
662         /* Switch stack and call */
663         call_do_irq(regs, irqsp);
664 
665         set_irq_regs(old_regs);
666 }
667 
668 static void *__init alloc_vm_stack(void)
669 {
670         return __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN, VMALLOC_START,
671                                     VMALLOC_END, THREADINFO_GFP, PAGE_KERNEL,
672                                      0, NUMA_NO_NODE, (void*)_RET_IP_);
673 }
674 
675 static void __init vmap_irqstack_init(void)
676 {
677         int i;
678 
679         for_each_possible_cpu(i) {
680                 softirq_ctx[i] = alloc_vm_stack();
681                 hardirq_ctx[i] = alloc_vm_stack();
682         }
683 }
684 
685 
686 void __init init_IRQ(void)
687 {
688         if (IS_ENABLED(CONFIG_VMAP_STACK))
689                 vmap_irqstack_init();
690 
691         if (ppc_md.init_IRQ)
692                 ppc_md.init_IRQ();
693 }
694 
695 #if defined(CONFIG_BOOKE) || defined(CONFIG_40x)
696 void   *critirq_ctx[NR_CPUS] __read_mostly;
697 void    *dbgirq_ctx[NR_CPUS] __read_mostly;
698 void *mcheckirq_ctx[NR_CPUS] __read_mostly;
699 #endif
700 
701 void *softirq_ctx[NR_CPUS] __read_mostly;
702 void *hardirq_ctx[NR_CPUS] __read_mostly;
703 
704 void do_softirq_own_stack(void)
705 {
706         call_do_softirq(softirq_ctx[smp_processor_id()]);
707 }
708 
709 irq_hw_number_t virq_to_hw(unsigned int virq)
710 {
711         struct irq_data *irq_data = irq_get_irq_data(virq);
712         return WARN_ON(!irq_data) ? 0 : irq_data->hwirq;
713 }
714 EXPORT_SYMBOL_GPL(virq_to_hw);
715 
716 #ifdef CONFIG_SMP
717 int irq_choose_cpu(const struct cpumask *mask)
718 {
719         int cpuid;
720 
721         if (cpumask_equal(mask, cpu_online_mask)) {
722                 static int irq_rover;
723                 static DEFINE_RAW_SPINLOCK(irq_rover_lock);
724                 unsigned long flags;
725 
726                 /* Round-robin distribution... */
727 do_round_robin:
728                 raw_spin_lock_irqsave(&irq_rover_lock, flags);
729 
730                 irq_rover = cpumask_next(irq_rover, cpu_online_mask);
731                 if (irq_rover >= nr_cpu_ids)
732                         irq_rover = cpumask_first(cpu_online_mask);
733 
734                 cpuid = irq_rover;
735 
736                 raw_spin_unlock_irqrestore(&irq_rover_lock, flags);
737         } else {
738                 cpuid = cpumask_first_and(mask, cpu_online_mask);
739                 if (cpuid >= nr_cpu_ids)
740                         goto do_round_robin;
741         }
742 
743         return get_hard_smp_processor_id(cpuid);
744 }
745 #else
746 int irq_choose_cpu(const struct cpumask *mask)
747 {
748         return hard_smp_processor_id();
749 }
750 #endif
751 
752 #ifdef CONFIG_PPC64
753 static int __init setup_noirqdistrib(char *str)
754 {
755         distribute_irqs = 0;
756         return 1;
757 }
758 
759 __setup("noirqdistrib", setup_noirqdistrib);
760 #endif /* CONFIG_PPC64 */
761 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp