~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/sparc/kernel/irq_64.c

Version: ~ [ linux-4.20-rc6 ] ~ [ linux-4.19.8 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.87 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.144 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.166 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.128 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.61 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.31.14 ] ~ [ linux-2.6.30.10 ] ~ [ linux-2.6.29.6 ] ~ [ linux-2.6.28.10 ] ~ [ linux-2.6.27.62 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /* irq.c: UltraSparc IRQ handling/init/registry.
  3  *
  4  * Copyright (C) 1997, 2007, 2008 David S. Miller (davem@davemloft.net)
  5  * Copyright (C) 1998  Eddie C. Dost    (ecd@skynet.be)
  6  * Copyright (C) 1998  Jakub Jelinek    (jj@ultra.linux.cz)
  7  */
  8 
  9 #include <linux/sched.h>
 10 #include <linux/linkage.h>
 11 #include <linux/ptrace.h>
 12 #include <linux/errno.h>
 13 #include <linux/kernel_stat.h>
 14 #include <linux/signal.h>
 15 #include <linux/mm.h>
 16 #include <linux/interrupt.h>
 17 #include <linux/slab.h>
 18 #include <linux/random.h>
 19 #include <linux/init.h>
 20 #include <linux/delay.h>
 21 #include <linux/proc_fs.h>
 22 #include <linux/seq_file.h>
 23 #include <linux/ftrace.h>
 24 #include <linux/irq.h>
 25 
 26 #include <asm/ptrace.h>
 27 #include <asm/processor.h>
 28 #include <linux/atomic.h>
 29 #include <asm/irq.h>
 30 #include <asm/io.h>
 31 #include <asm/iommu.h>
 32 #include <asm/upa.h>
 33 #include <asm/oplib.h>
 34 #include <asm/prom.h>
 35 #include <asm/timer.h>
 36 #include <asm/smp.h>
 37 #include <asm/starfire.h>
 38 #include <linux/uaccess.h>
 39 #include <asm/cache.h>
 40 #include <asm/cpudata.h>
 41 #include <asm/auxio.h>
 42 #include <asm/head.h>
 43 #include <asm/hypervisor.h>
 44 #include <asm/cacheflush.h>
 45 
 46 #include "entry.h"
 47 #include "cpumap.h"
 48 #include "kstack.h"
 49 
 50 struct ino_bucket *ivector_table;
 51 unsigned long ivector_table_pa;
 52 
 53 /* On several sun4u processors, it is illegal to mix bypass and
 54  * non-bypass accesses.  Therefore we access all INO buckets
 55  * using bypass accesses only.
 56  */
 57 static unsigned long bucket_get_chain_pa(unsigned long bucket_pa)
 58 {
 59         unsigned long ret;
 60 
 61         __asm__ __volatile__("ldxa      [%1] %2, %0"
 62                              : "=&r" (ret)
 63                              : "r" (bucket_pa +
 64                                     offsetof(struct ino_bucket,
 65                                              __irq_chain_pa)),
 66                                "i" (ASI_PHYS_USE_EC));
 67 
 68         return ret;
 69 }
 70 
 71 static void bucket_clear_chain_pa(unsigned long bucket_pa)
 72 {
 73         __asm__ __volatile__("stxa      %%g0, [%0] %1"
 74                              : /* no outputs */
 75                              : "r" (bucket_pa +
 76                                     offsetof(struct ino_bucket,
 77                                              __irq_chain_pa)),
 78                                "i" (ASI_PHYS_USE_EC));
 79 }
 80 
 81 static unsigned int bucket_get_irq(unsigned long bucket_pa)
 82 {
 83         unsigned int ret;
 84 
 85         __asm__ __volatile__("lduwa     [%1] %2, %0"
 86                              : "=&r" (ret)
 87                              : "r" (bucket_pa +
 88                                     offsetof(struct ino_bucket,
 89                                              __irq)),
 90                                "i" (ASI_PHYS_USE_EC));
 91 
 92         return ret;
 93 }
 94 
 95 static void bucket_set_irq(unsigned long bucket_pa, unsigned int irq)
 96 {
 97         __asm__ __volatile__("stwa      %0, [%1] %2"
 98                              : /* no outputs */
 99                              : "r" (irq),
100                                "r" (bucket_pa +
101                                     offsetof(struct ino_bucket,
102                                              __irq)),
103                                "i" (ASI_PHYS_USE_EC));
104 }
105 
106 #define irq_work_pa(__cpu)      &(trap_block[(__cpu)].irq_worklist_pa)
107 
108 static unsigned long hvirq_major __initdata;
109 static int __init early_hvirq_major(char *p)
110 {
111         int rc = kstrtoul(p, 10, &hvirq_major);
112 
113         return rc;
114 }
115 early_param("hvirq", early_hvirq_major);
116 
117 static int hv_irq_version;
118 
119 /* Major version 2.0 of HV_GRP_INTR added support for the VIRQ cookie
120  * based interfaces, but:
121  *
122  * 1) Several OSs, Solaris and Linux included, use them even when only
123  *    negotiating version 1.0 (or failing to negotiate at all).  So the
124  *    hypervisor has a workaround that provides the VIRQ interfaces even
125  *    when only verion 1.0 of the API is in use.
126  *
127  * 2) Second, and more importantly, with major version 2.0 these VIRQ
128  *    interfaces only were actually hooked up for LDC interrupts, even
129  *    though the Hypervisor specification clearly stated:
130  *
131  *      The new interrupt API functions will be available to a guest
132  *      when it negotiates version 2.0 in the interrupt API group 0x2. When
133  *      a guest negotiates version 2.0, all interrupt sources will only
134  *      support using the cookie interface, and any attempt to use the
135  *      version 1.0 interrupt APIs numbered 0xa0 to 0xa6 will result in the
136  *      ENOTSUPPORTED error being returned.
137  *
138  *   with an emphasis on "all interrupt sources".
139  *
140  * To correct this, major version 3.0 was created which does actually
141  * support VIRQs for all interrupt sources (not just LDC devices).  So
142  * if we want to move completely over the cookie based VIRQs we must
143  * negotiate major version 3.0 or later of HV_GRP_INTR.
144  */
145 static bool sun4v_cookie_only_virqs(void)
146 {
147         if (hv_irq_version >= 3)
148                 return true;
149         return false;
150 }
151 
152 static void __init irq_init_hv(void)
153 {
154         unsigned long hv_error, major, minor = 0;
155 
156         if (tlb_type != hypervisor)
157                 return;
158 
159         if (hvirq_major)
160                 major = hvirq_major;
161         else
162                 major = 3;
163 
164         hv_error = sun4v_hvapi_register(HV_GRP_INTR, major, &minor);
165         if (!hv_error)
166                 hv_irq_version = major;
167         else
168                 hv_irq_version = 1;
169 
170         pr_info("SUN4V: Using IRQ API major %d, cookie only virqs %s\n",
171                 hv_irq_version,
172                 sun4v_cookie_only_virqs() ? "enabled" : "disabled");
173 }
174 
175 /* This function is for the timer interrupt.*/
176 int __init arch_probe_nr_irqs(void)
177 {
178         return 1;
179 }
180 
181 #define DEFAULT_NUM_IVECS       (0xfffU)
182 static unsigned int nr_ivec = DEFAULT_NUM_IVECS;
183 #define NUM_IVECS (nr_ivec)
184 
185 static unsigned int __init size_nr_ivec(void)
186 {
187         if (tlb_type == hypervisor) {
188                 switch (sun4v_chip_type) {
189                 /* Athena's devhandle|devino is large.*/
190                 case SUN4V_CHIP_SPARC64X:
191                         nr_ivec = 0xffff;
192                         break;
193                 }
194         }
195         return nr_ivec;
196 }
197 
198 struct irq_handler_data {
199         union {
200                 struct {
201                         unsigned int dev_handle;
202                         unsigned int dev_ino;
203                 };
204                 unsigned long sysino;
205         };
206         struct ino_bucket bucket;
207         unsigned long   iclr;
208         unsigned long   imap;
209 };
210 
211 static inline unsigned int irq_data_to_handle(struct irq_data *data)
212 {
213         struct irq_handler_data *ihd = irq_data_get_irq_handler_data(data);
214 
215         return ihd->dev_handle;
216 }
217 
218 static inline unsigned int irq_data_to_ino(struct irq_data *data)
219 {
220         struct irq_handler_data *ihd = irq_data_get_irq_handler_data(data);
221 
222         return ihd->dev_ino;
223 }
224 
225 static inline unsigned long irq_data_to_sysino(struct irq_data *data)
226 {
227         struct irq_handler_data *ihd = irq_data_get_irq_handler_data(data);
228 
229         return ihd->sysino;
230 }
231 
232 void irq_free(unsigned int irq)
233 {
234         void *data = irq_get_handler_data(irq);
235 
236         kfree(data);
237         irq_set_handler_data(irq, NULL);
238         irq_free_descs(irq, 1);
239 }
240 
241 unsigned int irq_alloc(unsigned int dev_handle, unsigned int dev_ino)
242 {
243         int irq;
244 
245         irq = __irq_alloc_descs(-1, 1, 1, numa_node_id(), NULL, NULL);
246         if (irq <= 0)
247                 goto out;
248 
249         return irq;
250 out:
251         return 0;
252 }
253 
254 static unsigned int cookie_exists(u32 devhandle, unsigned int devino)
255 {
256         unsigned long hv_err, cookie;
257         struct ino_bucket *bucket;
258         unsigned int irq = 0U;
259 
260         hv_err = sun4v_vintr_get_cookie(devhandle, devino, &cookie);
261         if (hv_err) {
262                 pr_err("HV get cookie failed hv_err = %ld\n", hv_err);
263                 goto out;
264         }
265 
266         if (cookie & ((1UL << 63UL))) {
267                 cookie = ~cookie;
268                 bucket = (struct ino_bucket *) __va(cookie);
269                 irq = bucket->__irq;
270         }
271 out:
272         return irq;
273 }
274 
275 static unsigned int sysino_exists(u32 devhandle, unsigned int devino)
276 {
277         unsigned long sysino = sun4v_devino_to_sysino(devhandle, devino);
278         struct ino_bucket *bucket;
279         unsigned int irq;
280 
281         bucket = &ivector_table[sysino];
282         irq = bucket_get_irq(__pa(bucket));
283 
284         return irq;
285 }
286 
287 void ack_bad_irq(unsigned int irq)
288 {
289         pr_crit("BAD IRQ ack %d\n", irq);
290 }
291 
292 void irq_install_pre_handler(int irq,
293                              void (*func)(unsigned int, void *, void *),
294                              void *arg1, void *arg2)
295 {
296         pr_warn("IRQ pre handler NOT supported.\n");
297 }
298 
299 /*
300  * /proc/interrupts printing:
301  */
302 int arch_show_interrupts(struct seq_file *p, int prec)
303 {
304         int j;
305 
306         seq_printf(p, "NMI: ");
307         for_each_online_cpu(j)
308                 seq_printf(p, "%10u ", cpu_data(j).__nmi_count);
309         seq_printf(p, "     Non-maskable interrupts\n");
310         return 0;
311 }
312 
313 static unsigned int sun4u_compute_tid(unsigned long imap, unsigned long cpuid)
314 {
315         unsigned int tid;
316 
317         if (this_is_starfire) {
318                 tid = starfire_translate(imap, cpuid);
319                 tid <<= IMAP_TID_SHIFT;
320                 tid &= IMAP_TID_UPA;
321         } else {
322                 if (tlb_type == cheetah || tlb_type == cheetah_plus) {
323                         unsigned long ver;
324 
325                         __asm__ ("rdpr %%ver, %0" : "=r" (ver));
326                         if ((ver >> 32UL) == __JALAPENO_ID ||
327                             (ver >> 32UL) == __SERRANO_ID) {
328                                 tid = cpuid << IMAP_TID_SHIFT;
329                                 tid &= IMAP_TID_JBUS;
330                         } else {
331                                 unsigned int a = cpuid & 0x1f;
332                                 unsigned int n = (cpuid >> 5) & 0x1f;
333 
334                                 tid = ((a << IMAP_AID_SHIFT) |
335                                        (n << IMAP_NID_SHIFT));
336                                 tid &= (IMAP_AID_SAFARI |
337                                         IMAP_NID_SAFARI);
338                         }
339                 } else {
340                         tid = cpuid << IMAP_TID_SHIFT;
341                         tid &= IMAP_TID_UPA;
342                 }
343         }
344 
345         return tid;
346 }
347 
348 #ifdef CONFIG_SMP
349 static int irq_choose_cpu(unsigned int irq, const struct cpumask *affinity)
350 {
351         cpumask_t mask;
352         int cpuid;
353 
354         cpumask_copy(&mask, affinity);
355         if (cpumask_equal(&mask, cpu_online_mask)) {
356                 cpuid = map_to_cpu(irq);
357         } else {
358                 cpumask_t tmp;
359 
360                 cpumask_and(&tmp, cpu_online_mask, &mask);
361                 cpuid = cpumask_empty(&tmp) ? map_to_cpu(irq) : cpumask_first(&tmp);
362         }
363 
364         return cpuid;
365 }
366 #else
367 #define irq_choose_cpu(irq, affinity)   \
368         real_hard_smp_processor_id()
369 #endif
370 
371 static void sun4u_irq_enable(struct irq_data *data)
372 {
373         struct irq_handler_data *handler_data;
374 
375         handler_data = irq_data_get_irq_handler_data(data);
376         if (likely(handler_data)) {
377                 unsigned long cpuid, imap, val;
378                 unsigned int tid;
379 
380                 cpuid = irq_choose_cpu(data->irq,
381                                        irq_data_get_affinity_mask(data));
382                 imap = handler_data->imap;
383 
384                 tid = sun4u_compute_tid(imap, cpuid);
385 
386                 val = upa_readq(imap);
387                 val &= ~(IMAP_TID_UPA | IMAP_TID_JBUS |
388                          IMAP_AID_SAFARI | IMAP_NID_SAFARI);
389                 val |= tid | IMAP_VALID;
390                 upa_writeq(val, imap);
391                 upa_writeq(ICLR_IDLE, handler_data->iclr);
392         }
393 }
394 
395 static int sun4u_set_affinity(struct irq_data *data,
396                                const struct cpumask *mask, bool force)
397 {
398         struct irq_handler_data *handler_data;
399 
400         handler_data = irq_data_get_irq_handler_data(data);
401         if (likely(handler_data)) {
402                 unsigned long cpuid, imap, val;
403                 unsigned int tid;
404 
405                 cpuid = irq_choose_cpu(data->irq, mask);
406                 imap = handler_data->imap;
407 
408                 tid = sun4u_compute_tid(imap, cpuid);
409 
410                 val = upa_readq(imap);
411                 val &= ~(IMAP_TID_UPA | IMAP_TID_JBUS |
412                          IMAP_AID_SAFARI | IMAP_NID_SAFARI);
413                 val |= tid | IMAP_VALID;
414                 upa_writeq(val, imap);
415                 upa_writeq(ICLR_IDLE, handler_data->iclr);
416         }
417 
418         return 0;
419 }
420 
421 /* Don't do anything.  The desc->status check for IRQ_DISABLED in
422  * handler_irq() will skip the handler call and that will leave the
423  * interrupt in the sent state.  The next ->enable() call will hit the
424  * ICLR register to reset the state machine.
425  *
426  * This scheme is necessary, instead of clearing the Valid bit in the
427  * IMAP register, to handle the case of IMAP registers being shared by
428  * multiple INOs (and thus ICLR registers).  Since we use a different
429  * virtual IRQ for each shared IMAP instance, the generic code thinks
430  * there is only one user so it prematurely calls ->disable() on
431  * free_irq().
432  *
433  * We have to provide an explicit ->disable() method instead of using
434  * NULL to get the default.  The reason is that if the generic code
435  * sees that, it also hooks up a default ->shutdown method which
436  * invokes ->mask() which we do not want.  See irq_chip_set_defaults().
437  */
438 static void sun4u_irq_disable(struct irq_data *data)
439 {
440 }
441 
442 static void sun4u_irq_eoi(struct irq_data *data)
443 {
444         struct irq_handler_data *handler_data;
445 
446         handler_data = irq_data_get_irq_handler_data(data);
447         if (likely(handler_data))
448                 upa_writeq(ICLR_IDLE, handler_data->iclr);
449 }
450 
451 static void sun4v_irq_enable(struct irq_data *data)
452 {
453         unsigned long cpuid = irq_choose_cpu(data->irq,
454                                              irq_data_get_affinity_mask(data));
455         unsigned int ino = irq_data_to_sysino(data);
456         int err;
457 
458         err = sun4v_intr_settarget(ino, cpuid);
459         if (err != HV_EOK)
460                 printk(KERN_ERR "sun4v_intr_settarget(%x,%lu): "
461                        "err(%d)\n", ino, cpuid, err);
462         err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE);
463         if (err != HV_EOK)
464                 printk(KERN_ERR "sun4v_intr_setstate(%x): "
465                        "err(%d)\n", ino, err);
466         err = sun4v_intr_setenabled(ino, HV_INTR_ENABLED);
467         if (err != HV_EOK)
468                 printk(KERN_ERR "sun4v_intr_setenabled(%x): err(%d)\n",
469                        ino, err);
470 }
471 
472 static int sun4v_set_affinity(struct irq_data *data,
473                                const struct cpumask *mask, bool force)
474 {
475         unsigned long cpuid = irq_choose_cpu(data->irq, mask);
476         unsigned int ino = irq_data_to_sysino(data);
477         int err;
478 
479         err = sun4v_intr_settarget(ino, cpuid);
480         if (err != HV_EOK)
481                 printk(KERN_ERR "sun4v_intr_settarget(%x,%lu): "
482                        "err(%d)\n", ino, cpuid, err);
483 
484         return 0;
485 }
486 
487 static void sun4v_irq_disable(struct irq_data *data)
488 {
489         unsigned int ino = irq_data_to_sysino(data);
490         int err;
491 
492         err = sun4v_intr_setenabled(ino, HV_INTR_DISABLED);
493         if (err != HV_EOK)
494                 printk(KERN_ERR "sun4v_intr_setenabled(%x): "
495                        "err(%d)\n", ino, err);
496 }
497 
498 static void sun4v_irq_eoi(struct irq_data *data)
499 {
500         unsigned int ino = irq_data_to_sysino(data);
501         int err;
502 
503         err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE);
504         if (err != HV_EOK)
505                 printk(KERN_ERR "sun4v_intr_setstate(%x): "
506                        "err(%d)\n", ino, err);
507 }
508 
509 static void sun4v_virq_enable(struct irq_data *data)
510 {
511         unsigned long dev_handle = irq_data_to_handle(data);
512         unsigned long dev_ino = irq_data_to_ino(data);
513         unsigned long cpuid;
514         int err;
515 
516         cpuid = irq_choose_cpu(data->irq, irq_data_get_affinity_mask(data));
517 
518         err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid);
519         if (err != HV_EOK)
520                 printk(KERN_ERR "sun4v_vintr_set_target(%lx,%lx,%lu): "
521                        "err(%d)\n",
522                        dev_handle, dev_ino, cpuid, err);
523         err = sun4v_vintr_set_state(dev_handle, dev_ino,
524                                     HV_INTR_STATE_IDLE);
525         if (err != HV_EOK)
526                 printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx,"
527                        "HV_INTR_STATE_IDLE): err(%d)\n",
528                        dev_handle, dev_ino, err);
529         err = sun4v_vintr_set_valid(dev_handle, dev_ino,
530                                     HV_INTR_ENABLED);
531         if (err != HV_EOK)
532                 printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx,"
533                        "HV_INTR_ENABLED): err(%d)\n",
534                        dev_handle, dev_ino, err);
535 }
536 
537 static int sun4v_virt_set_affinity(struct irq_data *data,
538                                     const struct cpumask *mask, bool force)
539 {
540         unsigned long dev_handle = irq_data_to_handle(data);
541         unsigned long dev_ino = irq_data_to_ino(data);
542         unsigned long cpuid;
543         int err;
544 
545         cpuid = irq_choose_cpu(data->irq, mask);
546 
547         err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid);
548         if (err != HV_EOK)
549                 printk(KERN_ERR "sun4v_vintr_set_target(%lx,%lx,%lu): "
550                        "err(%d)\n",
551                        dev_handle, dev_ino, cpuid, err);
552 
553         return 0;
554 }
555 
556 static void sun4v_virq_disable(struct irq_data *data)
557 {
558         unsigned long dev_handle = irq_data_to_handle(data);
559         unsigned long dev_ino = irq_data_to_ino(data);
560         int err;
561 
562 
563         err = sun4v_vintr_set_valid(dev_handle, dev_ino,
564                                     HV_INTR_DISABLED);
565         if (err != HV_EOK)
566                 printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx,"
567                        "HV_INTR_DISABLED): err(%d)\n",
568                        dev_handle, dev_ino, err);
569 }
570 
571 static void sun4v_virq_eoi(struct irq_data *data)
572 {
573         unsigned long dev_handle = irq_data_to_handle(data);
574         unsigned long dev_ino = irq_data_to_ino(data);
575         int err;
576 
577         err = sun4v_vintr_set_state(dev_handle, dev_ino,
578                                     HV_INTR_STATE_IDLE);
579         if (err != HV_EOK)
580                 printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx,"
581                        "HV_INTR_STATE_IDLE): err(%d)\n",
582                        dev_handle, dev_ino, err);
583 }
584 
585 static struct irq_chip sun4u_irq = {
586         .name                   = "sun4u",
587         .irq_enable             = sun4u_irq_enable,
588         .irq_disable            = sun4u_irq_disable,
589         .irq_eoi                = sun4u_irq_eoi,
590         .irq_set_affinity       = sun4u_set_affinity,
591         .flags                  = IRQCHIP_EOI_IF_HANDLED,
592 };
593 
594 static struct irq_chip sun4v_irq = {
595         .name                   = "sun4v",
596         .irq_enable             = sun4v_irq_enable,
597         .irq_disable            = sun4v_irq_disable,
598         .irq_eoi                = sun4v_irq_eoi,
599         .irq_set_affinity       = sun4v_set_affinity,
600         .flags                  = IRQCHIP_EOI_IF_HANDLED,
601 };
602 
603 static struct irq_chip sun4v_virq = {
604         .name                   = "vsun4v",
605         .irq_enable             = sun4v_virq_enable,
606         .irq_disable            = sun4v_virq_disable,
607         .irq_eoi                = sun4v_virq_eoi,
608         .irq_set_affinity       = sun4v_virt_set_affinity,
609         .flags                  = IRQCHIP_EOI_IF_HANDLED,
610 };
611 
612 unsigned int build_irq(int inofixup, unsigned long iclr, unsigned long imap)
613 {
614         struct irq_handler_data *handler_data;
615         struct ino_bucket *bucket;
616         unsigned int irq;
617         int ino;
618 
619         BUG_ON(tlb_type == hypervisor);
620 
621         ino = (upa_readq(imap) & (IMAP_IGN | IMAP_INO)) + inofixup;
622         bucket = &ivector_table[ino];
623         irq = bucket_get_irq(__pa(bucket));
624         if (!irq) {
625                 irq = irq_alloc(0, ino);
626                 bucket_set_irq(__pa(bucket), irq);
627                 irq_set_chip_and_handler_name(irq, &sun4u_irq,
628                                               handle_fasteoi_irq, "IVEC");
629         }
630 
631         handler_data = irq_get_handler_data(irq);
632         if (unlikely(handler_data))
633                 goto out;
634 
635         handler_data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC);
636         if (unlikely(!handler_data)) {
637                 prom_printf("IRQ: kzalloc(irq_handler_data) failed.\n");
638                 prom_halt();
639         }
640         irq_set_handler_data(irq, handler_data);
641 
642         handler_data->imap  = imap;
643         handler_data->iclr  = iclr;
644 
645 out:
646         return irq;
647 }
648 
649 static unsigned int sun4v_build_common(u32 devhandle, unsigned int devino,
650                 void (*handler_data_init)(struct irq_handler_data *data,
651                 u32 devhandle, unsigned int devino),
652                 struct irq_chip *chip)
653 {
654         struct irq_handler_data *data;
655         unsigned int irq;
656 
657         irq = irq_alloc(devhandle, devino);
658         if (!irq)
659                 goto out;
660 
661         data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC);
662         if (unlikely(!data)) {
663                 pr_err("IRQ handler data allocation failed.\n");
664                 irq_free(irq);
665                 irq = 0;
666                 goto out;
667         }
668 
669         irq_set_handler_data(irq, data);
670         handler_data_init(data, devhandle, devino);
671         irq_set_chip_and_handler_name(irq, chip, handle_fasteoi_irq, "IVEC");
672         data->imap = ~0UL;
673         data->iclr = ~0UL;
674 out:
675         return irq;
676 }
677 
678 static unsigned long cookie_assign(unsigned int irq, u32 devhandle,
679                 unsigned int devino)
680 {
681         struct irq_handler_data *ihd = irq_get_handler_data(irq);
682         unsigned long hv_error, cookie;
683 
684         /* handler_irq needs to find the irq. cookie is seen signed in
685          * sun4v_dev_mondo and treated as a non ivector_table delivery.
686          */
687         ihd->bucket.__irq = irq;
688         cookie = ~__pa(&ihd->bucket);
689 
690         hv_error = sun4v_vintr_set_cookie(devhandle, devino, cookie);
691         if (hv_error)
692                 pr_err("HV vintr set cookie failed = %ld\n", hv_error);
693 
694         return hv_error;
695 }
696 
697 static void cookie_handler_data(struct irq_handler_data *data,
698                                 u32 devhandle, unsigned int devino)
699 {
700         data->dev_handle = devhandle;
701         data->dev_ino = devino;
702 }
703 
704 static unsigned int cookie_build_irq(u32 devhandle, unsigned int devino,
705                                      struct irq_chip *chip)
706 {
707         unsigned long hv_error;
708         unsigned int irq;
709 
710         irq = sun4v_build_common(devhandle, devino, cookie_handler_data, chip);
711 
712         hv_error = cookie_assign(irq, devhandle, devino);
713         if (hv_error) {
714                 irq_free(irq);
715                 irq = 0;
716         }
717 
718         return irq;
719 }
720 
721 static unsigned int sun4v_build_cookie(u32 devhandle, unsigned int devino)
722 {
723         unsigned int irq;
724 
725         irq = cookie_exists(devhandle, devino);
726         if (irq)
727                 goto out;
728 
729         irq = cookie_build_irq(devhandle, devino, &sun4v_virq);
730 
731 out:
732         return irq;
733 }
734 
735 static void sysino_set_bucket(unsigned int irq)
736 {
737         struct irq_handler_data *ihd = irq_get_handler_data(irq);
738         struct ino_bucket *bucket;
739         unsigned long sysino;
740 
741         sysino = sun4v_devino_to_sysino(ihd->dev_handle, ihd->dev_ino);
742         BUG_ON(sysino >= nr_ivec);
743         bucket = &ivector_table[sysino];
744         bucket_set_irq(__pa(bucket), irq);
745 }
746 
747 static void sysino_handler_data(struct irq_handler_data *data,
748                                 u32 devhandle, unsigned int devino)
749 {
750         unsigned long sysino;
751 
752         sysino = sun4v_devino_to_sysino(devhandle, devino);
753         data->sysino = sysino;
754 }
755 
756 static unsigned int sysino_build_irq(u32 devhandle, unsigned int devino,
757                                      struct irq_chip *chip)
758 {
759         unsigned int irq;
760 
761         irq = sun4v_build_common(devhandle, devino, sysino_handler_data, chip);
762         if (!irq)
763                 goto out;
764 
765         sysino_set_bucket(irq);
766 out:
767         return irq;
768 }
769 
770 static int sun4v_build_sysino(u32 devhandle, unsigned int devino)
771 {
772         int irq;
773 
774         irq = sysino_exists(devhandle, devino);
775         if (irq)
776                 goto out;
777 
778         irq = sysino_build_irq(devhandle, devino, &sun4v_irq);
779 out:
780         return irq;
781 }
782 
783 unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino)
784 {
785         unsigned int irq;
786 
787         if (sun4v_cookie_only_virqs())
788                 irq = sun4v_build_cookie(devhandle, devino);
789         else
790                 irq = sun4v_build_sysino(devhandle, devino);
791 
792         return irq;
793 }
794 
795 unsigned int sun4v_build_virq(u32 devhandle, unsigned int devino)
796 {
797         int irq;
798 
799         irq = cookie_build_irq(devhandle, devino, &sun4v_virq);
800         if (!irq)
801                 goto out;
802 
803         /* This is borrowed from the original function.
804          */
805         irq_set_status_flags(irq, IRQ_NOAUTOEN);
806 
807 out:
808         return irq;
809 }
810 
811 void *hardirq_stack[NR_CPUS];
812 void *softirq_stack[NR_CPUS];
813 
814 void __irq_entry handler_irq(int pil, struct pt_regs *regs)
815 {
816         unsigned long pstate, bucket_pa;
817         struct pt_regs *old_regs;
818         void *orig_sp;
819 
820         clear_softint(1 << pil);
821 
822         old_regs = set_irq_regs(regs);
823         irq_enter();
824 
825         /* Grab an atomic snapshot of the pending IVECs.  */
826         __asm__ __volatile__("rdpr      %%pstate, %0\n\t"
827                              "wrpr      %0, %3, %%pstate\n\t"
828                              "ldx       [%2], %1\n\t"
829                              "stx       %%g0, [%2]\n\t"
830                              "wrpr      %0, 0x0, %%pstate\n\t"
831                              : "=&r" (pstate), "=&r" (bucket_pa)
832                              : "r" (irq_work_pa(smp_processor_id())),
833                                "i" (PSTATE_IE)
834                              : "memory");
835 
836         orig_sp = set_hardirq_stack();
837 
838         while (bucket_pa) {
839                 unsigned long next_pa;
840                 unsigned int irq;
841 
842                 next_pa = bucket_get_chain_pa(bucket_pa);
843                 irq = bucket_get_irq(bucket_pa);
844                 bucket_clear_chain_pa(bucket_pa);
845 
846                 generic_handle_irq(irq);
847 
848                 bucket_pa = next_pa;
849         }
850 
851         restore_hardirq_stack(orig_sp);
852 
853         irq_exit();
854         set_irq_regs(old_regs);
855 }
856 
857 void do_softirq_own_stack(void)
858 {
859         void *orig_sp, *sp = softirq_stack[smp_processor_id()];
860 
861         sp += THREAD_SIZE - 192 - STACK_BIAS;
862 
863         __asm__ __volatile__("mov %%sp, %0\n\t"
864                              "mov %1, %%sp"
865                              : "=&r" (orig_sp)
866                              : "r" (sp));
867         __do_softirq();
868         __asm__ __volatile__("mov %0, %%sp"
869                              : : "r" (orig_sp));
870 }
871 
872 #ifdef CONFIG_HOTPLUG_CPU
873 void fixup_irqs(void)
874 {
875         unsigned int irq;
876 
877         for (irq = 0; irq < NR_IRQS; irq++) {
878                 struct irq_desc *desc = irq_to_desc(irq);
879                 struct irq_data *data;
880                 unsigned long flags;
881 
882                 if (!desc)
883                         continue;
884                 data = irq_desc_get_irq_data(desc);
885                 raw_spin_lock_irqsave(&desc->lock, flags);
886                 if (desc->action && !irqd_is_per_cpu(data)) {
887                         if (data->chip->irq_set_affinity)
888                                 data->chip->irq_set_affinity(data,
889                                         irq_data_get_affinity_mask(data),
890                                         false);
891                 }
892                 raw_spin_unlock_irqrestore(&desc->lock, flags);
893         }
894 
895         tick_ops->disable_irq();
896 }
897 #endif
898 
899 struct sun5_timer {
900         u64     count0;
901         u64     limit0;
902         u64     count1;
903         u64     limit1;
904 };
905 
906 static struct sun5_timer *prom_timers;
907 static u64 prom_limit0, prom_limit1;
908 
909 static void map_prom_timers(void)
910 {
911         struct device_node *dp;
912         const unsigned int *addr;
913 
914         /* PROM timer node hangs out in the top level of device siblings... */
915         dp = of_find_node_by_path("/");
916         dp = dp->child;
917         while (dp) {
918                 if (!strcmp(dp->name, "counter-timer"))
919                         break;
920                 dp = dp->sibling;
921         }
922 
923         /* Assume if node is not present, PROM uses different tick mechanism
924          * which we should not care about.
925          */
926         if (!dp) {
927                 prom_timers = (struct sun5_timer *) 0;
928                 return;
929         }
930 
931         /* If PROM is really using this, it must be mapped by him. */
932         addr = of_get_property(dp, "address", NULL);
933         if (!addr) {
934                 prom_printf("PROM does not have timer mapped, trying to continue.\n");
935                 prom_timers = (struct sun5_timer *) 0;
936                 return;
937         }
938         prom_timers = (struct sun5_timer *) ((unsigned long)addr[0]);
939 }
940 
941 static void kill_prom_timer(void)
942 {
943         if (!prom_timers)
944                 return;
945 
946         /* Save them away for later. */
947         prom_limit0 = prom_timers->limit0;
948         prom_limit1 = prom_timers->limit1;
949 
950         /* Just as in sun4c PROM uses timer which ticks at IRQ 14.
951          * We turn both off here just to be paranoid.
952          */
953         prom_timers->limit0 = 0;
954         prom_timers->limit1 = 0;
955 
956         /* Wheee, eat the interrupt packet too... */
957         __asm__ __volatile__(
958 "       mov     0x40, %%g2\n"
959 "       ldxa    [%%g0] %0, %%g1\n"
960 "       ldxa    [%%g2] %1, %%g1\n"
961 "       stxa    %%g0, [%%g0] %0\n"
962 "       membar  #Sync\n"
963         : /* no outputs */
964         : "i" (ASI_INTR_RECEIVE), "i" (ASI_INTR_R)
965         : "g1", "g2");
966 }
967 
968 void notrace init_irqwork_curcpu(void)
969 {
970         int cpu = hard_smp_processor_id();
971 
972         trap_block[cpu].irq_worklist_pa = 0UL;
973 }
974 
975 /* Please be very careful with register_one_mondo() and
976  * sun4v_register_mondo_queues().
977  *
978  * On SMP this gets invoked from the CPU trampoline before
979  * the cpu has fully taken over the trap table from OBP,
980  * and it's kernel stack + %g6 thread register state is
981  * not fully cooked yet.
982  *
983  * Therefore you cannot make any OBP calls, not even prom_printf,
984  * from these two routines.
985  */
986 static void notrace register_one_mondo(unsigned long paddr, unsigned long type,
987                                        unsigned long qmask)
988 {
989         unsigned long num_entries = (qmask + 1) / 64;
990         unsigned long status;
991 
992         status = sun4v_cpu_qconf(type, paddr, num_entries);
993         if (status != HV_EOK) {
994                 prom_printf("SUN4V: sun4v_cpu_qconf(%lu:%lx:%lu) failed, "
995                             "err %lu\n", type, paddr, num_entries, status);
996                 prom_halt();
997         }
998 }
999 
1000 void notrace sun4v_register_mondo_queues(int this_cpu)
1001 {
1002         struct trap_per_cpu *tb = &trap_block[this_cpu];
1003 
1004         register_one_mondo(tb->cpu_mondo_pa, HV_CPU_QUEUE_CPU_MONDO,
1005                            tb->cpu_mondo_qmask);
1006         register_one_mondo(tb->dev_mondo_pa, HV_CPU_QUEUE_DEVICE_MONDO,
1007                            tb->dev_mondo_qmask);
1008         register_one_mondo(tb->resum_mondo_pa, HV_CPU_QUEUE_RES_ERROR,
1009                            tb->resum_qmask);
1010         register_one_mondo(tb->nonresum_mondo_pa, HV_CPU_QUEUE_NONRES_ERROR,
1011                            tb->nonresum_qmask);
1012 }
1013 
1014 /* Each queue region must be a power of 2 multiple of 64 bytes in
1015  * size.  The base real address must be aligned to the size of the
1016  * region.  Thus, an 8KB queue must be 8KB aligned, for example.
1017  */
1018 static void __init alloc_one_queue(unsigned long *pa_ptr, unsigned long qmask)
1019 {
1020         unsigned long size = PAGE_ALIGN(qmask + 1);
1021         unsigned long order = get_order(size);
1022         unsigned long p;
1023 
1024         p = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
1025         if (!p) {
1026                 prom_printf("SUN4V: Error, cannot allocate queue.\n");
1027                 prom_halt();
1028         }
1029 
1030         *pa_ptr = __pa(p);
1031 }
1032 
1033 static void __init init_cpu_send_mondo_info(struct trap_per_cpu *tb)
1034 {
1035 #ifdef CONFIG_SMP
1036         unsigned long page;
1037         void *mondo, *p;
1038 
1039         BUILD_BUG_ON((NR_CPUS * sizeof(u16)) > PAGE_SIZE);
1040 
1041         /* Make sure mondo block is 64byte aligned */
1042         p = kzalloc(127, GFP_KERNEL);
1043         if (!p) {
1044                 prom_printf("SUN4V: Error, cannot allocate mondo block.\n");
1045                 prom_halt();
1046         }
1047         mondo = (void *)(((unsigned long)p + 63) & ~0x3f);
1048         tb->cpu_mondo_block_pa = __pa(mondo);
1049 
1050         page = get_zeroed_page(GFP_KERNEL);
1051         if (!page) {
1052                 prom_printf("SUN4V: Error, cannot allocate cpu list page.\n");
1053                 prom_halt();
1054         }
1055 
1056         tb->cpu_list_pa = __pa(page);
1057 #endif
1058 }
1059 
1060 /* Allocate mondo and error queues for all possible cpus.  */
1061 static void __init sun4v_init_mondo_queues(void)
1062 {
1063         int cpu;
1064 
1065         for_each_possible_cpu(cpu) {
1066                 struct trap_per_cpu *tb = &trap_block[cpu];
1067 
1068                 alloc_one_queue(&tb->cpu_mondo_pa, tb->cpu_mondo_qmask);
1069                 alloc_one_queue(&tb->dev_mondo_pa, tb->dev_mondo_qmask);
1070                 alloc_one_queue(&tb->resum_mondo_pa, tb->resum_qmask);
1071                 alloc_one_queue(&tb->resum_kernel_buf_pa, tb->resum_qmask);
1072                 alloc_one_queue(&tb->nonresum_mondo_pa, tb->nonresum_qmask);
1073                 alloc_one_queue(&tb->nonresum_kernel_buf_pa,
1074                                 tb->nonresum_qmask);
1075         }
1076 }
1077 
1078 static void __init init_send_mondo_info(void)
1079 {
1080         int cpu;
1081 
1082         for_each_possible_cpu(cpu) {
1083                 struct trap_per_cpu *tb = &trap_block[cpu];
1084 
1085                 init_cpu_send_mondo_info(tb);
1086         }
1087 }
1088 
1089 static struct irqaction timer_irq_action = {
1090         .name = "timer",
1091 };
1092 
1093 static void __init irq_ivector_init(void)
1094 {
1095         unsigned long size, order;
1096         unsigned int ivecs;
1097 
1098         /* If we are doing cookie only VIRQs then we do not need the ivector
1099          * table to process interrupts.
1100          */
1101         if (sun4v_cookie_only_virqs())
1102                 return;
1103 
1104         ivecs = size_nr_ivec();
1105         size = sizeof(struct ino_bucket) * ivecs;
1106         order = get_order(size);
1107         ivector_table = (struct ino_bucket *)
1108                 __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
1109         if (!ivector_table) {
1110                 prom_printf("Fatal error, cannot allocate ivector_table\n");
1111                 prom_halt();
1112         }
1113         __flush_dcache_range((unsigned long) ivector_table,
1114                              ((unsigned long) ivector_table) + size);
1115 
1116         ivector_table_pa = __pa(ivector_table);
1117 }
1118 
1119 /* Only invoked on boot processor.*/
1120 void __init init_IRQ(void)
1121 {
1122         irq_init_hv();
1123         irq_ivector_init();
1124         map_prom_timers();
1125         kill_prom_timer();
1126 
1127         if (tlb_type == hypervisor)
1128                 sun4v_init_mondo_queues();
1129 
1130         init_send_mondo_info();
1131 
1132         if (tlb_type == hypervisor) {
1133                 /* Load up the boot cpu's entries.  */
1134                 sun4v_register_mondo_queues(hard_smp_processor_id());
1135         }
1136 
1137         /* We need to clear any IRQ's pending in the soft interrupt
1138          * registers, a spurious one could be left around from the
1139          * PROM timer which we just disabled.
1140          */
1141         clear_softint(get_softint());
1142 
1143         /* Now that ivector table is initialized, it is safe
1144          * to receive IRQ vector traps.  We will normally take
1145          * one or two right now, in case some device PROM used
1146          * to boot us wants to speak to us.  We just ignore them.
1147          */
1148         __asm__ __volatile__("rdpr      %%pstate, %%g1\n\t"
1149                              "or        %%g1, %0, %%g1\n\t"
1150                              "wrpr      %%g1, 0x0, %%pstate"
1151                              : /* No outputs */
1152                              : "i" (PSTATE_IE)
1153                              : "g1");
1154 
1155         irq_to_desc(0)->action = &timer_irq_action;
1156 }
1157 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp