~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/mips/cavium-octeon/octeon-irq.c

Version: ~ [ linux-5.4-rc3 ] ~ [ linux-5.3.6 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.79 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.149 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.196 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.196 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.75 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * This file is subject to the terms and conditions of the GNU General Public
  3  * License.  See the file "COPYING" in the main directory of this archive
  4  * for more details.
  5  *
  6  * Copyright (C) 2004-2014 Cavium, Inc.
  7  */
  8 
  9 #include <linux/of_address.h>
 10 #include <linux/interrupt.h>
 11 #include <linux/irqdomain.h>
 12 #include <linux/bitops.h>
 13 #include <linux/of_irq.h>
 14 #include <linux/percpu.h>
 15 #include <linux/slab.h>
 16 #include <linux/irq.h>
 17 #include <linux/smp.h>
 18 #include <linux/of.h>
 19 
 20 #include <asm/octeon/octeon.h>
 21 #include <asm/octeon/cvmx-ciu2-defs.h>
 22 
 23 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu0_en_mirror);
 24 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu1_en_mirror);
 25 static DEFINE_PER_CPU(raw_spinlock_t, octeon_irq_ciu_spinlock);
 26 
 27 struct octeon_irq_ciu_domain_data {
 28         int num_sum;  /* number of sum registers (2 or 3). */
 29 };
 30 
 31 static __read_mostly u8 octeon_irq_ciu_to_irq[8][64];
 32 
 33 struct octeon_ciu_chip_data {
 34         union {
 35                 struct {                /* only used for ciu3 */
 36                         u64 ciu3_addr;
 37                         unsigned int intsn;
 38                 };
 39                 struct {                /* only used for ciu/ciu2 */
 40                         u8 line;
 41                         u8 bit;
 42                         u8 gpio_line;
 43                 };
 44         };
 45         int current_cpu;        /* Next CPU expected to take this irq */
 46 };
 47 
 48 struct octeon_core_chip_data {
 49         struct mutex core_irq_mutex;
 50         bool current_en;
 51         bool desired_en;
 52         u8 bit;
 53 };
 54 
 55 #define MIPS_CORE_IRQ_LINES 8
 56 
 57 static struct octeon_core_chip_data octeon_irq_core_chip_data[MIPS_CORE_IRQ_LINES];
 58 
 59 static int octeon_irq_set_ciu_mapping(int irq, int line, int bit, int gpio_line,
 60                                       struct irq_chip *chip,
 61                                       irq_flow_handler_t handler)
 62 {
 63         struct octeon_ciu_chip_data *cd;
 64 
 65         cd = kzalloc(sizeof(*cd), GFP_KERNEL);
 66         if (!cd)
 67                 return -ENOMEM;
 68 
 69         irq_set_chip_and_handler(irq, chip, handler);
 70 
 71         cd->line = line;
 72         cd->bit = bit;
 73         cd->gpio_line = gpio_line;
 74 
 75         irq_set_chip_data(irq, cd);
 76         octeon_irq_ciu_to_irq[line][bit] = irq;
 77         return 0;
 78 }
 79 
 80 static void octeon_irq_free_cd(struct irq_domain *d, unsigned int irq)
 81 {
 82         struct irq_data *data = irq_get_irq_data(irq);
 83         struct octeon_ciu_chip_data *cd = irq_data_get_irq_chip_data(data);
 84 
 85         irq_set_chip_data(irq, NULL);
 86         kfree(cd);
 87 }
 88 
 89 static int octeon_irq_force_ciu_mapping(struct irq_domain *domain,
 90                                         int irq, int line, int bit)
 91 {
 92         return irq_domain_associate(domain, irq, line << 6 | bit);
 93 }
 94 
 95 static int octeon_coreid_for_cpu(int cpu)
 96 {
 97 #ifdef CONFIG_SMP
 98         return cpu_logical_map(cpu);
 99 #else
100         return cvmx_get_core_num();
101 #endif
102 }
103 
104 static int octeon_cpu_for_coreid(int coreid)
105 {
106 #ifdef CONFIG_SMP
107         return cpu_number_map(coreid);
108 #else
109         return smp_processor_id();
110 #endif
111 }
112 
113 static void octeon_irq_core_ack(struct irq_data *data)
114 {
115         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
116         unsigned int bit = cd->bit;
117 
118         /*
119          * We don't need to disable IRQs to make these atomic since
120          * they are already disabled earlier in the low level
121          * interrupt code.
122          */
123         clear_c0_status(0x100 << bit);
124         /* The two user interrupts must be cleared manually. */
125         if (bit < 2)
126                 clear_c0_cause(0x100 << bit);
127 }
128 
129 static void octeon_irq_core_eoi(struct irq_data *data)
130 {
131         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
132 
133         /*
134          * We don't need to disable IRQs to make these atomic since
135          * they are already disabled earlier in the low level
136          * interrupt code.
137          */
138         set_c0_status(0x100 << cd->bit);
139 }
140 
141 static void octeon_irq_core_set_enable_local(void *arg)
142 {
143         struct irq_data *data = arg;
144         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
145         unsigned int mask = 0x100 << cd->bit;
146 
147         /*
148          * Interrupts are already disabled, so these are atomic.
149          */
150         if (cd->desired_en)
151                 set_c0_status(mask);
152         else
153                 clear_c0_status(mask);
154 
155 }
156 
157 static void octeon_irq_core_disable(struct irq_data *data)
158 {
159         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
160         cd->desired_en = false;
161 }
162 
163 static void octeon_irq_core_enable(struct irq_data *data)
164 {
165         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
166         cd->desired_en = true;
167 }
168 
169 static void octeon_irq_core_bus_lock(struct irq_data *data)
170 {
171         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
172 
173         mutex_lock(&cd->core_irq_mutex);
174 }
175 
176 static void octeon_irq_core_bus_sync_unlock(struct irq_data *data)
177 {
178         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
179 
180         if (cd->desired_en != cd->current_en) {
181                 on_each_cpu(octeon_irq_core_set_enable_local, data, 1);
182 
183                 cd->current_en = cd->desired_en;
184         }
185 
186         mutex_unlock(&cd->core_irq_mutex);
187 }
188 
189 static struct irq_chip octeon_irq_chip_core = {
190         .name = "Core",
191         .irq_enable = octeon_irq_core_enable,
192         .irq_disable = octeon_irq_core_disable,
193         .irq_ack = octeon_irq_core_ack,
194         .irq_eoi = octeon_irq_core_eoi,
195         .irq_bus_lock = octeon_irq_core_bus_lock,
196         .irq_bus_sync_unlock = octeon_irq_core_bus_sync_unlock,
197 
198         .irq_cpu_online = octeon_irq_core_eoi,
199         .irq_cpu_offline = octeon_irq_core_ack,
200         .flags = IRQCHIP_ONOFFLINE_ENABLED,
201 };
202 
203 static void __init octeon_irq_init_core(void)
204 {
205         int i;
206         int irq;
207         struct octeon_core_chip_data *cd;
208 
209         for (i = 0; i < MIPS_CORE_IRQ_LINES; i++) {
210                 cd = &octeon_irq_core_chip_data[i];
211                 cd->current_en = false;
212                 cd->desired_en = false;
213                 cd->bit = i;
214                 mutex_init(&cd->core_irq_mutex);
215 
216                 irq = OCTEON_IRQ_SW0 + i;
217                 irq_set_chip_data(irq, cd);
218                 irq_set_chip_and_handler(irq, &octeon_irq_chip_core,
219                                          handle_percpu_irq);
220         }
221 }
222 
223 static int next_cpu_for_irq(struct irq_data *data)
224 {
225 
226 #ifdef CONFIG_SMP
227         int cpu;
228         int weight = cpumask_weight(data->affinity);
229         struct octeon_ciu_chip_data *cd = irq_data_get_irq_chip_data(data);
230 
231         if (weight > 1) {
232                 cpu = cd->current_cpu;
233                 for (;;) {
234                         cpu = cpumask_next(cpu, data->affinity);
235                         if (cpu >= nr_cpu_ids) {
236                                 cpu = -1;
237                                 continue;
238                         } else if (cpumask_test_cpu(cpu, cpu_online_mask)) {
239                                 break;
240                         }
241                 }
242         } else if (weight == 1) {
243                 cpu = cpumask_first(data->affinity);
244         } else {
245                 cpu = smp_processor_id();
246         }
247         cd->current_cpu = cpu;
248         return cpu;
249 #else
250         return smp_processor_id();
251 #endif
252 }
253 
254 static void octeon_irq_ciu_enable(struct irq_data *data)
255 {
256         int cpu = next_cpu_for_irq(data);
257         int coreid = octeon_coreid_for_cpu(cpu);
258         unsigned long *pen;
259         unsigned long flags;
260         struct octeon_ciu_chip_data *cd;
261         raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
262 
263         cd = irq_data_get_irq_chip_data(data);
264 
265         raw_spin_lock_irqsave(lock, flags);
266         if (cd->line == 0) {
267                 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
268                 __set_bit(cd->bit, pen);
269                 /*
270                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
271                  * enabling the irq.
272                  */
273                 wmb();
274                 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
275         } else {
276                 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
277                 __set_bit(cd->bit, pen);
278                 /*
279                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
280                  * enabling the irq.
281                  */
282                 wmb();
283                 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
284         }
285         raw_spin_unlock_irqrestore(lock, flags);
286 }
287 
288 static void octeon_irq_ciu_enable_local(struct irq_data *data)
289 {
290         unsigned long *pen;
291         unsigned long flags;
292         struct octeon_ciu_chip_data *cd;
293         raw_spinlock_t *lock = this_cpu_ptr(&octeon_irq_ciu_spinlock);
294 
295         cd = irq_data_get_irq_chip_data(data);
296 
297         raw_spin_lock_irqsave(lock, flags);
298         if (cd->line == 0) {
299                 pen = this_cpu_ptr(&octeon_irq_ciu0_en_mirror);
300                 __set_bit(cd->bit, pen);
301                 /*
302                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
303                  * enabling the irq.
304                  */
305                 wmb();
306                 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen);
307         } else {
308                 pen = this_cpu_ptr(&octeon_irq_ciu1_en_mirror);
309                 __set_bit(cd->bit, pen);
310                 /*
311                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
312                  * enabling the irq.
313                  */
314                 wmb();
315                 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen);
316         }
317         raw_spin_unlock_irqrestore(lock, flags);
318 }
319 
320 static void octeon_irq_ciu_disable_local(struct irq_data *data)
321 {
322         unsigned long *pen;
323         unsigned long flags;
324         struct octeon_ciu_chip_data *cd;
325         raw_spinlock_t *lock = this_cpu_ptr(&octeon_irq_ciu_spinlock);
326 
327         cd = irq_data_get_irq_chip_data(data);
328 
329         raw_spin_lock_irqsave(lock, flags);
330         if (cd->line == 0) {
331                 pen = this_cpu_ptr(&octeon_irq_ciu0_en_mirror);
332                 __clear_bit(cd->bit, pen);
333                 /*
334                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
335                  * enabling the irq.
336                  */
337                 wmb();
338                 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen);
339         } else {
340                 pen = this_cpu_ptr(&octeon_irq_ciu1_en_mirror);
341                 __clear_bit(cd->bit, pen);
342                 /*
343                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
344                  * enabling the irq.
345                  */
346                 wmb();
347                 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen);
348         }
349         raw_spin_unlock_irqrestore(lock, flags);
350 }
351 
352 static void octeon_irq_ciu_disable_all(struct irq_data *data)
353 {
354         unsigned long flags;
355         unsigned long *pen;
356         int cpu;
357         struct octeon_ciu_chip_data *cd;
358         raw_spinlock_t *lock;
359 
360         cd = irq_data_get_irq_chip_data(data);
361 
362         for_each_online_cpu(cpu) {
363                 int coreid = octeon_coreid_for_cpu(cpu);
364                 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
365                 if (cd->line == 0)
366                         pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
367                 else
368                         pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
369 
370                 raw_spin_lock_irqsave(lock, flags);
371                 __clear_bit(cd->bit, pen);
372                 /*
373                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
374                  * enabling the irq.
375                  */
376                 wmb();
377                 if (cd->line == 0)
378                         cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
379                 else
380                         cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
381                 raw_spin_unlock_irqrestore(lock, flags);
382         }
383 }
384 
385 static void octeon_irq_ciu_enable_all(struct irq_data *data)
386 {
387         unsigned long flags;
388         unsigned long *pen;
389         int cpu;
390         struct octeon_ciu_chip_data *cd;
391         raw_spinlock_t *lock;
392 
393         cd = irq_data_get_irq_chip_data(data);
394 
395         for_each_online_cpu(cpu) {
396                 int coreid = octeon_coreid_for_cpu(cpu);
397                 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
398                 if (cd->line == 0)
399                         pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
400                 else
401                         pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
402 
403                 raw_spin_lock_irqsave(lock, flags);
404                 __set_bit(cd->bit, pen);
405                 /*
406                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
407                  * enabling the irq.
408                  */
409                 wmb();
410                 if (cd->line == 0)
411                         cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
412                 else
413                         cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
414                 raw_spin_unlock_irqrestore(lock, flags);
415         }
416 }
417 
418 /*
419  * Enable the irq on the next core in the affinity set for chips that
420  * have the EN*_W1{S,C} registers.
421  */
422 static void octeon_irq_ciu_enable_v2(struct irq_data *data)
423 {
424         u64 mask;
425         int cpu = next_cpu_for_irq(data);
426         struct octeon_ciu_chip_data *cd;
427 
428         cd = irq_data_get_irq_chip_data(data);
429         mask = 1ull << (cd->bit);
430 
431         /*
432          * Called under the desc lock, so these should never get out
433          * of sync.
434          */
435         if (cd->line == 0) {
436                 int index = octeon_coreid_for_cpu(cpu) * 2;
437                 set_bit(cd->bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu));
438                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
439         } else {
440                 int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
441                 set_bit(cd->bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
442                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
443         }
444 }
445 
446 /*
447  * Enable the irq in the sum2 registers.
448  */
449 static void octeon_irq_ciu_enable_sum2(struct irq_data *data)
450 {
451         u64 mask;
452         int cpu = next_cpu_for_irq(data);
453         int index = octeon_coreid_for_cpu(cpu);
454         struct octeon_ciu_chip_data *cd;
455 
456         cd = irq_data_get_irq_chip_data(data);
457         mask = 1ull << (cd->bit);
458 
459         cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1S(index), mask);
460 }
461 
462 /*
463  * Disable the irq in the sum2 registers.
464  */
465 static void octeon_irq_ciu_disable_local_sum2(struct irq_data *data)
466 {
467         u64 mask;
468         int cpu = next_cpu_for_irq(data);
469         int index = octeon_coreid_for_cpu(cpu);
470         struct octeon_ciu_chip_data *cd;
471 
472         cd = irq_data_get_irq_chip_data(data);
473         mask = 1ull << (cd->bit);
474 
475         cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(index), mask);
476 }
477 
478 static void octeon_irq_ciu_ack_sum2(struct irq_data *data)
479 {
480         u64 mask;
481         int cpu = next_cpu_for_irq(data);
482         int index = octeon_coreid_for_cpu(cpu);
483         struct octeon_ciu_chip_data *cd;
484 
485         cd = irq_data_get_irq_chip_data(data);
486         mask = 1ull << (cd->bit);
487 
488         cvmx_write_csr(CVMX_CIU_SUM2_PPX_IP4(index), mask);
489 }
490 
491 static void octeon_irq_ciu_disable_all_sum2(struct irq_data *data)
492 {
493         int cpu;
494         struct octeon_ciu_chip_data *cd;
495         u64 mask;
496 
497         cd = irq_data_get_irq_chip_data(data);
498         mask = 1ull << (cd->bit);
499 
500         for_each_online_cpu(cpu) {
501                 int coreid = octeon_coreid_for_cpu(cpu);
502 
503                 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(coreid), mask);
504         }
505 }
506 
507 /*
508  * Enable the irq on the current CPU for chips that
509  * have the EN*_W1{S,C} registers.
510  */
511 static void octeon_irq_ciu_enable_local_v2(struct irq_data *data)
512 {
513         u64 mask;
514         struct octeon_ciu_chip_data *cd;
515 
516         cd = irq_data_get_irq_chip_data(data);
517         mask = 1ull << (cd->bit);
518 
519         if (cd->line == 0) {
520                 int index = cvmx_get_core_num() * 2;
521                 set_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu0_en_mirror));
522                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
523         } else {
524                 int index = cvmx_get_core_num() * 2 + 1;
525                 set_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu1_en_mirror));
526                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
527         }
528 }
529 
530 static void octeon_irq_ciu_disable_local_v2(struct irq_data *data)
531 {
532         u64 mask;
533         struct octeon_ciu_chip_data *cd;
534 
535         cd = irq_data_get_irq_chip_data(data);
536         mask = 1ull << (cd->bit);
537 
538         if (cd->line == 0) {
539                 int index = cvmx_get_core_num() * 2;
540                 clear_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu0_en_mirror));
541                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
542         } else {
543                 int index = cvmx_get_core_num() * 2 + 1;
544                 clear_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu1_en_mirror));
545                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
546         }
547 }
548 
549 /*
550  * Write to the W1C bit in CVMX_CIU_INTX_SUM0 to clear the irq.
551  */
552 static void octeon_irq_ciu_ack(struct irq_data *data)
553 {
554         u64 mask;
555         struct octeon_ciu_chip_data *cd;
556 
557         cd = irq_data_get_irq_chip_data(data);
558         mask = 1ull << (cd->bit);
559 
560         if (cd->line == 0) {
561                 int index = cvmx_get_core_num() * 2;
562                 cvmx_write_csr(CVMX_CIU_INTX_SUM0(index), mask);
563         } else {
564                 cvmx_write_csr(CVMX_CIU_INT_SUM1, mask);
565         }
566 }
567 
568 /*
569  * Disable the irq on the all cores for chips that have the EN*_W1{S,C}
570  * registers.
571  */
572 static void octeon_irq_ciu_disable_all_v2(struct irq_data *data)
573 {
574         int cpu;
575         u64 mask;
576         struct octeon_ciu_chip_data *cd;
577 
578         cd = irq_data_get_irq_chip_data(data);
579         mask = 1ull << (cd->bit);
580 
581         if (cd->line == 0) {
582                 for_each_online_cpu(cpu) {
583                         int index = octeon_coreid_for_cpu(cpu) * 2;
584                         clear_bit(cd->bit,
585                                 &per_cpu(octeon_irq_ciu0_en_mirror, cpu));
586                         cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
587                 }
588         } else {
589                 for_each_online_cpu(cpu) {
590                         int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
591                         clear_bit(cd->bit,
592                                 &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
593                         cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
594                 }
595         }
596 }
597 
598 /*
599  * Enable the irq on the all cores for chips that have the EN*_W1{S,C}
600  * registers.
601  */
602 static void octeon_irq_ciu_enable_all_v2(struct irq_data *data)
603 {
604         int cpu;
605         u64 mask;
606         struct octeon_ciu_chip_data *cd;
607 
608         cd = irq_data_get_irq_chip_data(data);
609         mask = 1ull << (cd->bit);
610 
611         if (cd->line == 0) {
612                 for_each_online_cpu(cpu) {
613                         int index = octeon_coreid_for_cpu(cpu) * 2;
614                         set_bit(cd->bit,
615                                 &per_cpu(octeon_irq_ciu0_en_mirror, cpu));
616                         cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
617                 }
618         } else {
619                 for_each_online_cpu(cpu) {
620                         int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
621                         set_bit(cd->bit,
622                                 &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
623                         cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
624                 }
625         }
626 }
627 
628 static void octeon_irq_gpio_setup(struct irq_data *data)
629 {
630         union cvmx_gpio_bit_cfgx cfg;
631         struct octeon_ciu_chip_data *cd;
632         u32 t = irqd_get_trigger_type(data);
633 
634         cd = irq_data_get_irq_chip_data(data);
635 
636         cfg.u64 = 0;
637         cfg.s.int_en = 1;
638         cfg.s.int_type = (t & IRQ_TYPE_EDGE_BOTH) != 0;
639         cfg.s.rx_xor = (t & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_EDGE_FALLING)) != 0;
640 
641         /* 140 nS glitch filter*/
642         cfg.s.fil_cnt = 7;
643         cfg.s.fil_sel = 3;
644 
645         cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), cfg.u64);
646 }
647 
648 static void octeon_irq_ciu_enable_gpio_v2(struct irq_data *data)
649 {
650         octeon_irq_gpio_setup(data);
651         octeon_irq_ciu_enable_v2(data);
652 }
653 
654 static void octeon_irq_ciu_enable_gpio(struct irq_data *data)
655 {
656         octeon_irq_gpio_setup(data);
657         octeon_irq_ciu_enable(data);
658 }
659 
660 static int octeon_irq_ciu_gpio_set_type(struct irq_data *data, unsigned int t)
661 {
662         irqd_set_trigger_type(data, t);
663         octeon_irq_gpio_setup(data);
664 
665         return IRQ_SET_MASK_OK;
666 }
667 
668 static void octeon_irq_ciu_disable_gpio_v2(struct irq_data *data)
669 {
670         struct octeon_ciu_chip_data *cd;
671 
672         cd = irq_data_get_irq_chip_data(data);
673         cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0);
674 
675         octeon_irq_ciu_disable_all_v2(data);
676 }
677 
678 static void octeon_irq_ciu_disable_gpio(struct irq_data *data)
679 {
680         struct octeon_ciu_chip_data *cd;
681 
682         cd = irq_data_get_irq_chip_data(data);
683         cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0);
684 
685         octeon_irq_ciu_disable_all(data);
686 }
687 
688 static void octeon_irq_ciu_gpio_ack(struct irq_data *data)
689 {
690         struct octeon_ciu_chip_data *cd;
691         u64 mask;
692 
693         cd = irq_data_get_irq_chip_data(data);
694         mask = 1ull << (cd->gpio_line);
695 
696         cvmx_write_csr(CVMX_GPIO_INT_CLR, mask);
697 }
698 
699 static void octeon_irq_handle_trigger(unsigned int irq, struct irq_desc *desc)
700 {
701         struct irq_data *data = irq_desc_get_irq_data(desc);
702 
703         if (irqd_get_trigger_type(data) & IRQ_TYPE_EDGE_BOTH)
704                 handle_edge_irq(irq, desc);
705         else
706                 handle_level_irq(irq, desc);
707 }
708 
709 #ifdef CONFIG_SMP
710 
711 static void octeon_irq_cpu_offline_ciu(struct irq_data *data)
712 {
713         int cpu = smp_processor_id();
714         cpumask_t new_affinity;
715 
716         if (!cpumask_test_cpu(cpu, data->affinity))
717                 return;
718 
719         if (cpumask_weight(data->affinity) > 1) {
720                 /*
721                  * It has multi CPU affinity, just remove this CPU
722                  * from the affinity set.
723                  */
724                 cpumask_copy(&new_affinity, data->affinity);
725                 cpumask_clear_cpu(cpu, &new_affinity);
726         } else {
727                 /* Otherwise, put it on lowest numbered online CPU. */
728                 cpumask_clear(&new_affinity);
729                 cpumask_set_cpu(cpumask_first(cpu_online_mask), &new_affinity);
730         }
731         irq_set_affinity_locked(data, &new_affinity, false);
732 }
733 
734 static int octeon_irq_ciu_set_affinity(struct irq_data *data,
735                                        const struct cpumask *dest, bool force)
736 {
737         int cpu;
738         bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data);
739         unsigned long flags;
740         struct octeon_ciu_chip_data *cd;
741         unsigned long *pen;
742         raw_spinlock_t *lock;
743 
744         cd = irq_data_get_irq_chip_data(data);
745 
746         /*
747          * For non-v2 CIU, we will allow only single CPU affinity.
748          * This removes the need to do locking in the .ack/.eoi
749          * functions.
750          */
751         if (cpumask_weight(dest) != 1)
752                 return -EINVAL;
753 
754         if (!enable_one)
755                 return 0;
756 
757 
758         for_each_online_cpu(cpu) {
759                 int coreid = octeon_coreid_for_cpu(cpu);
760 
761                 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
762                 raw_spin_lock_irqsave(lock, flags);
763 
764                 if (cd->line == 0)
765                         pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
766                 else
767                         pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
768 
769                 if (cpumask_test_cpu(cpu, dest) && enable_one) {
770                         enable_one = 0;
771                         __set_bit(cd->bit, pen);
772                 } else {
773                         __clear_bit(cd->bit, pen);
774                 }
775                 /*
776                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
777                  * enabling the irq.
778                  */
779                 wmb();
780 
781                 if (cd->line == 0)
782                         cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
783                 else
784                         cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
785 
786                 raw_spin_unlock_irqrestore(lock, flags);
787         }
788         return 0;
789 }
790 
791 /*
792  * Set affinity for the irq for chips that have the EN*_W1{S,C}
793  * registers.
794  */
795 static int octeon_irq_ciu_set_affinity_v2(struct irq_data *data,
796                                           const struct cpumask *dest,
797                                           bool force)
798 {
799         int cpu;
800         bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data);
801         u64 mask;
802         struct octeon_ciu_chip_data *cd;
803 
804         if (!enable_one)
805                 return 0;
806 
807         cd = irq_data_get_irq_chip_data(data);
808         mask = 1ull << cd->bit;
809 
810         if (cd->line == 0) {
811                 for_each_online_cpu(cpu) {
812                         unsigned long *pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
813                         int index = octeon_coreid_for_cpu(cpu) * 2;
814                         if (cpumask_test_cpu(cpu, dest) && enable_one) {
815                                 enable_one = false;
816                                 set_bit(cd->bit, pen);
817                                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
818                         } else {
819                                 clear_bit(cd->bit, pen);
820                                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
821                         }
822                 }
823         } else {
824                 for_each_online_cpu(cpu) {
825                         unsigned long *pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
826                         int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
827                         if (cpumask_test_cpu(cpu, dest) && enable_one) {
828                                 enable_one = false;
829                                 set_bit(cd->bit, pen);
830                                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
831                         } else {
832                                 clear_bit(cd->bit, pen);
833                                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
834                         }
835                 }
836         }
837         return 0;
838 }
839 
840 static int octeon_irq_ciu_set_affinity_sum2(struct irq_data *data,
841                                             const struct cpumask *dest,
842                                             bool force)
843 {
844         int cpu;
845         bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data);
846         u64 mask;
847         struct octeon_ciu_chip_data *cd;
848 
849         if (!enable_one)
850                 return 0;
851 
852         cd = irq_data_get_irq_chip_data(data);
853         mask = 1ull << cd->bit;
854 
855         for_each_online_cpu(cpu) {
856                 int index = octeon_coreid_for_cpu(cpu);
857 
858                 if (cpumask_test_cpu(cpu, dest) && enable_one) {
859                         enable_one = false;
860                         cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1S(index), mask);
861                 } else {
862                         cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(index), mask);
863                 }
864         }
865         return 0;
866 }
867 #endif
868 
869 /*
870  * Newer octeon chips have support for lockless CIU operation.
871  */
872 static struct irq_chip octeon_irq_chip_ciu_v2 = {
873         .name = "CIU",
874         .irq_enable = octeon_irq_ciu_enable_v2,
875         .irq_disable = octeon_irq_ciu_disable_all_v2,
876         .irq_mask = octeon_irq_ciu_disable_local_v2,
877         .irq_unmask = octeon_irq_ciu_enable_v2,
878 #ifdef CONFIG_SMP
879         .irq_set_affinity = octeon_irq_ciu_set_affinity_v2,
880         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
881 #endif
882 };
883 
884 static struct irq_chip octeon_irq_chip_ciu_v2_edge = {
885         .name = "CIU",
886         .irq_enable = octeon_irq_ciu_enable_v2,
887         .irq_disable = octeon_irq_ciu_disable_all_v2,
888         .irq_ack = octeon_irq_ciu_ack,
889         .irq_mask = octeon_irq_ciu_disable_local_v2,
890         .irq_unmask = octeon_irq_ciu_enable_v2,
891 #ifdef CONFIG_SMP
892         .irq_set_affinity = octeon_irq_ciu_set_affinity_v2,
893         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
894 #endif
895 };
896 
897 /*
898  * Newer octeon chips have support for lockless CIU operation.
899  */
900 static struct irq_chip octeon_irq_chip_ciu_sum2 = {
901         .name = "CIU",
902         .irq_enable = octeon_irq_ciu_enable_sum2,
903         .irq_disable = octeon_irq_ciu_disable_all_sum2,
904         .irq_mask = octeon_irq_ciu_disable_local_sum2,
905         .irq_unmask = octeon_irq_ciu_enable_sum2,
906 #ifdef CONFIG_SMP
907         .irq_set_affinity = octeon_irq_ciu_set_affinity_sum2,
908         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
909 #endif
910 };
911 
912 static struct irq_chip octeon_irq_chip_ciu_sum2_edge = {
913         .name = "CIU",
914         .irq_enable = octeon_irq_ciu_enable_sum2,
915         .irq_disable = octeon_irq_ciu_disable_all_sum2,
916         .irq_ack = octeon_irq_ciu_ack_sum2,
917         .irq_mask = octeon_irq_ciu_disable_local_sum2,
918         .irq_unmask = octeon_irq_ciu_enable_sum2,
919 #ifdef CONFIG_SMP
920         .irq_set_affinity = octeon_irq_ciu_set_affinity_sum2,
921         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
922 #endif
923 };
924 
925 static struct irq_chip octeon_irq_chip_ciu = {
926         .name = "CIU",
927         .irq_enable = octeon_irq_ciu_enable,
928         .irq_disable = octeon_irq_ciu_disable_all,
929         .irq_mask = octeon_irq_ciu_disable_local,
930         .irq_unmask = octeon_irq_ciu_enable,
931 #ifdef CONFIG_SMP
932         .irq_set_affinity = octeon_irq_ciu_set_affinity,
933         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
934 #endif
935 };
936 
937 static struct irq_chip octeon_irq_chip_ciu_edge = {
938         .name = "CIU",
939         .irq_enable = octeon_irq_ciu_enable,
940         .irq_disable = octeon_irq_ciu_disable_all,
941         .irq_ack = octeon_irq_ciu_ack,
942         .irq_mask = octeon_irq_ciu_disable_local,
943         .irq_unmask = octeon_irq_ciu_enable,
944 #ifdef CONFIG_SMP
945         .irq_set_affinity = octeon_irq_ciu_set_affinity,
946         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
947 #endif
948 };
949 
950 /* The mbox versions don't do any affinity or round-robin. */
951 static struct irq_chip octeon_irq_chip_ciu_mbox_v2 = {
952         .name = "CIU-M",
953         .irq_enable = octeon_irq_ciu_enable_all_v2,
954         .irq_disable = octeon_irq_ciu_disable_all_v2,
955         .irq_ack = octeon_irq_ciu_disable_local_v2,
956         .irq_eoi = octeon_irq_ciu_enable_local_v2,
957 
958         .irq_cpu_online = octeon_irq_ciu_enable_local_v2,
959         .irq_cpu_offline = octeon_irq_ciu_disable_local_v2,
960         .flags = IRQCHIP_ONOFFLINE_ENABLED,
961 };
962 
963 static struct irq_chip octeon_irq_chip_ciu_mbox = {
964         .name = "CIU-M",
965         .irq_enable = octeon_irq_ciu_enable_all,
966         .irq_disable = octeon_irq_ciu_disable_all,
967         .irq_ack = octeon_irq_ciu_disable_local,
968         .irq_eoi = octeon_irq_ciu_enable_local,
969 
970         .irq_cpu_online = octeon_irq_ciu_enable_local,
971         .irq_cpu_offline = octeon_irq_ciu_disable_local,
972         .flags = IRQCHIP_ONOFFLINE_ENABLED,
973 };
974 
975 static struct irq_chip octeon_irq_chip_ciu_gpio_v2 = {
976         .name = "CIU-GPIO",
977         .irq_enable = octeon_irq_ciu_enable_gpio_v2,
978         .irq_disable = octeon_irq_ciu_disable_gpio_v2,
979         .irq_ack = octeon_irq_ciu_gpio_ack,
980         .irq_mask = octeon_irq_ciu_disable_local_v2,
981         .irq_unmask = octeon_irq_ciu_enable_v2,
982         .irq_set_type = octeon_irq_ciu_gpio_set_type,
983 #ifdef CONFIG_SMP
984         .irq_set_affinity = octeon_irq_ciu_set_affinity_v2,
985         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
986 #endif
987         .flags = IRQCHIP_SET_TYPE_MASKED,
988 };
989 
990 static struct irq_chip octeon_irq_chip_ciu_gpio = {
991         .name = "CIU-GPIO",
992         .irq_enable = octeon_irq_ciu_enable_gpio,
993         .irq_disable = octeon_irq_ciu_disable_gpio,
994         .irq_mask = octeon_irq_ciu_disable_local,
995         .irq_unmask = octeon_irq_ciu_enable,
996         .irq_ack = octeon_irq_ciu_gpio_ack,
997         .irq_set_type = octeon_irq_ciu_gpio_set_type,
998 #ifdef CONFIG_SMP
999         .irq_set_affinity = octeon_irq_ciu_set_affinity,
1000         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
1001 #endif
1002         .flags = IRQCHIP_SET_TYPE_MASKED,
1003 };
1004 
1005 /*
1006  * Watchdog interrupts are special.  They are associated with a single
1007  * core, so we hardwire the affinity to that core.
1008  */
1009 static void octeon_irq_ciu_wd_enable(struct irq_data *data)
1010 {
1011         unsigned long flags;
1012         unsigned long *pen;
1013         int coreid = data->irq - OCTEON_IRQ_WDOG0;      /* Bit 0-63 of EN1 */
1014         int cpu = octeon_cpu_for_coreid(coreid);
1015         raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
1016 
1017         raw_spin_lock_irqsave(lock, flags);
1018         pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
1019         __set_bit(coreid, pen);
1020         /*
1021          * Must be visible to octeon_irq_ip{2,3}_ciu() before enabling
1022          * the irq.
1023          */
1024         wmb();
1025         cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
1026         raw_spin_unlock_irqrestore(lock, flags);
1027 }
1028 
1029 /*
1030  * Watchdog interrupts are special.  They are associated with a single
1031  * core, so we hardwire the affinity to that core.
1032  */
1033 static void octeon_irq_ciu1_wd_enable_v2(struct irq_data *data)
1034 {
1035         int coreid = data->irq - OCTEON_IRQ_WDOG0;
1036         int cpu = octeon_cpu_for_coreid(coreid);
1037 
1038         set_bit(coreid, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
1039         cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(coreid * 2 + 1), 1ull << coreid);
1040 }
1041 
1042 
1043 static struct irq_chip octeon_irq_chip_ciu_wd_v2 = {
1044         .name = "CIU-W",
1045         .irq_enable = octeon_irq_ciu1_wd_enable_v2,
1046         .irq_disable = octeon_irq_ciu_disable_all_v2,
1047         .irq_mask = octeon_irq_ciu_disable_local_v2,
1048         .irq_unmask = octeon_irq_ciu_enable_local_v2,
1049 };
1050 
1051 static struct irq_chip octeon_irq_chip_ciu_wd = {
1052         .name = "CIU-W",
1053         .irq_enable = octeon_irq_ciu_wd_enable,
1054         .irq_disable = octeon_irq_ciu_disable_all,
1055         .irq_mask = octeon_irq_ciu_disable_local,
1056         .irq_unmask = octeon_irq_ciu_enable_local,
1057 };
1058 
1059 static bool octeon_irq_ciu_is_edge(unsigned int line, unsigned int bit)
1060 {
1061         bool edge = false;
1062 
1063         if (line == 0)
1064                 switch (bit) {
1065                 case 48 ... 49: /* GMX DRP */
1066                 case 50: /* IPD_DRP */
1067                 case 52 ... 55: /* Timers */
1068                 case 58: /* MPI */
1069                         edge = true;
1070                         break;
1071                 default:
1072                         break;
1073                 }
1074         else /* line == 1 */
1075                 switch (bit) {
1076                 case 47: /* PTP */
1077                         edge = true;
1078                         break;
1079                 default:
1080                         break;
1081                 }
1082         return edge;
1083 }
1084 
1085 struct octeon_irq_gpio_domain_data {
1086         unsigned int base_hwirq;
1087 };
1088 
1089 static int octeon_irq_gpio_xlat(struct irq_domain *d,
1090                                 struct device_node *node,
1091                                 const u32 *intspec,
1092                                 unsigned int intsize,
1093                                 unsigned long *out_hwirq,
1094                                 unsigned int *out_type)
1095 {
1096         unsigned int type;
1097         unsigned int pin;
1098         unsigned int trigger;
1099 
1100         if (d->of_node != node)
1101                 return -EINVAL;
1102 
1103         if (intsize < 2)
1104                 return -EINVAL;
1105 
1106         pin = intspec[0];
1107         if (pin >= 16)
1108                 return -EINVAL;
1109 
1110         trigger = intspec[1];
1111 
1112         switch (trigger) {
1113         case 1:
1114                 type = IRQ_TYPE_EDGE_RISING;
1115                 break;
1116         case 2:
1117                 type = IRQ_TYPE_EDGE_FALLING;
1118                 break;
1119         case 4:
1120                 type = IRQ_TYPE_LEVEL_HIGH;
1121                 break;
1122         case 8:
1123                 type = IRQ_TYPE_LEVEL_LOW;
1124                 break;
1125         default:
1126                 pr_err("Error: (%s) Invalid irq trigger specification: %x\n",
1127                        node->name,
1128                        trigger);
1129                 type = IRQ_TYPE_LEVEL_LOW;
1130                 break;
1131         }
1132         *out_type = type;
1133         *out_hwirq = pin;
1134 
1135         return 0;
1136 }
1137 
1138 static int octeon_irq_ciu_xlat(struct irq_domain *d,
1139                                struct device_node *node,
1140                                const u32 *intspec,
1141                                unsigned int intsize,
1142                                unsigned long *out_hwirq,
1143                                unsigned int *out_type)
1144 {
1145         unsigned int ciu, bit;
1146         struct octeon_irq_ciu_domain_data *dd = d->host_data;
1147 
1148         ciu = intspec[0];
1149         bit = intspec[1];
1150 
1151         if (ciu >= dd->num_sum || bit > 63)
1152                 return -EINVAL;
1153 
1154         *out_hwirq = (ciu << 6) | bit;
1155         *out_type = 0;
1156 
1157         return 0;
1158 }
1159 
1160 static struct irq_chip *octeon_irq_ciu_chip;
1161 static struct irq_chip *octeon_irq_ciu_chip_edge;
1162 static struct irq_chip *octeon_irq_gpio_chip;
1163 
1164 static bool octeon_irq_virq_in_range(unsigned int virq)
1165 {
1166         /* We cannot let it overflow the mapping array. */
1167         if (virq < (1ul << 8 * sizeof(octeon_irq_ciu_to_irq[0][0])))
1168                 return true;
1169 
1170         WARN_ONCE(true, "virq out of range %u.\n", virq);
1171         return false;
1172 }
1173 
1174 static int octeon_irq_ciu_map(struct irq_domain *d,
1175                               unsigned int virq, irq_hw_number_t hw)
1176 {
1177         int rv;
1178         unsigned int line = hw >> 6;
1179         unsigned int bit = hw & 63;
1180         struct octeon_irq_ciu_domain_data *dd = d->host_data;
1181 
1182         if (!octeon_irq_virq_in_range(virq))
1183                 return -EINVAL;
1184 
1185         /* Don't map irq if it is reserved for GPIO. */
1186         if (line == 0 && bit >= 16 && bit <32)
1187                 return 0;
1188 
1189         if (line >= dd->num_sum || octeon_irq_ciu_to_irq[line][bit] != 0)
1190                 return -EINVAL;
1191 
1192         if (line == 2) {
1193                 if (octeon_irq_ciu_is_edge(line, bit))
1194                         rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1195                                 &octeon_irq_chip_ciu_sum2_edge,
1196                                 handle_edge_irq);
1197                 else
1198                         rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1199                                 &octeon_irq_chip_ciu_sum2,
1200                                 handle_level_irq);
1201         } else {
1202                 if (octeon_irq_ciu_is_edge(line, bit))
1203                         rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1204                                 octeon_irq_ciu_chip_edge,
1205                                 handle_edge_irq);
1206                 else
1207                         rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1208                                 octeon_irq_ciu_chip,
1209                                 handle_level_irq);
1210         }
1211         return rv;
1212 }
1213 
1214 static int octeon_irq_gpio_map(struct irq_domain *d,
1215                                unsigned int virq, irq_hw_number_t hw)
1216 {
1217         struct octeon_irq_gpio_domain_data *gpiod = d->host_data;
1218         unsigned int line, bit;
1219         int r;
1220 
1221         if (!octeon_irq_virq_in_range(virq))
1222                 return -EINVAL;
1223 
1224         line = (hw + gpiod->base_hwirq) >> 6;
1225         bit = (hw + gpiod->base_hwirq) & 63;
1226         if (line > ARRAY_SIZE(octeon_irq_ciu_to_irq) ||
1227                 octeon_irq_ciu_to_irq[line][bit] != 0)
1228                 return -EINVAL;
1229 
1230         r = octeon_irq_set_ciu_mapping(virq, line, bit, hw,
1231                 octeon_irq_gpio_chip, octeon_irq_handle_trigger);
1232         return r;
1233 }
1234 
1235 static struct irq_domain_ops octeon_irq_domain_ciu_ops = {
1236         .map = octeon_irq_ciu_map,
1237         .unmap = octeon_irq_free_cd,
1238         .xlate = octeon_irq_ciu_xlat,
1239 };
1240 
1241 static struct irq_domain_ops octeon_irq_domain_gpio_ops = {
1242         .map = octeon_irq_gpio_map,
1243         .unmap = octeon_irq_free_cd,
1244         .xlate = octeon_irq_gpio_xlat,
1245 };
1246 
1247 static void octeon_irq_ip2_ciu(void)
1248 {
1249         const unsigned long core_id = cvmx_get_core_num();
1250         u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INTX_SUM0(core_id * 2));
1251 
1252         ciu_sum &= __this_cpu_read(octeon_irq_ciu0_en_mirror);
1253         if (likely(ciu_sum)) {
1254                 int bit = fls64(ciu_sum) - 1;
1255                 int irq = octeon_irq_ciu_to_irq[0][bit];
1256                 if (likely(irq))
1257                         do_IRQ(irq);
1258                 else
1259                         spurious_interrupt();
1260         } else {
1261                 spurious_interrupt();
1262         }
1263 }
1264 
1265 static void octeon_irq_ip3_ciu(void)
1266 {
1267         u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INT_SUM1);
1268 
1269         ciu_sum &= __this_cpu_read(octeon_irq_ciu1_en_mirror);
1270         if (likely(ciu_sum)) {
1271                 int bit = fls64(ciu_sum) - 1;
1272                 int irq = octeon_irq_ciu_to_irq[1][bit];
1273                 if (likely(irq))
1274                         do_IRQ(irq);
1275                 else
1276                         spurious_interrupt();
1277         } else {
1278                 spurious_interrupt();
1279         }
1280 }
1281 
1282 static void octeon_irq_ip4_ciu(void)
1283 {
1284         int coreid = cvmx_get_core_num();
1285         u64 ciu_sum = cvmx_read_csr(CVMX_CIU_SUM2_PPX_IP4(coreid));
1286         u64 ciu_en = cvmx_read_csr(CVMX_CIU_EN2_PPX_IP4(coreid));
1287 
1288         ciu_sum &= ciu_en;
1289         if (likely(ciu_sum)) {
1290                 int bit = fls64(ciu_sum) - 1;
1291                 int irq = octeon_irq_ciu_to_irq[2][bit];
1292 
1293                 if (likely(irq))
1294                         do_IRQ(irq);
1295                 else
1296                         spurious_interrupt();
1297         } else {
1298                 spurious_interrupt();
1299         }
1300 }
1301 
1302 static bool octeon_irq_use_ip4;
1303 
1304 static void octeon_irq_local_enable_ip4(void *arg)
1305 {
1306         set_c0_status(STATUSF_IP4);
1307 }
1308 
1309 static void octeon_irq_ip4_mask(void)
1310 {
1311         clear_c0_status(STATUSF_IP4);
1312         spurious_interrupt();
1313 }
1314 
1315 static void (*octeon_irq_ip2)(void);
1316 static void (*octeon_irq_ip3)(void);
1317 static void (*octeon_irq_ip4)(void);
1318 
1319 void (*octeon_irq_setup_secondary)(void);
1320 
1321 void octeon_irq_set_ip4_handler(octeon_irq_ip4_handler_t h)
1322 {
1323         octeon_irq_ip4 = h;
1324         octeon_irq_use_ip4 = true;
1325         on_each_cpu(octeon_irq_local_enable_ip4, NULL, 1);
1326 }
1327 
1328 static void octeon_irq_percpu_enable(void)
1329 {
1330         irq_cpu_online();
1331 }
1332 
1333 static void octeon_irq_init_ciu_percpu(void)
1334 {
1335         int coreid = cvmx_get_core_num();
1336 
1337 
1338         __this_cpu_write(octeon_irq_ciu0_en_mirror, 0);
1339         __this_cpu_write(octeon_irq_ciu1_en_mirror, 0);
1340         wmb();
1341         raw_spin_lock_init(this_cpu_ptr(&octeon_irq_ciu_spinlock));
1342         /*
1343          * Disable All CIU Interrupts. The ones we need will be
1344          * enabled later.  Read the SUM register so we know the write
1345          * completed.
1346          */
1347         cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2)), 0);
1348         cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2 + 1)), 0);
1349         cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2)), 0);
1350         cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2 + 1)), 0);
1351         cvmx_read_csr(CVMX_CIU_INTX_SUM0((coreid * 2)));
1352 }
1353 
1354 static void octeon_irq_init_ciu2_percpu(void)
1355 {
1356         u64 regx, ipx;
1357         int coreid = cvmx_get_core_num();
1358         u64 base = CVMX_CIU2_EN_PPX_IP2_WRKQ(coreid);
1359 
1360         /*
1361          * Disable All CIU2 Interrupts. The ones we need will be
1362          * enabled later.  Read the SUM register so we know the write
1363          * completed.
1364          *
1365          * There are 9 registers and 3 IPX levels with strides 0x1000
1366          * and 0x200 respectivly.  Use loops to clear them.
1367          */
1368         for (regx = 0; regx <= 0x8000; regx += 0x1000) {
1369                 for (ipx = 0; ipx <= 0x400; ipx += 0x200)
1370                         cvmx_write_csr(base + regx + ipx, 0);
1371         }
1372 
1373         cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(coreid));
1374 }
1375 
1376 static void octeon_irq_setup_secondary_ciu(void)
1377 {
1378         octeon_irq_init_ciu_percpu();
1379         octeon_irq_percpu_enable();
1380 
1381         /* Enable the CIU lines */
1382         set_c0_status(STATUSF_IP3 | STATUSF_IP2);
1383         if (octeon_irq_use_ip4)
1384                 set_c0_status(STATUSF_IP4);
1385         else
1386                 clear_c0_status(STATUSF_IP4);
1387 }
1388 
1389 static void octeon_irq_setup_secondary_ciu2(void)
1390 {
1391         octeon_irq_init_ciu2_percpu();
1392         octeon_irq_percpu_enable();
1393 
1394         /* Enable the CIU lines */
1395         set_c0_status(STATUSF_IP3 | STATUSF_IP2);
1396         if (octeon_irq_use_ip4)
1397                 set_c0_status(STATUSF_IP4);
1398         else
1399                 clear_c0_status(STATUSF_IP4);
1400 }
1401 
1402 static int __init octeon_irq_init_ciu(
1403         struct device_node *ciu_node, struct device_node *parent)
1404 {
1405         unsigned int i, r;
1406         struct irq_chip *chip;
1407         struct irq_chip *chip_edge;
1408         struct irq_chip *chip_mbox;
1409         struct irq_chip *chip_wd;
1410         struct irq_domain *ciu_domain = NULL;
1411         struct octeon_irq_ciu_domain_data *dd;
1412 
1413         dd = kzalloc(sizeof(*dd), GFP_KERNEL);
1414         if (!dd)
1415                 return -ENOMEM;
1416 
1417         octeon_irq_init_ciu_percpu();
1418         octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu;
1419 
1420         octeon_irq_ip2 = octeon_irq_ip2_ciu;
1421         octeon_irq_ip3 = octeon_irq_ip3_ciu;
1422         if ((OCTEON_IS_OCTEON2() || OCTEON_IS_OCTEON3())
1423                 && !OCTEON_IS_MODEL(OCTEON_CN63XX)) {
1424                 octeon_irq_ip4 =  octeon_irq_ip4_ciu;
1425                 dd->num_sum = 3;
1426                 octeon_irq_use_ip4 = true;
1427         } else {
1428                 octeon_irq_ip4 = octeon_irq_ip4_mask;
1429                 dd->num_sum = 2;
1430                 octeon_irq_use_ip4 = false;
1431         }
1432         if (OCTEON_IS_MODEL(OCTEON_CN58XX_PASS2_X) ||
1433             OCTEON_IS_MODEL(OCTEON_CN56XX_PASS2_X) ||
1434             OCTEON_IS_MODEL(OCTEON_CN52XX_PASS2_X) ||
1435             OCTEON_IS_OCTEON2() || OCTEON_IS_OCTEON3()) {
1436                 chip = &octeon_irq_chip_ciu_v2;
1437                 chip_edge = &octeon_irq_chip_ciu_v2_edge;
1438                 chip_mbox = &octeon_irq_chip_ciu_mbox_v2;
1439                 chip_wd = &octeon_irq_chip_ciu_wd_v2;
1440                 octeon_irq_gpio_chip = &octeon_irq_chip_ciu_gpio_v2;
1441         } else {
1442                 chip = &octeon_irq_chip_ciu;
1443                 chip_edge = &octeon_irq_chip_ciu_edge;
1444                 chip_mbox = &octeon_irq_chip_ciu_mbox;
1445                 chip_wd = &octeon_irq_chip_ciu_wd;
1446                 octeon_irq_gpio_chip = &octeon_irq_chip_ciu_gpio;
1447         }
1448         octeon_irq_ciu_chip = chip;
1449         octeon_irq_ciu_chip_edge = chip_edge;
1450 
1451         /* Mips internal */
1452         octeon_irq_init_core();
1453 
1454         ciu_domain = irq_domain_add_tree(
1455                 ciu_node, &octeon_irq_domain_ciu_ops, dd);
1456         irq_set_default_host(ciu_domain);
1457 
1458         /* CIU_0 */
1459         for (i = 0; i < 16; i++) {
1460                 r = octeon_irq_force_ciu_mapping(
1461                         ciu_domain, i + OCTEON_IRQ_WORKQ0, 0, i + 0);
1462                 if (r)
1463                         goto err;
1464         }
1465 
1466         r = octeon_irq_set_ciu_mapping(
1467                 OCTEON_IRQ_MBOX0, 0, 32, 0, chip_mbox, handle_percpu_irq);
1468         if (r)
1469                 goto err;
1470         r = octeon_irq_set_ciu_mapping(
1471                 OCTEON_IRQ_MBOX1, 0, 33, 0, chip_mbox, handle_percpu_irq);
1472         if (r)
1473                 goto err;
1474 
1475         for (i = 0; i < 4; i++) {
1476                 r = octeon_irq_force_ciu_mapping(
1477                         ciu_domain, i + OCTEON_IRQ_PCI_INT0, 0, i + 36);
1478                 if (r)
1479                         goto err;
1480         }
1481         for (i = 0; i < 4; i++) {
1482                 r = octeon_irq_force_ciu_mapping(
1483                         ciu_domain, i + OCTEON_IRQ_PCI_MSI0, 0, i + 40);
1484                 if (r)
1485                         goto err;
1486         }
1487 
1488         r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_TWSI, 0, 45);
1489         if (r)
1490                 goto err;
1491 
1492         r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_RML, 0, 46);
1493         if (r)
1494                 goto err;
1495 
1496         for (i = 0; i < 4; i++) {
1497                 r = octeon_irq_force_ciu_mapping(
1498                         ciu_domain, i + OCTEON_IRQ_TIMER0, 0, i + 52);
1499                 if (r)
1500                         goto err;
1501         }
1502 
1503         r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_USB0, 0, 56);
1504         if (r)
1505                 goto err;
1506 
1507         r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_TWSI2, 0, 59);
1508         if (r)
1509                 goto err;
1510 
1511         /* CIU_1 */
1512         for (i = 0; i < 16; i++) {
1513                 r = octeon_irq_set_ciu_mapping(
1514                         i + OCTEON_IRQ_WDOG0, 1, i + 0, 0, chip_wd,
1515                         handle_level_irq);
1516                 if (r)
1517                         goto err;
1518         }
1519 
1520         r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_USB1, 1, 17);
1521         if (r)
1522                 goto err;
1523 
1524         /* Enable the CIU lines */
1525         set_c0_status(STATUSF_IP3 | STATUSF_IP2);
1526         if (octeon_irq_use_ip4)
1527                 set_c0_status(STATUSF_IP4);
1528         else
1529                 clear_c0_status(STATUSF_IP4);
1530 
1531         return 0;
1532 err:
1533         return r;
1534 }
1535 
1536 static int __init octeon_irq_init_gpio(
1537         struct device_node *gpio_node, struct device_node *parent)
1538 {
1539         struct octeon_irq_gpio_domain_data *gpiod;
1540         u32 interrupt_cells;
1541         unsigned int base_hwirq;
1542         int r;
1543 
1544         r = of_property_read_u32(parent, "#interrupt-cells", &interrupt_cells);
1545         if (r)
1546                 return r;
1547 
1548         if (interrupt_cells == 1) {
1549                 u32 v;
1550 
1551                 r = of_property_read_u32_index(gpio_node, "interrupts", 0, &v);
1552                 if (r) {
1553                         pr_warn("No \"interrupts\" property.\n");
1554                         return r;
1555                 }
1556                 base_hwirq = v;
1557         } else if (interrupt_cells == 2) {
1558                 u32 v0, v1;
1559 
1560                 r = of_property_read_u32_index(gpio_node, "interrupts", 0, &v0);
1561                 if (r) {
1562                         pr_warn("No \"interrupts\" property.\n");
1563                         return r;
1564                 }
1565                 r = of_property_read_u32_index(gpio_node, "interrupts", 1, &v1);
1566                 if (r) {
1567                         pr_warn("No \"interrupts\" property.\n");
1568                         return r;
1569                 }
1570                 base_hwirq = (v0 << 6) | v1;
1571         } else {
1572                 pr_warn("Bad \"#interrupt-cells\" property: %u\n",
1573                         interrupt_cells);
1574                 return -EINVAL;
1575         }
1576 
1577         gpiod = kzalloc(sizeof(*gpiod), GFP_KERNEL);
1578         if (gpiod) {
1579                 /* gpio domain host_data is the base hwirq number. */
1580                 gpiod->base_hwirq = base_hwirq;
1581                 irq_domain_add_linear(
1582                         gpio_node, 16, &octeon_irq_domain_gpio_ops, gpiod);
1583         } else {
1584                 pr_warn("Cannot allocate memory for GPIO irq_domain.\n");
1585                 return -ENOMEM;
1586         }
1587 
1588         return 0;
1589 }
1590 /*
1591  * Watchdog interrupts are special.  They are associated with a single
1592  * core, so we hardwire the affinity to that core.
1593  */
1594 static void octeon_irq_ciu2_wd_enable(struct irq_data *data)
1595 {
1596         u64 mask;
1597         u64 en_addr;
1598         int coreid = data->irq - OCTEON_IRQ_WDOG0;
1599         struct octeon_ciu_chip_data *cd;
1600 
1601         cd = irq_data_get_irq_chip_data(data);
1602         mask = 1ull << (cd->bit);
1603 
1604         en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) +
1605                 (0x1000ull * cd->line);
1606         cvmx_write_csr(en_addr, mask);
1607 
1608 }
1609 
1610 static void octeon_irq_ciu2_enable(struct irq_data *data)
1611 {
1612         u64 mask;
1613         u64 en_addr;
1614         int cpu = next_cpu_for_irq(data);
1615         int coreid = octeon_coreid_for_cpu(cpu);
1616         struct octeon_ciu_chip_data *cd;
1617 
1618         cd = irq_data_get_irq_chip_data(data);
1619         mask = 1ull << (cd->bit);
1620 
1621         en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) +
1622                 (0x1000ull * cd->line);
1623         cvmx_write_csr(en_addr, mask);
1624 }
1625 
1626 static void octeon_irq_ciu2_enable_local(struct irq_data *data)
1627 {
1628         u64 mask;
1629         u64 en_addr;
1630         int coreid = cvmx_get_core_num();
1631         struct octeon_ciu_chip_data *cd;
1632 
1633         cd = irq_data_get_irq_chip_data(data);
1634         mask = 1ull << (cd->bit);
1635 
1636         en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) +
1637                 (0x1000ull * cd->line);
1638         cvmx_write_csr(en_addr, mask);
1639 
1640 }
1641 
1642 static void octeon_irq_ciu2_disable_local(struct irq_data *data)
1643 {
1644         u64 mask;
1645         u64 en_addr;
1646         int coreid = cvmx_get_core_num();
1647         struct octeon_ciu_chip_data *cd;
1648 
1649         cd = irq_data_get_irq_chip_data(data);
1650         mask = 1ull << (cd->bit);
1651 
1652         en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C(coreid) +
1653                 (0x1000ull * cd->line);
1654         cvmx_write_csr(en_addr, mask);
1655 
1656 }
1657 
1658 static void octeon_irq_ciu2_ack(struct irq_data *data)
1659 {
1660         u64 mask;
1661         u64 en_addr;
1662         int coreid = cvmx_get_core_num();
1663         struct octeon_ciu_chip_data *cd;
1664 
1665         cd = irq_data_get_irq_chip_data(data);
1666         mask = 1ull << (cd->bit);
1667 
1668         en_addr = CVMX_CIU2_RAW_PPX_IP2_WRKQ(coreid) + (0x1000ull * cd->line);
1669         cvmx_write_csr(en_addr, mask);
1670 
1671 }
1672 
1673 static void octeon_irq_ciu2_disable_all(struct irq_data *data)
1674 {
1675         int cpu;
1676         u64 mask;
1677         struct octeon_ciu_chip_data *cd;
1678 
1679         cd = irq_data_get_irq_chip_data(data);
1680         mask = 1ull << (cd->bit);
1681 
1682         for_each_online_cpu(cpu) {
1683                 u64 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C(
1684                         octeon_coreid_for_cpu(cpu)) + (0x1000ull * cd->line);
1685                 cvmx_write_csr(en_addr, mask);
1686         }
1687 }
1688 
1689 static void octeon_irq_ciu2_mbox_enable_all(struct irq_data *data)
1690 {
1691         int cpu;
1692         u64 mask;
1693 
1694         mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0);
1695 
1696         for_each_online_cpu(cpu) {
1697                 u64 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1S(
1698                         octeon_coreid_for_cpu(cpu));
1699                 cvmx_write_csr(en_addr, mask);
1700         }
1701 }
1702 
1703 static void octeon_irq_ciu2_mbox_disable_all(struct irq_data *data)
1704 {
1705         int cpu;
1706         u64 mask;
1707 
1708         mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0);
1709 
1710         for_each_online_cpu(cpu) {
1711                 u64 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1C(
1712                         octeon_coreid_for_cpu(cpu));
1713                 cvmx_write_csr(en_addr, mask);
1714         }
1715 }
1716 
1717 static void octeon_irq_ciu2_mbox_enable_local(struct irq_data *data)
1718 {
1719         u64 mask;
1720         u64 en_addr;
1721         int coreid = cvmx_get_core_num();
1722 
1723         mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0);
1724         en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1S(coreid);
1725         cvmx_write_csr(en_addr, mask);
1726 }
1727 
1728 static void octeon_irq_ciu2_mbox_disable_local(struct irq_data *data)
1729 {
1730         u64 mask;
1731         u64 en_addr;
1732         int coreid = cvmx_get_core_num();
1733 
1734         mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0);
1735         en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1C(coreid);
1736         cvmx_write_csr(en_addr, mask);
1737 }
1738 
1739 #ifdef CONFIG_SMP
1740 static int octeon_irq_ciu2_set_affinity(struct irq_data *data,
1741                                         const struct cpumask *dest, bool force)
1742 {
1743         int cpu;
1744         bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data);
1745         u64 mask;
1746         struct octeon_ciu_chip_data *cd;
1747 
1748         if (!enable_one)
1749                 return 0;
1750 
1751         cd = irq_data_get_irq_chip_data(data);
1752         mask = 1ull << cd->bit;
1753 
1754         for_each_online_cpu(cpu) {
1755                 u64 en_addr;
1756                 if (cpumask_test_cpu(cpu, dest) && enable_one) {
1757                         enable_one = false;
1758                         en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(
1759                                 octeon_coreid_for_cpu(cpu)) +
1760                                 (0x1000ull * cd->line);
1761                 } else {
1762                         en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C(
1763                                 octeon_coreid_for_cpu(cpu)) +
1764                                 (0x1000ull * cd->line);
1765                 }
1766                 cvmx_write_csr(en_addr, mask);
1767         }
1768 
1769         return 0;
1770 }
1771 #endif
1772 
1773 static void octeon_irq_ciu2_enable_gpio(struct irq_data *data)
1774 {
1775         octeon_irq_gpio_setup(data);
1776         octeon_irq_ciu2_enable(data);
1777 }
1778 
1779 static void octeon_irq_ciu2_disable_gpio(struct irq_data *data)
1780 {
1781         struct octeon_ciu_chip_data *cd;
1782 
1783         cd = irq_data_get_irq_chip_data(data);
1784 
1785         cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0);
1786 
1787         octeon_irq_ciu2_disable_all(data);
1788 }
1789 
1790 static struct irq_chip octeon_irq_chip_ciu2 = {
1791         .name = "CIU2-E",
1792         .irq_enable = octeon_irq_ciu2_enable,
1793         .irq_disable = octeon_irq_ciu2_disable_all,
1794         .irq_mask = octeon_irq_ciu2_disable_local,
1795         .irq_unmask = octeon_irq_ciu2_enable,
1796 #ifdef CONFIG_SMP
1797         .irq_set_affinity = octeon_irq_ciu2_set_affinity,
1798         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
1799 #endif
1800 };
1801 
1802 static struct irq_chip octeon_irq_chip_ciu2_edge = {
1803         .name = "CIU2-E",
1804         .irq_enable = octeon_irq_ciu2_enable,
1805         .irq_disable = octeon_irq_ciu2_disable_all,
1806         .irq_ack = octeon_irq_ciu2_ack,
1807         .irq_mask = octeon_irq_ciu2_disable_local,
1808         .irq_unmask = octeon_irq_ciu2_enable,
1809 #ifdef CONFIG_SMP
1810         .irq_set_affinity = octeon_irq_ciu2_set_affinity,
1811         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
1812 #endif
1813 };
1814 
1815 static struct irq_chip octeon_irq_chip_ciu2_mbox = {
1816         .name = "CIU2-M",
1817         .irq_enable = octeon_irq_ciu2_mbox_enable_all,
1818         .irq_disable = octeon_irq_ciu2_mbox_disable_all,
1819         .irq_ack = octeon_irq_ciu2_mbox_disable_local,
1820         .irq_eoi = octeon_irq_ciu2_mbox_enable_local,
1821 
1822         .irq_cpu_online = octeon_irq_ciu2_mbox_enable_local,
1823         .irq_cpu_offline = octeon_irq_ciu2_mbox_disable_local,
1824         .flags = IRQCHIP_ONOFFLINE_ENABLED,
1825 };
1826 
1827 static struct irq_chip octeon_irq_chip_ciu2_wd = {
1828         .name = "CIU2-W",
1829         .irq_enable = octeon_irq_ciu2_wd_enable,
1830         .irq_disable = octeon_irq_ciu2_disable_all,
1831         .irq_mask = octeon_irq_ciu2_disable_local,
1832         .irq_unmask = octeon_irq_ciu2_enable_local,
1833 };
1834 
1835 static struct irq_chip octeon_irq_chip_ciu2_gpio = {
1836         .name = "CIU-GPIO",
1837         .irq_enable = octeon_irq_ciu2_enable_gpio,
1838         .irq_disable = octeon_irq_ciu2_disable_gpio,
1839         .irq_ack = octeon_irq_ciu_gpio_ack,
1840         .irq_mask = octeon_irq_ciu2_disable_local,
1841         .irq_unmask = octeon_irq_ciu2_enable,
1842         .irq_set_type = octeon_irq_ciu_gpio_set_type,
1843 #ifdef CONFIG_SMP
1844         .irq_set_affinity = octeon_irq_ciu2_set_affinity,
1845         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
1846 #endif
1847         .flags = IRQCHIP_SET_TYPE_MASKED,
1848 };
1849 
1850 static int octeon_irq_ciu2_xlat(struct irq_domain *d,
1851                                 struct device_node *node,
1852                                 const u32 *intspec,
1853                                 unsigned int intsize,
1854                                 unsigned long *out_hwirq,
1855                                 unsigned int *out_type)
1856 {
1857         unsigned int ciu, bit;
1858 
1859         ciu = intspec[0];
1860         bit = intspec[1];
1861 
1862         *out_hwirq = (ciu << 6) | bit;
1863         *out_type = 0;
1864 
1865         return 0;
1866 }
1867 
1868 static bool octeon_irq_ciu2_is_edge(unsigned int line, unsigned int bit)
1869 {
1870         bool edge = false;
1871 
1872         if (line == 3) /* MIO */
1873                 switch (bit) {
1874                 case 2:  /* IPD_DRP */
1875                 case 8 ... 11: /* Timers */
1876                 case 48: /* PTP */
1877                         edge = true;
1878                         break;
1879                 default:
1880                         break;
1881                 }
1882         else if (line == 6) /* PKT */
1883                 switch (bit) {
1884                 case 52 ... 53: /* ILK_DRP */
1885                 case 8 ... 12:  /* GMX_DRP */
1886                         edge = true;
1887                         break;
1888                 default:
1889                         break;
1890                 }
1891         return edge;
1892 }
1893 
1894 static int octeon_irq_ciu2_map(struct irq_domain *d,
1895                                unsigned int virq, irq_hw_number_t hw)
1896 {
1897         unsigned int line = hw >> 6;
1898         unsigned int bit = hw & 63;
1899 
1900         if (!octeon_irq_virq_in_range(virq))
1901                 return -EINVAL;
1902 
1903         /*
1904          * Don't map irq if it is reserved for GPIO.
1905          * (Line 7 are the GPIO lines.)
1906          */
1907         if (line == 7)
1908                 return 0;
1909 
1910         if (line > 7 || octeon_irq_ciu_to_irq[line][bit] != 0)
1911                 return -EINVAL;
1912 
1913         if (octeon_irq_ciu2_is_edge(line, bit))
1914                 octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1915                                            &octeon_irq_chip_ciu2_edge,
1916                                            handle_edge_irq);
1917         else
1918                 octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1919                                            &octeon_irq_chip_ciu2,
1920                                            handle_level_irq);
1921 
1922         return 0;
1923 }
1924 
1925 static struct irq_domain_ops octeon_irq_domain_ciu2_ops = {
1926         .map = octeon_irq_ciu2_map,
1927         .unmap = octeon_irq_free_cd,
1928         .xlate = octeon_irq_ciu2_xlat,
1929 };
1930 
1931 static void octeon_irq_ciu2(void)
1932 {
1933         int line;
1934         int bit;
1935         int irq;
1936         u64 src_reg, src, sum;
1937         const unsigned long core_id = cvmx_get_core_num();
1938 
1939         sum = cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(core_id)) & 0xfful;
1940 
1941         if (unlikely(!sum))
1942                 goto spurious;
1943 
1944         line = fls64(sum) - 1;
1945         src_reg = CVMX_CIU2_SRC_PPX_IP2_WRKQ(core_id) + (0x1000 * line);
1946         src = cvmx_read_csr(src_reg);
1947 
1948         if (unlikely(!src))
1949                 goto spurious;
1950 
1951         bit = fls64(src) - 1;
1952         irq = octeon_irq_ciu_to_irq[line][bit];
1953         if (unlikely(!irq))
1954                 goto spurious;
1955 
1956         do_IRQ(irq);
1957         goto out;
1958 
1959 spurious:
1960         spurious_interrupt();
1961 out:
1962         /* CN68XX pass 1.x has an errata that accessing the ACK registers
1963                 can stop interrupts from propagating */
1964         if (OCTEON_IS_MODEL(OCTEON_CN68XX))
1965                 cvmx_read_csr(CVMX_CIU2_INTR_CIU_READY);
1966         else
1967                 cvmx_read_csr(CVMX_CIU2_ACK_PPX_IP2(core_id));
1968         return;
1969 }
1970 
1971 static void octeon_irq_ciu2_mbox(void)
1972 {
1973         int line;
1974 
1975         const unsigned long core_id = cvmx_get_core_num();
1976         u64 sum = cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP3(core_id)) >> 60;
1977 
1978         if (unlikely(!sum))
1979                 goto spurious;
1980 
1981         line = fls64(sum) - 1;
1982 
1983         do_IRQ(OCTEON_IRQ_MBOX0 + line);
1984         goto out;
1985 
1986 spurious:
1987         spurious_interrupt();
1988 out:
1989         /* CN68XX pass 1.x has an errata that accessing the ACK registers
1990                 can stop interrupts from propagating */
1991         if (OCTEON_IS_MODEL(OCTEON_CN68XX))
1992                 cvmx_read_csr(CVMX_CIU2_INTR_CIU_READY);
1993         else
1994                 cvmx_read_csr(CVMX_CIU2_ACK_PPX_IP3(core_id));
1995         return;
1996 }
1997 
1998 static int __init octeon_irq_init_ciu2(
1999         struct device_node *ciu_node, struct device_node *parent)
2000 {
2001         unsigned int i, r;
2002         struct irq_domain *ciu_domain = NULL;
2003 
2004         octeon_irq_init_ciu2_percpu();
2005         octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu2;
2006 
2007         octeon_irq_gpio_chip = &octeon_irq_chip_ciu2_gpio;
2008         octeon_irq_ip2 = octeon_irq_ciu2;
2009         octeon_irq_ip3 = octeon_irq_ciu2_mbox;
2010         octeon_irq_ip4 = octeon_irq_ip4_mask;
2011 
2012         /* Mips internal */
2013         octeon_irq_init_core();
2014 
2015         ciu_domain = irq_domain_add_tree(
2016                 ciu_node, &octeon_irq_domain_ciu2_ops, NULL);
2017         irq_set_default_host(ciu_domain);
2018 
2019         /* CUI2 */
2020         for (i = 0; i < 64; i++) {
2021                 r = octeon_irq_force_ciu_mapping(
2022                         ciu_domain, i + OCTEON_IRQ_WORKQ0, 0, i);
2023                 if (r)
2024                         goto err;
2025         }
2026 
2027         for (i = 0; i < 32; i++) {
2028                 r = octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_WDOG0, 1, i, 0,
2029                         &octeon_irq_chip_ciu2_wd, handle_level_irq);
2030                 if (r)
2031                         goto err;
2032         }
2033 
2034         for (i = 0; i < 4; i++) {
2035                 r = octeon_irq_force_ciu_mapping(
2036                         ciu_domain, i + OCTEON_IRQ_TIMER0, 3, i + 8);
2037                 if (r)
2038                         goto err;
2039         }
2040 
2041         r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_USB0, 3, 44);
2042         if (r)
2043                 goto err;
2044 
2045         for (i = 0; i < 4; i++) {
2046                 r = octeon_irq_force_ciu_mapping(
2047                         ciu_domain, i + OCTEON_IRQ_PCI_INT0, 4, i);
2048                 if (r)
2049                         goto err;
2050         }
2051 
2052         for (i = 0; i < 4; i++) {
2053                 r = octeon_irq_force_ciu_mapping(
2054                         ciu_domain, i + OCTEON_IRQ_PCI_MSI0, 4, i + 8);
2055                 if (r)
2056                         goto err;
2057         }
2058 
2059         irq_set_chip_and_handler(OCTEON_IRQ_MBOX0, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq);
2060         irq_set_chip_and_handler(OCTEON_IRQ_MBOX1, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq);
2061         irq_set_chip_and_handler(OCTEON_IRQ_MBOX2, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq);
2062         irq_set_chip_and_handler(OCTEON_IRQ_MBOX3, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq);
2063 
2064         /* Enable the CIU lines */
2065         set_c0_status(STATUSF_IP3 | STATUSF_IP2);
2066         clear_c0_status(STATUSF_IP4);
2067         return 0;
2068 err:
2069         return r;
2070 }
2071 
2072 struct octeon_irq_cib_host_data {
2073         raw_spinlock_t lock;
2074         u64 raw_reg;
2075         u64 en_reg;
2076         int max_bits;
2077 };
2078 
2079 struct octeon_irq_cib_chip_data {
2080         struct octeon_irq_cib_host_data *host_data;
2081         int bit;
2082 };
2083 
2084 static void octeon_irq_cib_enable(struct irq_data *data)
2085 {
2086         unsigned long flags;
2087         u64 en;
2088         struct octeon_irq_cib_chip_data *cd = irq_data_get_irq_chip_data(data);
2089         struct octeon_irq_cib_host_data *host_data = cd->host_data;
2090 
2091         raw_spin_lock_irqsave(&host_data->lock, flags);
2092         en = cvmx_read_csr(host_data->en_reg);
2093         en |= 1ull << cd->bit;
2094         cvmx_write_csr(host_data->en_reg, en);
2095         raw_spin_unlock_irqrestore(&host_data->lock, flags);
2096 }
2097 
2098 static void octeon_irq_cib_disable(struct irq_data *data)
2099 {
2100         unsigned long flags;
2101         u64 en;
2102         struct octeon_irq_cib_chip_data *cd = irq_data_get_irq_chip_data(data);
2103         struct octeon_irq_cib_host_data *host_data = cd->host_data;
2104 
2105         raw_spin_lock_irqsave(&host_data->lock, flags);
2106         en = cvmx_read_csr(host_data->en_reg);
2107         en &= ~(1ull << cd->bit);
2108         cvmx_write_csr(host_data->en_reg, en);
2109         raw_spin_unlock_irqrestore(&host_data->lock, flags);
2110 }
2111 
2112 static int octeon_irq_cib_set_type(struct irq_data *data, unsigned int t)
2113 {
2114         irqd_set_trigger_type(data, t);
2115         return IRQ_SET_MASK_OK;
2116 }
2117 
2118 static struct irq_chip octeon_irq_chip_cib = {
2119         .name = "CIB",
2120         .irq_enable = octeon_irq_cib_enable,
2121         .irq_disable = octeon_irq_cib_disable,
2122         .irq_mask = octeon_irq_cib_disable,
2123         .irq_unmask = octeon_irq_cib_enable,
2124         .irq_set_type = octeon_irq_cib_set_type,
2125 };
2126 
2127 static int octeon_irq_cib_xlat(struct irq_domain *d,
2128                                    struct device_node *node,
2129                                    const u32 *intspec,
2130                                    unsigned int intsize,
2131                                    unsigned long *out_hwirq,
2132                                    unsigned int *out_type)
2133 {
2134         unsigned int type = 0;
2135 
2136         if (intsize == 2)
2137                 type = intspec[1];
2138 
2139         switch (type) {
2140         case 0: /* unofficial value, but we might as well let it work. */
2141         case 4: /* official value for level triggering. */
2142                 *out_type = IRQ_TYPE_LEVEL_HIGH;
2143                 break;
2144         case 1: /* official value for edge triggering. */
2145                 *out_type = IRQ_TYPE_EDGE_RISING;
2146                 break;
2147         default: /* Nothing else is acceptable. */
2148                 return -EINVAL;
2149         }
2150 
2151         *out_hwirq = intspec[0];
2152 
2153         return 0;
2154 }
2155 
2156 static int octeon_irq_cib_map(struct irq_domain *d,
2157                               unsigned int virq, irq_hw_number_t hw)
2158 {
2159         struct octeon_irq_cib_host_data *host_data = d->host_data;
2160         struct octeon_irq_cib_chip_data *cd;
2161 
2162         if (hw >= host_data->max_bits) {
2163                 pr_err("ERROR: %s mapping %u is to big!\n",
2164                        d->of_node->name, (unsigned)hw);
2165                 return -EINVAL;
2166         }
2167 
2168         cd = kzalloc(sizeof(*cd), GFP_KERNEL);
2169         cd->host_data = host_data;
2170         cd->bit = hw;
2171 
2172         irq_set_chip_and_handler(virq, &octeon_irq_chip_cib,
2173                                  handle_simple_irq);
2174         irq_set_chip_data(virq, cd);
2175         return 0;
2176 }
2177 
2178 static struct irq_domain_ops octeon_irq_domain_cib_ops = {
2179         .map = octeon_irq_cib_map,
2180         .unmap = octeon_irq_free_cd,
2181         .xlate = octeon_irq_cib_xlat,
2182 };
2183 
2184 /* Chain to real handler. */
2185 static irqreturn_t octeon_irq_cib_handler(int my_irq, void *data)
2186 {
2187         u64 en;
2188         u64 raw;
2189         u64 bits;
2190         int i;
2191         int irq;
2192         struct irq_domain *cib_domain = data;
2193         struct octeon_irq_cib_host_data *host_data = cib_domain->host_data;
2194 
2195         en = cvmx_read_csr(host_data->en_reg);
2196         raw = cvmx_read_csr(host_data->raw_reg);
2197 
2198         bits = en & raw;
2199 
2200         for (i = 0; i < host_data->max_bits; i++) {
2201                 if ((bits & 1ull << i) == 0)
2202                         continue;
2203                 irq = irq_find_mapping(cib_domain, i);
2204                 if (!irq) {
2205                         unsigned long flags;
2206 
2207                         pr_err("ERROR: CIB bit %d@%llx IRQ unhandled, disabling\n",
2208                                 i, host_data->raw_reg);
2209                         raw_spin_lock_irqsave(&host_data->lock, flags);
2210                         en = cvmx_read_csr(host_data->en_reg);
2211                         en &= ~(1ull << i);
2212                         cvmx_write_csr(host_data->en_reg, en);
2213                         cvmx_write_csr(host_data->raw_reg, 1ull << i);
2214                         raw_spin_unlock_irqrestore(&host_data->lock, flags);
2215                 } else {
2216                         struct irq_desc *desc = irq_to_desc(irq);
2217                         struct irq_data *irq_data = irq_desc_get_irq_data(desc);
2218                         /* If edge, acknowledge the bit we will be sending. */
2219                         if (irqd_get_trigger_type(irq_data) &
2220                                 IRQ_TYPE_EDGE_BOTH)
2221                                 cvmx_write_csr(host_data->raw_reg, 1ull << i);
2222                         generic_handle_irq_desc(irq, desc);
2223                 }
2224         }
2225 
2226         return IRQ_HANDLED;
2227 }
2228 
2229 static int __init octeon_irq_init_cib(struct device_node *ciu_node,
2230                                       struct device_node *parent)
2231 {
2232         const __be32 *addr;
2233         u32 val;
2234         struct octeon_irq_cib_host_data *host_data;
2235         int parent_irq;
2236         int r;
2237         struct irq_domain *cib_domain;
2238 
2239         parent_irq = irq_of_parse_and_map(ciu_node, 0);
2240         if (!parent_irq) {
2241                 pr_err("ERROR: Couldn't acquire parent_irq for %s\n.",
2242                         ciu_node->name);
2243                 return -EINVAL;
2244         }
2245 
2246         host_data = kzalloc(sizeof(*host_data), GFP_KERNEL);
2247         raw_spin_lock_init(&host_data->lock);
2248 
2249         addr = of_get_address(ciu_node, 0, NULL, NULL);
2250         if (!addr) {
2251                 pr_err("ERROR: Couldn't acquire reg(0) %s\n.", ciu_node->name);
2252                 return -EINVAL;
2253         }
2254         host_data->raw_reg = (u64)phys_to_virt(
2255                 of_translate_address(ciu_node, addr));
2256 
2257         addr = of_get_address(ciu_node, 1, NULL, NULL);
2258         if (!addr) {
2259                 pr_err("ERROR: Couldn't acquire reg(1) %s\n.", ciu_node->name);
2260                 return -EINVAL;
2261         }
2262         host_data->en_reg = (u64)phys_to_virt(
2263                 of_translate_address(ciu_node, addr));
2264 
2265         r = of_property_read_u32(ciu_node, "cavium,max-bits", &val);
2266         if (r) {
2267                 pr_err("ERROR: Couldn't read cavium,max-bits from %s\n.",
2268                         ciu_node->name);
2269                 return r;
2270         }
2271         host_data->max_bits = val;
2272 
2273         cib_domain = irq_domain_add_linear(ciu_node, host_data->max_bits,
2274                                            &octeon_irq_domain_cib_ops,
2275                                            host_data);
2276         if (!cib_domain) {
2277                 pr_err("ERROR: Couldn't irq_domain_add_linear()\n.");
2278                 return -ENOMEM;
2279         }
2280 
2281         cvmx_write_csr(host_data->en_reg, 0); /* disable all IRQs */
2282         cvmx_write_csr(host_data->raw_reg, ~0); /* ack any outstanding */
2283 
2284         r = request_irq(parent_irq, octeon_irq_cib_handler,
2285                         IRQF_NO_THREAD, "cib", cib_domain);
2286         if (r) {
2287                 pr_err("request_irq cib failed %d\n", r);
2288                 return r;
2289         }
2290         pr_info("CIB interrupt controller probed: %llx %d\n",
2291                 host_data->raw_reg, host_data->max_bits);
2292         return 0;
2293 }
2294 
2295 static struct of_device_id ciu_types[] __initdata = {
2296         {.compatible = "cavium,octeon-3860-ciu", .data = octeon_irq_init_ciu},
2297         {.compatible = "cavium,octeon-3860-gpio", .data = octeon_irq_init_gpio},
2298         {.compatible = "cavium,octeon-6880-ciu2", .data = octeon_irq_init_ciu2},
2299         {.compatible = "cavium,octeon-7130-cib", .data = octeon_irq_init_cib},
2300         {}
2301 };
2302 
2303 void __init arch_init_irq(void)
2304 {
2305 #ifdef CONFIG_SMP
2306         /* Set the default affinity to the boot cpu. */
2307         cpumask_clear(irq_default_affinity);
2308         cpumask_set_cpu(smp_processor_id(), irq_default_affinity);
2309 #endif
2310         of_irq_init(ciu_types);
2311 }
2312 
2313 asmlinkage void plat_irq_dispatch(void)
2314 {
2315         unsigned long cop0_cause;
2316         unsigned long cop0_status;
2317 
2318         while (1) {
2319                 cop0_cause = read_c0_cause();
2320                 cop0_status = read_c0_status();
2321                 cop0_cause &= cop0_status;
2322                 cop0_cause &= ST0_IM;
2323 
2324                 if (cop0_cause & STATUSF_IP2)
2325                         octeon_irq_ip2();
2326                 else if (cop0_cause & STATUSF_IP3)
2327                         octeon_irq_ip3();
2328                 else if (cop0_cause & STATUSF_IP4)
2329                         octeon_irq_ip4();
2330                 else if (cop0_cause)
2331                         do_IRQ(fls(cop0_cause) - 9 + MIPS_CPU_IRQ_BASE);
2332                 else
2333                         break;
2334         }
2335 }
2336 
2337 #ifdef CONFIG_HOTPLUG_CPU
2338 
2339 void octeon_fixup_irqs(void)
2340 {
2341         irq_cpu_offline();
2342 }
2343 
2344 #endif /* CONFIG_HOTPLUG_CPU */
2345 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp