1 /* arch/sparc64/kernel/traps.c
2 *
3 * Copyright (C) 1995,1997,2008,2009,2012 David S. Miller (davem@davemloft.net)
4 * Copyright (C) 1997,1999,2000 Jakub Jelinek (jakub@redhat.com)
5 */
6
7 /*
8 * I like traps on v9, :))))
9 */
10
11 #include <linux/extable.h>
12 #include <linux/sched/mm.h>
13 #include <linux/sched/debug.h>
14 #include <linux/linkage.h>
15 #include <linux/kernel.h>
16 #include <linux/signal.h>
17 #include <linux/smp.h>
18 #include <linux/mm.h>
19 #include <linux/init.h>
20 #include <linux/kdebug.h>
21 #include <linux/ftrace.h>
22 #include <linux/reboot.h>
23 #include <linux/gfp.h>
24 #include <linux/context_tracking.h>
25
26 #include <asm/smp.h>
27 #include <asm/delay.h>
28 #include <asm/ptrace.h>
29 #include <asm/oplib.h>
30 #include <asm/page.h>
31 #include <asm/pgtable.h>
32 #include <asm/unistd.h>
33 #include <linux/uaccess.h>
34 #include <asm/fpumacro.h>
35 #include <asm/lsu.h>
36 #include <asm/dcu.h>
37 #include <asm/estate.h>
38 #include <asm/chafsr.h>
39 #include <asm/sfafsr.h>
40 #include <asm/psrcompat.h>
41 #include <asm/processor.h>
42 #include <asm/timer.h>
43 #include <asm/head.h>
44 #include <asm/prom.h>
45 #include <asm/memctrl.h>
46 #include <asm/cacheflush.h>
47 #include <asm/setup.h>
48
49 #include "entry.h"
50 #include "kernel.h"
51 #include "kstack.h"
52
53 /* When an irrecoverable trap occurs at tl > 0, the trap entry
54 * code logs the trap state registers at every level in the trap
55 * stack. It is found at (pt_regs + sizeof(pt_regs)) and the layout
56 * is as follows:
57 */
58 struct tl1_traplog {
59 struct {
60 unsigned long tstate;
61 unsigned long tpc;
62 unsigned long tnpc;
63 unsigned long tt;
64 } trapstack[4];
65 unsigned long tl;
66 };
67
68 static void dump_tl1_traplog(struct tl1_traplog *p)
69 {
70 int i, limit;
71
72 printk(KERN_EMERG "TRAPLOG: Error at trap level 0x%lx, "
73 "dumping track stack.\n", p->tl);
74
75 limit = (tlb_type == hypervisor) ? 2 : 4;
76 for (i = 0; i < limit; i++) {
77 printk(KERN_EMERG
78 "TRAPLOG: Trap level %d TSTATE[%016lx] TPC[%016lx] "
79 "TNPC[%016lx] TT[%lx]\n",
80 i + 1,
81 p->trapstack[i].tstate, p->trapstack[i].tpc,
82 p->trapstack[i].tnpc, p->trapstack[i].tt);
83 printk("TRAPLOG: TPC<%pS>\n", (void *) p->trapstack[i].tpc);
84 }
85 }
86
87 void bad_trap(struct pt_regs *regs, long lvl)
88 {
89 char buffer[36];
90 siginfo_t info;
91
92 if (notify_die(DIE_TRAP, "bad trap", regs,
93 0, lvl, SIGTRAP) == NOTIFY_STOP)
94 return;
95
96 if (lvl < 0x100) {
97 sprintf(buffer, "Bad hw trap %lx at tl0\n", lvl);
98 die_if_kernel(buffer, regs);
99 }
100
101 lvl -= 0x100;
102 if (regs->tstate & TSTATE_PRIV) {
103 sprintf(buffer, "Kernel bad sw trap %lx", lvl);
104 die_if_kernel(buffer, regs);
105 }
106 if (test_thread_flag(TIF_32BIT)) {
107 regs->tpc &= 0xffffffff;
108 regs->tnpc &= 0xffffffff;
109 }
110 info.si_signo = SIGILL;
111 info.si_errno = 0;
112 info.si_code = ILL_ILLTRP;
113 info.si_addr = (void __user *)regs->tpc;
114 info.si_trapno = lvl;
115 force_sig_info(SIGILL, &info, current);
116 }
117
118 void bad_trap_tl1(struct pt_regs *regs, long lvl)
119 {
120 char buffer[36];
121
122 if (notify_die(DIE_TRAP_TL1, "bad trap tl1", regs,
123 0, lvl, SIGTRAP) == NOTIFY_STOP)
124 return;
125
126 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
127
128 sprintf (buffer, "Bad trap %lx at tl>0", lvl);
129 die_if_kernel (buffer, regs);
130 }
131
132 #ifdef CONFIG_DEBUG_BUGVERBOSE
133 void do_BUG(const char *file, int line)
134 {
135 bust_spinlocks(1);
136 printk("kernel BUG at %s:%d!\n", file, line);
137 }
138 EXPORT_SYMBOL(do_BUG);
139 #endif
140
141 static DEFINE_SPINLOCK(dimm_handler_lock);
142 static dimm_printer_t dimm_handler;
143
144 static int sprintf_dimm(int synd_code, unsigned long paddr, char *buf, int buflen)
145 {
146 unsigned long flags;
147 int ret = -ENODEV;
148
149 spin_lock_irqsave(&dimm_handler_lock, flags);
150 if (dimm_handler) {
151 ret = dimm_handler(synd_code, paddr, buf, buflen);
152 } else if (tlb_type == spitfire) {
153 if (prom_getunumber(synd_code, paddr, buf, buflen) == -1)
154 ret = -EINVAL;
155 else
156 ret = 0;
157 } else
158 ret = -ENODEV;
159 spin_unlock_irqrestore(&dimm_handler_lock, flags);
160
161 return ret;
162 }
163
164 int register_dimm_printer(dimm_printer_t func)
165 {
166 unsigned long flags;
167 int ret = 0;
168
169 spin_lock_irqsave(&dimm_handler_lock, flags);
170 if (!dimm_handler)
171 dimm_handler = func;
172 else
173 ret = -EEXIST;
174 spin_unlock_irqrestore(&dimm_handler_lock, flags);
175
176 return ret;
177 }
178 EXPORT_SYMBOL_GPL(register_dimm_printer);
179
180 void unregister_dimm_printer(dimm_printer_t func)
181 {
182 unsigned long flags;
183
184 spin_lock_irqsave(&dimm_handler_lock, flags);
185 if (dimm_handler == func)
186 dimm_handler = NULL;
187 spin_unlock_irqrestore(&dimm_handler_lock, flags);
188 }
189 EXPORT_SYMBOL_GPL(unregister_dimm_printer);
190
191 void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
192 {
193 enum ctx_state prev_state = exception_enter();
194 siginfo_t info;
195
196 if (notify_die(DIE_TRAP, "instruction access exception", regs,
197 0, 0x8, SIGTRAP) == NOTIFY_STOP)
198 goto out;
199
200 if (regs->tstate & TSTATE_PRIV) {
201 printk("spitfire_insn_access_exception: SFSR[%016lx] "
202 "SFAR[%016lx], going.\n", sfsr, sfar);
203 die_if_kernel("Iax", regs);
204 }
205 if (test_thread_flag(TIF_32BIT)) {
206 regs->tpc &= 0xffffffff;
207 regs->tnpc &= 0xffffffff;
208 }
209 info.si_signo = SIGSEGV;
210 info.si_errno = 0;
211 info.si_code = SEGV_MAPERR;
212 info.si_addr = (void __user *)regs->tpc;
213 info.si_trapno = 0;
214 force_sig_info(SIGSEGV, &info, current);
215 out:
216 exception_exit(prev_state);
217 }
218
219 void spitfire_insn_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
220 {
221 if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs,
222 0, 0x8, SIGTRAP) == NOTIFY_STOP)
223 return;
224
225 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
226 spitfire_insn_access_exception(regs, sfsr, sfar);
227 }
228
229 void sun4v_insn_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
230 {
231 unsigned short type = (type_ctx >> 16);
232 unsigned short ctx = (type_ctx & 0xffff);
233 siginfo_t info;
234
235 if (notify_die(DIE_TRAP, "instruction access exception", regs,
236 0, 0x8, SIGTRAP) == NOTIFY_STOP)
237 return;
238
239 if (regs->tstate & TSTATE_PRIV) {
240 printk("sun4v_insn_access_exception: ADDR[%016lx] "
241 "CTX[%04x] TYPE[%04x], going.\n",
242 addr, ctx, type);
243 die_if_kernel("Iax", regs);
244 }
245
246 if (test_thread_flag(TIF_32BIT)) {
247 regs->tpc &= 0xffffffff;
248 regs->tnpc &= 0xffffffff;
249 }
250 info.si_signo = SIGSEGV;
251 info.si_errno = 0;
252 info.si_code = SEGV_MAPERR;
253 info.si_addr = (void __user *) addr;
254 info.si_trapno = 0;
255 force_sig_info(SIGSEGV, &info, current);
256 }
257
258 void sun4v_insn_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
259 {
260 if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs,
261 0, 0x8, SIGTRAP) == NOTIFY_STOP)
262 return;
263
264 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
265 sun4v_insn_access_exception(regs, addr, type_ctx);
266 }
267
268 bool is_no_fault_exception(struct pt_regs *regs)
269 {
270 unsigned char asi;
271 u32 insn;
272
273 if (get_user(insn, (u32 __user *)regs->tpc) == -EFAULT)
274 return false;
275
276 /*
277 * Must do a little instruction decoding here in order to
278 * decide on a course of action. The bits of interest are:
279 * insn[31:30] = op, where 3 indicates the load/store group
280 * insn[24:19] = op3, which identifies individual opcodes
281 * insn[13] indicates an immediate offset
282 * op3[4]=1 identifies alternate space instructions
283 * op3[5:4]=3 identifies floating point instructions
284 * op3[2]=1 identifies stores
285 * See "Opcode Maps" in the appendix of any Sparc V9
286 * architecture spec for full details.
287 */
288 if ((insn & 0xc0800000) == 0xc0800000) { /* op=3, op3[4]=1 */
289 if (insn & 0x2000) /* immediate offset */
290 asi = (regs->tstate >> 24); /* saved %asi */
291 else
292 asi = (insn >> 5); /* immediate asi */
293 if ((asi & 0xf2) == ASI_PNF) {
294 if (insn & 0x1000000) { /* op3[5:4]=3 */
295 handle_ldf_stq(insn, regs);
296 return true;
297 } else if (insn & 0x200000) { /* op3[2], stores */
298 return false;
299 }
300 handle_ld_nf(insn, regs);
301 return true;
302 }
303 }
304 return false;
305 }
306
307 void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
308 {
309 enum ctx_state prev_state = exception_enter();
310 siginfo_t info;
311
312 if (notify_die(DIE_TRAP, "data access exception", regs,
313 0, 0x30, SIGTRAP) == NOTIFY_STOP)
314 goto out;
315
316 if (regs->tstate & TSTATE_PRIV) {
317 /* Test if this comes from uaccess places. */
318 const struct exception_table_entry *entry;
319
320 entry = search_exception_tables(regs->tpc);
321 if (entry) {
322 /* Ouch, somebody is trying VM hole tricks on us... */
323 #ifdef DEBUG_EXCEPTIONS
324 printk("Exception: PC<%016lx> faddr<UNKNOWN>\n", regs->tpc);
325 printk("EX_TABLE: insn<%016lx> fixup<%016lx>\n",
326 regs->tpc, entry->fixup);
327 #endif
328 regs->tpc = entry->fixup;
329 regs->tnpc = regs->tpc + 4;
330 goto out;
331 }
332 /* Shit... */
333 printk("spitfire_data_access_exception: SFSR[%016lx] "
334 "SFAR[%016lx], going.\n", sfsr, sfar);
335 die_if_kernel("Dax", regs);
336 }
337
338 if (is_no_fault_exception(regs))
339 return;
340
341 info.si_signo = SIGSEGV;
342 info.si_errno = 0;
343 info.si_code = SEGV_MAPERR;
344 info.si_addr = (void __user *)sfar;
345 info.si_trapno = 0;
346 force_sig_info(SIGSEGV, &info, current);
347 out:
348 exception_exit(prev_state);
349 }
350
351 void spitfire_data_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
352 {
353 if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs,
354 0, 0x30, SIGTRAP) == NOTIFY_STOP)
355 return;
356
357 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
358 spitfire_data_access_exception(regs, sfsr, sfar);
359 }
360
361 void sun4v_data_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
362 {
363 unsigned short type = (type_ctx >> 16);
364 unsigned short ctx = (type_ctx & 0xffff);
365 siginfo_t info;
366
367 if (notify_die(DIE_TRAP, "data access exception", regs,
368 0, 0x8, SIGTRAP) == NOTIFY_STOP)
369 return;
370
371 if (regs->tstate & TSTATE_PRIV) {
372 /* Test if this comes from uaccess places. */
373 const struct exception_table_entry *entry;
374
375 entry = search_exception_tables(regs->tpc);
376 if (entry) {
377 /* Ouch, somebody is trying VM hole tricks on us... */
378 #ifdef DEBUG_EXCEPTIONS
379 printk("Exception: PC<%016lx> faddr<UNKNOWN>\n", regs->tpc);
380 printk("EX_TABLE: insn<%016lx> fixup<%016lx>\n",
381 regs->tpc, entry->fixup);
382 #endif
383 regs->tpc = entry->fixup;
384 regs->tnpc = regs->tpc + 4;
385 return;
386 }
387 printk("sun4v_data_access_exception: ADDR[%016lx] "
388 "CTX[%04x] TYPE[%04x], going.\n",
389 addr, ctx, type);
390 die_if_kernel("Dax", regs);
391 }
392
393 if (test_thread_flag(TIF_32BIT)) {
394 regs->tpc &= 0xffffffff;
395 regs->tnpc &= 0xffffffff;
396 }
397 if (is_no_fault_exception(regs))
398 return;
399
400 info.si_signo = SIGSEGV;
401 info.si_errno = 0;
402 info.si_code = SEGV_MAPERR;
403 info.si_addr = (void __user *) addr;
404 info.si_trapno = 0;
405 force_sig_info(SIGSEGV, &info, current);
406 }
407
408 void sun4v_data_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
409 {
410 if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs,
411 0, 0x8, SIGTRAP) == NOTIFY_STOP)
412 return;
413
414 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
415 sun4v_data_access_exception(regs, addr, type_ctx);
416 }
417
418 #ifdef CONFIG_PCI
419 #include "pci_impl.h"
420 #endif
421
422 /* When access exceptions happen, we must do this. */
423 static void spitfire_clean_and_reenable_l1_caches(void)
424 {
425 unsigned long va;
426
427 if (tlb_type != spitfire)
428 BUG();
429
430 /* Clean 'em. */
431 for (va = 0; va < (PAGE_SIZE << 1); va += 32) {
432 spitfire_put_icache_tag(va, 0x0);
433 spitfire_put_dcache_tag(va, 0x0);
434 }
435
436 /* Re-enable in LSU. */
437 __asm__ __volatile__("flush %%g6\n\t"
438 "membar #Sync\n\t"
439 "stxa %0, [%%g0] %1\n\t"
440 "membar #Sync"
441 : /* no outputs */
442 : "r" (LSU_CONTROL_IC | LSU_CONTROL_DC |
443 LSU_CONTROL_IM | LSU_CONTROL_DM),
444 "i" (ASI_LSU_CONTROL)
445 : "memory");
446 }
447
448 static void spitfire_enable_estate_errors(void)
449 {
450 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
451 "membar #Sync"
452 : /* no outputs */
453 : "r" (ESTATE_ERR_ALL),
454 "i" (ASI_ESTATE_ERROR_EN));
455 }
456
457 static char ecc_syndrome_table[] = {
458 0x4c, 0x40, 0x41, 0x48, 0x42, 0x48, 0x48, 0x49,
459 0x43, 0x48, 0x48, 0x49, 0x48, 0x49, 0x49, 0x4a,
460 0x44, 0x48, 0x48, 0x20, 0x48, 0x39, 0x4b, 0x48,
461 0x48, 0x25, 0x31, 0x48, 0x28, 0x48, 0x48, 0x2c,
462 0x45, 0x48, 0x48, 0x21, 0x48, 0x3d, 0x04, 0x48,
463 0x48, 0x4b, 0x35, 0x48, 0x2d, 0x48, 0x48, 0x29,
464 0x48, 0x00, 0x01, 0x48, 0x0a, 0x48, 0x48, 0x4b,
465 0x0f, 0x48, 0x48, 0x4b, 0x48, 0x49, 0x49, 0x48,
466 0x46, 0x48, 0x48, 0x2a, 0x48, 0x3b, 0x27, 0x48,
467 0x48, 0x4b, 0x33, 0x48, 0x22, 0x48, 0x48, 0x2e,
468 0x48, 0x19, 0x1d, 0x48, 0x1b, 0x4a, 0x48, 0x4b,
469 0x1f, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48,
470 0x48, 0x4b, 0x24, 0x48, 0x07, 0x48, 0x48, 0x36,
471 0x4b, 0x48, 0x48, 0x3e, 0x48, 0x30, 0x38, 0x48,
472 0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x16, 0x48,
473 0x48, 0x12, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b,
474 0x47, 0x48, 0x48, 0x2f, 0x48, 0x3f, 0x4b, 0x48,
475 0x48, 0x06, 0x37, 0x48, 0x23, 0x48, 0x48, 0x2b,
476 0x48, 0x05, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x32,
477 0x26, 0x48, 0x48, 0x3a, 0x48, 0x34, 0x3c, 0x48,
478 0x48, 0x11, 0x15, 0x48, 0x13, 0x4a, 0x48, 0x4b,
479 0x17, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48,
480 0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x1e, 0x48,
481 0x48, 0x1a, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b,
482 0x48, 0x08, 0x0d, 0x48, 0x02, 0x48, 0x48, 0x49,
483 0x03, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x4b, 0x48,
484 0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x10, 0x48,
485 0x48, 0x14, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b,
486 0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x18, 0x48,
487 0x48, 0x1c, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b,
488 0x4a, 0x0c, 0x09, 0x48, 0x0e, 0x48, 0x48, 0x4b,
489 0x0b, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x4b, 0x4a
490 };
491
492 static char *syndrome_unknown = "<Unknown>";
493
494 static void spitfire_log_udb_syndrome(unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long bit)
495 {
496 unsigned short scode;
497 char memmod_str[64], *p;
498
499 if (udbl & bit) {
500 scode = ecc_syndrome_table[udbl & 0xff];
501 if (sprintf_dimm(scode, afar, memmod_str, sizeof(memmod_str)) < 0)
502 p = syndrome_unknown;
503 else
504 p = memmod_str;
505 printk(KERN_WARNING "CPU[%d]: UDBL Syndrome[%x] "
506 "Memory Module \"%s\"\n",
507 smp_processor_id(), scode, p);
508 }
509
510 if (udbh & bit) {
511 scode = ecc_syndrome_table[udbh & 0xff];
512 if (sprintf_dimm(scode, afar, memmod_str, sizeof(memmod_str)) < 0)
513 p = syndrome_unknown;
514 else
515 p = memmod_str;
516 printk(KERN_WARNING "CPU[%d]: UDBH Syndrome[%x] "
517 "Memory Module \"%s\"\n",
518 smp_processor_id(), scode, p);
519 }
520
521 }
522
523 static void spitfire_cee_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, int tl1, struct pt_regs *regs)
524 {
525
526 printk(KERN_WARNING "CPU[%d]: Correctable ECC Error "
527 "AFSR[%lx] AFAR[%016lx] UDBL[%lx] UDBH[%lx] TL>1[%d]\n",
528 smp_processor_id(), afsr, afar, udbl, udbh, tl1);
529
530 spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_CE);
531
532 /* We always log it, even if someone is listening for this
533 * trap.
534 */
535 notify_die(DIE_TRAP, "Correctable ECC Error", regs,
536 0, TRAP_TYPE_CEE, SIGTRAP);
537
538 /* The Correctable ECC Error trap does not disable I/D caches. So
539 * we only have to restore the ESTATE Error Enable register.
540 */
541 spitfire_enable_estate_errors();
542 }
543
544 static void spitfire_ue_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long tt, int tl1, struct pt_regs *regs)
545 {
546 siginfo_t info;
547
548 printk(KERN_WARNING "CPU[%d]: Uncorrectable Error AFSR[%lx] "
549 "AFAR[%lx] UDBL[%lx] UDBH[%ld] TT[%lx] TL>1[%d]\n",
550 smp_processor_id(), afsr, afar, udbl, udbh, tt, tl1);
551
552 /* XXX add more human friendly logging of the error status
553 * XXX as is implemented for cheetah
554 */
555
556 spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_UE);
557
558 /* We always log it, even if someone is listening for this
559 * trap.
560 */
561 notify_die(DIE_TRAP, "Uncorrectable Error", regs,
562 0, tt, SIGTRAP);
563
564 if (regs->tstate & TSTATE_PRIV) {
565 if (tl1)
566 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
567 die_if_kernel("UE", regs);
568 }
569
570 /* XXX need more intelligent processing here, such as is implemented
571 * XXX for cheetah errors, in fact if the E-cache still holds the
572 * XXX line with bad parity this will loop
573 */
574
575 spitfire_clean_and_reenable_l1_caches();
576 spitfire_enable_estate_errors();
577
578 if (test_thread_flag(TIF_32BIT)) {
579 regs->tpc &= 0xffffffff;
580 regs->tnpc &= 0xffffffff;
581 }
582 info.si_signo = SIGBUS;
583 info.si_errno = 0;
584 info.si_code = BUS_OBJERR;
585 info.si_addr = (void *)0;
586 info.si_trapno = 0;
587 force_sig_info(SIGBUS, &info, current);
588 }
589
590 void spitfire_access_error(struct pt_regs *regs, unsigned long status_encoded, unsigned long afar)
591 {
592 unsigned long afsr, tt, udbh, udbl;
593 int tl1;
594
595 afsr = (status_encoded & SFSTAT_AFSR_MASK) >> SFSTAT_AFSR_SHIFT;
596 tt = (status_encoded & SFSTAT_TRAP_TYPE) >> SFSTAT_TRAP_TYPE_SHIFT;
597 tl1 = (status_encoded & SFSTAT_TL_GT_ONE) ? 1 : 0;
598 udbl = (status_encoded & SFSTAT_UDBL_MASK) >> SFSTAT_UDBL_SHIFT;
599 udbh = (status_encoded & SFSTAT_UDBH_MASK) >> SFSTAT_UDBH_SHIFT;
600
601 #ifdef CONFIG_PCI
602 if (tt == TRAP_TYPE_DAE &&
603 pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) {
604 spitfire_clean_and_reenable_l1_caches();
605 spitfire_enable_estate_errors();
606
607 pci_poke_faulted = 1;
608 regs->tnpc = regs->tpc + 4;
609 return;
610 }
611 #endif
612
613 if (afsr & SFAFSR_UE)
614 spitfire_ue_log(afsr, afar, udbh, udbl, tt, tl1, regs);
615
616 if (tt == TRAP_TYPE_CEE) {
617 /* Handle the case where we took a CEE trap, but ACK'd
618 * only the UE state in the UDB error registers.
619 */
620 if (afsr & SFAFSR_UE) {
621 if (udbh & UDBE_CE) {
622 __asm__ __volatile__(
623 "stxa %0, [%1] %2\n\t"
624 "membar #Sync"
625 : /* no outputs */
626 : "r" (udbh & UDBE_CE),
627 "r" (0x0), "i" (ASI_UDB_ERROR_W));
628 }
629 if (udbl & UDBE_CE) {
630 __asm__ __volatile__(
631 "stxa %0, [%1] %2\n\t"
632 "membar #Sync"
633 : /* no outputs */
634 : "r" (udbl & UDBE_CE),
635 "r" (0x18), "i" (ASI_UDB_ERROR_W));
636 }
637 }
638
639 spitfire_cee_log(afsr, afar, udbh, udbl, tl1, regs);
640 }
641 }
642
643 int cheetah_pcache_forced_on;
644
645 void cheetah_enable_pcache(void)
646 {
647 unsigned long dcr;
648
649 printk("CHEETAH: Enabling P-Cache on cpu %d.\n",
650 smp_processor_id());
651
652 __asm__ __volatile__("ldxa [%%g0] %1, %0"
653 : "=r" (dcr)
654 : "i" (ASI_DCU_CONTROL_REG));
655 dcr |= (DCU_PE | DCU_HPE | DCU_SPE | DCU_SL);
656 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
657 "membar #Sync"
658 : /* no outputs */
659 : "r" (dcr), "i" (ASI_DCU_CONTROL_REG));
660 }
661
662 /* Cheetah error trap handling. */
663 static unsigned long ecache_flush_physbase;
664 static unsigned long ecache_flush_linesize;
665 static unsigned long ecache_flush_size;
666
667 /* This table is ordered in priority of errors and matches the
668 * AFAR overwrite policy as well.
669 */
670
671 struct afsr_error_table {
672 unsigned long mask;
673 const char *name;
674 };
675
676 static const char CHAFSR_PERR_msg[] =
677 "System interface protocol error";
678 static const char CHAFSR_IERR_msg[] =
679 "Internal processor error";
680 static const char CHAFSR_ISAP_msg[] =
681 "System request parity error on incoming address";
682 static const char CHAFSR_UCU_msg[] =
683 "Uncorrectable E-cache ECC error for ifetch/data";
684 static const char CHAFSR_UCC_msg[] =
685 "SW Correctable E-cache ECC error for ifetch/data";
686 static const char CHAFSR_UE_msg[] =
687 "Uncorrectable system bus data ECC error for read";
688 static const char CHAFSR_EDU_msg[] =
689 "Uncorrectable E-cache ECC error for stmerge/blkld";
690 static const char CHAFSR_EMU_msg[] =
691 "Uncorrectable system bus MTAG error";
692 static const char CHAFSR_WDU_msg[] =
693 "Uncorrectable E-cache ECC error for writeback";
694 static const char CHAFSR_CPU_msg[] =
695 "Uncorrectable ECC error for copyout";
696 static const char CHAFSR_CE_msg[] =
697 "HW corrected system bus data ECC error for read";
698 static const char CHAFSR_EDC_msg[] =
699 "HW corrected E-cache ECC error for stmerge/blkld";
700 static const char CHAFSR_EMC_msg[] =
701 "HW corrected system bus MTAG ECC error";
702 static const char CHAFSR_WDC_msg[] =
703 "HW corrected E-cache ECC error for writeback";
704 static const char CHAFSR_CPC_msg[] =
705 "HW corrected ECC error for copyout";
706 static const char CHAFSR_TO_msg[] =
707 "Unmapped error from system bus";
708 static const char CHAFSR_BERR_msg[] =
709 "Bus error response from system bus";
710 static const char CHAFSR_IVC_msg[] =
711 "HW corrected system bus data ECC error for ivec read";
712 static const char CHAFSR_IVU_msg[] =
713 "Uncorrectable system bus data ECC error for ivec read";
714 static struct afsr_error_table __cheetah_error_table[] = {
715 { CHAFSR_PERR, CHAFSR_PERR_msg },
716 { CHAFSR_IERR, CHAFSR_IERR_msg },
717 { CHAFSR_ISAP, CHAFSR_ISAP_msg },
718 { CHAFSR_UCU, CHAFSR_UCU_msg },
719 { CHAFSR_UCC, CHAFSR_UCC_msg },
720 { CHAFSR_UE, CHAFSR_UE_msg },
721 { CHAFSR_EDU, CHAFSR_EDU_msg },
722 { CHAFSR_EMU, CHAFSR_EMU_msg },
723 { CHAFSR_WDU, CHAFSR_WDU_msg },
724 { CHAFSR_CPU, CHAFSR_CPU_msg },
725 { CHAFSR_CE, CHAFSR_CE_msg },
726 { CHAFSR_EDC, CHAFSR_EDC_msg },
727 { CHAFSR_EMC, CHAFSR_EMC_msg },
728 { CHAFSR_WDC, CHAFSR_WDC_msg },
729 { CHAFSR_CPC, CHAFSR_CPC_msg },
730 { CHAFSR_TO, CHAFSR_TO_msg },
731 { CHAFSR_BERR, CHAFSR_BERR_msg },
732 /* These two do not update the AFAR. */
733 { CHAFSR_IVC, CHAFSR_IVC_msg },
734 { CHAFSR_IVU, CHAFSR_IVU_msg },
735 { 0, NULL },
736 };
737 static const char CHPAFSR_DTO_msg[] =
738 "System bus unmapped error for prefetch/storequeue-read";
739 static const char CHPAFSR_DBERR_msg[] =
740 "System bus error for prefetch/storequeue-read";
741 static const char CHPAFSR_THCE_msg[] =
742 "Hardware corrected E-cache Tag ECC error";
743 static const char CHPAFSR_TSCE_msg[] =
744 "SW handled correctable E-cache Tag ECC error";
745 static const char CHPAFSR_TUE_msg[] =
746 "Uncorrectable E-cache Tag ECC error";
747 static const char CHPAFSR_DUE_msg[] =
748 "System bus uncorrectable data ECC error due to prefetch/store-fill";
749 static struct afsr_error_table __cheetah_plus_error_table[] = {
750 { CHAFSR_PERR, CHAFSR_PERR_msg },
751 { CHAFSR_IERR, CHAFSR_IERR_msg },
752 { CHAFSR_ISAP, CHAFSR_ISAP_msg },
753 { CHAFSR_UCU, CHAFSR_UCU_msg },
754 { CHAFSR_UCC, CHAFSR_UCC_msg },
755 { CHAFSR_UE, CHAFSR_UE_msg },
756 { CHAFSR_EDU, CHAFSR_EDU_msg },
757 { CHAFSR_EMU, CHAFSR_EMU_msg },
758 { CHAFSR_WDU, CHAFSR_WDU_msg },
759 { CHAFSR_CPU, CHAFSR_CPU_msg },
760 { CHAFSR_CE, CHAFSR_CE_msg },
761 { CHAFSR_EDC, CHAFSR_EDC_msg },
762 { CHAFSR_EMC, CHAFSR_EMC_msg },
763 { CHAFSR_WDC, CHAFSR_WDC_msg },
764 { CHAFSR_CPC, CHAFSR_CPC_msg },
765 { CHAFSR_TO, CHAFSR_TO_msg },
766 { CHAFSR_BERR, CHAFSR_BERR_msg },
767 { CHPAFSR_DTO, CHPAFSR_DTO_msg },
768 { CHPAFSR_DBERR, CHPAFSR_DBERR_msg },
769 { CHPAFSR_THCE, CHPAFSR_THCE_msg },
770 { CHPAFSR_TSCE, CHPAFSR_TSCE_msg },
771 { CHPAFSR_TUE, CHPAFSR_TUE_msg },
772 { CHPAFSR_DUE, CHPAFSR_DUE_msg },
773 /* These two do not update the AFAR. */
774 { CHAFSR_IVC, CHAFSR_IVC_msg },
775 { CHAFSR_IVU, CHAFSR_IVU_msg },
776 { 0, NULL },
777 };
778 static const char JPAFSR_JETO_msg[] =
779 "System interface protocol error, hw timeout caused";
780 static const char JPAFSR_SCE_msg[] =
781 "Parity error on system snoop results";
782 static const char JPAFSR_JEIC_msg[] =
783 "System interface protocol error, illegal command detected";
784 static const char JPAFSR_JEIT_msg[] =
785 "System interface protocol error, illegal ADTYPE detected";
786 static const char JPAFSR_OM_msg[] =
787 "Out of range memory error has occurred";
788 static const char JPAFSR_ETP_msg[] =
789 "Parity error on L2 cache tag SRAM";
790 static const char JPAFSR_UMS_msg[] =
791 "Error due to unsupported store";
792 static const char JPAFSR_RUE_msg[] =
793 "Uncorrectable ECC error from remote cache/memory";
794 static const char JPAFSR_RCE_msg[] =
795 "Correctable ECC error from remote cache/memory";
796 static const char JPAFSR_BP_msg[] =
797 "JBUS parity error on returned read data";
798 static const char JPAFSR_WBP_msg[] =
799 "JBUS parity error on data for writeback or block store";
800 static const char JPAFSR_FRC_msg[] =
801 "Foreign read to DRAM incurring correctable ECC error";
802 static const char JPAFSR_FRU_msg[] =
803 "Foreign read to DRAM incurring uncorrectable ECC error";
804 static struct afsr_error_table __jalapeno_error_table[] = {
805 { JPAFSR_JETO, JPAFSR_JETO_msg },
806 { JPAFSR_SCE, JPAFSR_SCE_msg },
807 { JPAFSR_JEIC, JPAFSR_JEIC_msg },
808 { JPAFSR_JEIT, JPAFSR_JEIT_msg },
809 { CHAFSR_PERR, CHAFSR_PERR_msg },
810 { CHAFSR_IERR, CHAFSR_IERR_msg },
811 { CHAFSR_ISAP, CHAFSR_ISAP_msg },
812 { CHAFSR_UCU, CHAFSR_UCU_msg },
813 { CHAFSR_UCC, CHAFSR_UCC_msg },
814 { CHAFSR_UE, CHAFSR_UE_msg },
815 { CHAFSR_EDU, CHAFSR_EDU_msg },
816 { JPAFSR_OM, JPAFSR_OM_msg },
817 { CHAFSR_WDU, CHAFSR_WDU_msg },
818 { CHAFSR_CPU, CHAFSR_CPU_msg },
819 { CHAFSR_CE, CHAFSR_CE_msg },
820 { CHAFSR_EDC, CHAFSR_EDC_msg },
821 { JPAFSR_ETP, JPAFSR_ETP_msg },
822 { CHAFSR_WDC, CHAFSR_WDC_msg },
823 { CHAFSR_CPC, CHAFSR_CPC_msg },
824 { CHAFSR_TO, CHAFSR_TO_msg },
825 { CHAFSR_BERR, CHAFSR_BERR_msg },
826 { JPAFSR_UMS, JPAFSR_UMS_msg },
827 { JPAFSR_RUE, JPAFSR_RUE_msg },
828 { JPAFSR_RCE, JPAFSR_RCE_msg },
829 { JPAFSR_BP, JPAFSR_BP_msg },
830 { JPAFSR_WBP, JPAFSR_WBP_msg },
831 { JPAFSR_FRC, JPAFSR_FRC_msg },
832 { JPAFSR_FRU, JPAFSR_FRU_msg },
833 /* These two do not update the AFAR. */
834 { CHAFSR_IVU, CHAFSR_IVU_msg },
835 { 0, NULL },
836 };
837 static struct afsr_error_table *cheetah_error_table;
838 static unsigned long cheetah_afsr_errors;
839
840 struct cheetah_err_info *cheetah_error_log;
841
842 static inline struct cheetah_err_info *cheetah_get_error_log(unsigned long afsr)
843 {
844 struct cheetah_err_info *p;
845 int cpu = smp_processor_id();
846
847 if (!cheetah_error_log)
848 return NULL;
849
850 p = cheetah_error_log + (cpu * 2);
851 if ((afsr & CHAFSR_TL1) != 0UL)
852 p++;
853
854 return p;
855 }
856
857 extern unsigned int tl0_icpe[], tl1_icpe[];
858 extern unsigned int tl0_dcpe[], tl1_dcpe[];
859 extern unsigned int tl0_fecc[], tl1_fecc[];
860 extern unsigned int tl0_cee[], tl1_cee[];
861 extern unsigned int tl0_iae[], tl1_iae[];
862 extern unsigned int tl0_dae[], tl1_dae[];
863 extern unsigned int cheetah_plus_icpe_trap_vector[], cheetah_plus_icpe_trap_vector_tl1[];
864 extern unsigned int cheetah_plus_dcpe_trap_vector[], cheetah_plus_dcpe_trap_vector_tl1[];
865 extern unsigned int cheetah_fecc_trap_vector[], cheetah_fecc_trap_vector_tl1[];
866 extern unsigned int cheetah_cee_trap_vector[], cheetah_cee_trap_vector_tl1[];
867 extern unsigned int cheetah_deferred_trap_vector[], cheetah_deferred_trap_vector_tl1[];
868
869 void __init cheetah_ecache_flush_init(void)
870 {
871 unsigned long largest_size, smallest_linesize, order, ver;
872 int i, sz;
873
874 /* Scan all cpu device tree nodes, note two values:
875 * 1) largest E-cache size
876 * 2) smallest E-cache line size
877 */
878 largest_size = 0UL;
879 smallest_linesize = ~0UL;
880
881 for (i = 0; i < NR_CPUS; i++) {
882 unsigned long val;
883
884 val = cpu_data(i).ecache_size;
885 if (!val)
886 continue;
887
888 if (val > largest_size)
889 largest_size = val;
890
891 val = cpu_data(i).ecache_line_size;
892 if (val < smallest_linesize)
893 smallest_linesize = val;
894
895 }
896
897 if (largest_size == 0UL || smallest_linesize == ~0UL) {
898 prom_printf("cheetah_ecache_flush_init: Cannot probe cpu E-cache "
899 "parameters.\n");
900 prom_halt();
901 }
902
903 ecache_flush_size = (2 * largest_size);
904 ecache_flush_linesize = smallest_linesize;
905
906 ecache_flush_physbase = find_ecache_flush_span(ecache_flush_size);
907
908 if (ecache_flush_physbase == ~0UL) {
909 prom_printf("cheetah_ecache_flush_init: Cannot find %ld byte "
910 "contiguous physical memory.\n",
911 ecache_flush_size);
912 prom_halt();
913 }
914
915 /* Now allocate error trap reporting scoreboard. */
916 sz = NR_CPUS * (2 * sizeof(struct cheetah_err_info));
917 for (order = 0; order < MAX_ORDER; order++) {
918 if ((PAGE_SIZE << order) >= sz)
919 break;
920 }
921 cheetah_error_log = (struct cheetah_err_info *)
922 __get_free_pages(GFP_KERNEL, order);
923 if (!cheetah_error_log) {
924 prom_printf("cheetah_ecache_flush_init: Failed to allocate "
925 "error logging scoreboard (%d bytes).\n", sz);
926 prom_halt();
927 }
928 memset(cheetah_error_log, 0, PAGE_SIZE << order);
929
930 /* Mark all AFSRs as invalid so that the trap handler will
931 * log new new information there.
932 */
933 for (i = 0; i < 2 * NR_CPUS; i++)
934 cheetah_error_log[i].afsr = CHAFSR_INVALID;
935
936 __asm__ ("rdpr %%ver, %0" : "=r" (ver));
937 if ((ver >> 32) == __JALAPENO_ID ||
938 (ver >> 32) == __SERRANO_ID) {
939 cheetah_error_table = &__jalapeno_error_table[0];
940 cheetah_afsr_errors = JPAFSR_ERRORS;
941 } else if ((ver >> 32) == 0x003e0015) {
942 cheetah_error_table = &__cheetah_plus_error_table[0];
943 cheetah_afsr_errors = CHPAFSR_ERRORS;
944 } else {
945 cheetah_error_table = &__cheetah_error_table[0];
946 cheetah_afsr_errors = CHAFSR_ERRORS;
947 }
948
949 /* Now patch trap tables. */
950 memcpy(tl0_fecc, cheetah_fecc_trap_vector, (8 * 4));
951 memcpy(tl1_fecc, cheetah_fecc_trap_vector_tl1, (8 * 4));
952 memcpy(tl0_cee, cheetah_cee_trap_vector, (8 * 4));
953 memcpy(tl1_cee, cheetah_cee_trap_vector_tl1, (8 * 4));
954 memcpy(tl0_iae, cheetah_deferred_trap_vector, (8 * 4));
955 memcpy(tl1_iae, cheetah_deferred_trap_vector_tl1, (8 * 4));
956 memcpy(tl0_dae, cheetah_deferred_trap_vector, (8 * 4));
957 memcpy(tl1_dae, cheetah_deferred_trap_vector_tl1, (8 * 4));
958 if (tlb_type == cheetah_plus) {
959 memcpy(tl0_dcpe, cheetah_plus_dcpe_trap_vector, (8 * 4));
960 memcpy(tl1_dcpe, cheetah_plus_dcpe_trap_vector_tl1, (8 * 4));
961 memcpy(tl0_icpe, cheetah_plus_icpe_trap_vector, (8 * 4));
962 memcpy(tl1_icpe, cheetah_plus_icpe_trap_vector_tl1, (8 * 4));
963 }
964 flushi(PAGE_OFFSET);
965 }
966
967 static void cheetah_flush_ecache(void)
968 {
969 unsigned long flush_base = ecache_flush_physbase;
970 unsigned long flush_linesize = ecache_flush_linesize;
971 unsigned long flush_size = ecache_flush_size;
972
973 __asm__ __volatile__("1: subcc %0, %4, %0\n\t"
974 " bne,pt %%xcc, 1b\n\t"
975 " ldxa [%2 + %0] %3, %%g0\n\t"
976 : "=&r" (flush_size)
977 : "" (flush_size), "r" (flush_base),
978 "i" (ASI_PHYS_USE_EC), "r" (flush_linesize));
979 }
980
981 static void cheetah_flush_ecache_line(unsigned long physaddr)
982 {
983 unsigned long alias;
984
985 physaddr &= ~(8UL - 1UL);
986 physaddr = (ecache_flush_physbase +
987 (physaddr & ((ecache_flush_size>>1UL) - 1UL)));
988 alias = physaddr + (ecache_flush_size >> 1UL);
989 __asm__ __volatile__("ldxa [%0] %2, %%g0\n\t"
990 "ldxa [%1] %2, %%g0\n\t"
991 "membar #Sync"
992 : /* no outputs */
993 : "r" (physaddr), "r" (alias),
994 "i" (ASI_PHYS_USE_EC));
995 }
996
997 /* Unfortunately, the diagnostic access to the I-cache tags we need to
998 * use to clear the thing interferes with I-cache coherency transactions.
999 *
1000 * So we must only flush the I-cache when it is disabled.
1001 */
1002 static void __cheetah_flush_icache(void)
1003 {
1004 unsigned int icache_size, icache_line_size;
1005 unsigned long addr;
1006
1007 icache_size = local_cpu_data().icache_size;
1008 icache_line_size = local_cpu_data().icache_line_size;
1009
1010 /* Clear the valid bits in all the tags. */
1011 for (addr = 0; addr < icache_size; addr += icache_line_size) {
1012 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
1013 "membar #Sync"
1014 : /* no outputs */
1015 : "r" (addr | (2 << 3)),
1016 "i" (ASI_IC_TAG));
1017 }
1018 }
1019
1020 static void cheetah_flush_icache(void)
1021 {
1022 unsigned long dcu_save;
1023
1024 /* Save current DCU, disable I-cache. */
1025 __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
1026 "or %0, %2, %%g1\n\t"
1027 "stxa %%g1, [%%g0] %1\n\t"
1028 "membar #Sync"
1029 : "=r" (dcu_save)
1030 : "i" (ASI_DCU_CONTROL_REG), "i" (DCU_IC)
1031 : "g1");
1032
1033 __cheetah_flush_icache();
1034
1035 /* Restore DCU register */
1036 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
1037 "membar #Sync"
1038 : /* no outputs */
1039 : "r" (dcu_save), "i" (ASI_DCU_CONTROL_REG));
1040 }
1041
1042 static void cheetah_flush_dcache(void)
1043 {
1044 unsigned int dcache_size, dcache_line_size;
1045 unsigned long addr;
1046
1047 dcache_size = local_cpu_data().dcache_size;
1048 dcache_line_size = local_cpu_data().dcache_line_size;
1049
1050 for (addr = 0; addr < dcache_size; addr += dcache_line_size) {
1051 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
1052 "membar #Sync"
1053 : /* no outputs */
1054 : "r" (addr), "i" (ASI_DCACHE_TAG));
1055 }
1056 }
1057
1058 /* In order to make the even parity correct we must do two things.
1059 * First, we clear DC_data_parity and set DC_utag to an appropriate value.
1060 * Next, we clear out all 32-bytes of data for that line. Data of
1061 * all-zero + tag parity value of zero == correct parity.
1062 */
1063 static void cheetah_plus_zap_dcache_parity(void)
1064 {
1065 unsigned int dcache_size, dcache_line_size;
1066 unsigned long addr;
1067
1068 dcache_size = local_cpu_data().dcache_size;
1069 dcache_line_size = local_cpu_data().dcache_line_size;
1070
1071 for (addr = 0; addr < dcache_size; addr += dcache_line_size) {
1072 unsigned long tag = (addr >> 14);
1073 unsigned long line;
1074
1075 __asm__ __volatile__("membar #Sync\n\t"
1076 "stxa %0, [%1] %2\n\t"
1077 "membar #Sync"
1078 : /* no outputs */
1079 : "r" (tag), "r" (addr),
1080 "i" (ASI_DCACHE_UTAG));
1081 for (line = addr; line < addr + dcache_line_size; line += 8)
1082 __asm__ __volatile__("membar #Sync\n\t"
1083 "stxa %%g0, [%0] %1\n\t"
1084 "membar #Sync"
1085 : /* no outputs */
1086 : "r" (line),
1087 "i" (ASI_DCACHE_DATA));
1088 }
1089 }
1090
1091 /* Conversion tables used to frob Cheetah AFSR syndrome values into
1092 * something palatable to the memory controller driver get_unumber
1093 * routine.
1094 */
1095 #define MT0 137
1096 #define MT1 138
1097 #define MT2 139
1098 #define NONE 254
1099 #define MTC0 140
1100 #define MTC1 141
1101 #define MTC2 142
1102 #define MTC3 143
1103 #define C0 128
1104 #define C1 129
1105 #define C2 130
1106 #define C3 131
1107 #define C4 132
1108 #define C5 133
1109 #define C6 134
1110 #define C7 135
1111 #define C8 136
1112 #define M2 144
1113 #define M3 145
1114 #define M4 146
1115 #define M 147
1116 static unsigned char cheetah_ecc_syntab[] = {
1117 /*00*/NONE, C0, C1, M2, C2, M2, M3, 47, C3, M2, M2, 53, M2, 41, 29, M,
1118 /*01*/C4, M, M, 50, M2, 38, 25, M2, M2, 33, 24, M2, 11, M, M2, 16,
1119 /*02*/C5, M, M, 46, M2, 37, 19, M2, M, 31, 32, M, 7, M2, M2, 10,
1120 /*03*/M2, 40, 13, M2, 59, M, M2, 66, M, M2, M2, 0, M2, 67, 71, M,
1121 /*04*/C6, M, M, 43, M, 36, 18, M, M2, 49, 15, M, 63, M2, M2, 6,
1122 /*05*/M2, 44, 28, M2, M, M2, M2, 52, 68, M2, M2, 62, M2, M3, M3, M4,
1123 /*06*/M2, 26, 106, M2, 64, M, M2, 2, 120, M, M2, M3, M, M3, M3, M4,
1124 /*07*/116, M2, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3,
1125 /*08*/C7, M2, M, 42, M, 35, 17, M2, M, 45, 14, M2, 21, M2, M2, 5,
1126 /*09*/M, 27, M, M, 99, M, M, 3, 114, M2, M2, 20, M2, M3, M3, M,
1127 /*0a*/M2, 23, 113, M2, 112, M2, M, 51, 95, M, M2, M3, M2, M3, M3, M2,
1128 /*0b*/103, M, M2, M3, M2, M3, M3, M4, M2, 48, M, M, 73, M2, M, M3,
1129 /*0c*/M2, 22, 110, M2, 109, M2, M, 9, 108, M2, M, M3, M2, M3, M3, M,
1130 /*0d*/102, M2, M, M, M2, M3, M3, M, M2, M3, M3, M2, M, M4, M, M3,
1131 /*0e*/98, M, M2, M3, M2, M, M3, M4, M2, M3, M3, M4, M3, M, M, M,
1132 /*0f*/M2, M3, M3, M, M3, M, M, M, 56, M4, M, M3, M4, M, M, M,
1133 /*10*/C8, M, M2, 39, M, 34, 105, M2, M, 30, 104, M, 101, M, M, 4,
1134 /*11*/M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, M2, M, M3, M,
1135 /*12*/M2, 97, 82, M2, 78, M2, M2, 1, 96, M, M, M, M, M, M3, M2,
1136 /*13*/94, M, M2, M3, M2, M, M3, M, M2, M, 79, M, 69, M, M4, M,
1137 /*14*/M2, 93, 92, M, 91, M, M2, 8, 90, M2, M2, M, M, M, M, M4,
1138 /*15*/89, M, M, M3, M2, M3, M3, M, M, M, M3, M2, M3, M2, M, M3,
1139 /*16*/86, M, M2, M3, M2, M, M3, M, M2, M, M3, M, M3, M, M, M3,
1140 /*17*/M, M, M3, M2, M3, M2, M4, M, 60, M, M2, M3, M4, M, M, M2,
1141 /*18*/M2, 88, 85, M2, 84, M, M2, 55, 81, M2, M2, M3, M2, M3, M3, M4,
1142 /*19*/77, M, M, M, M2, M3, M, M, M2, M3, M3, M4, M3, M2, M, M,
1143 /*1a*/74, M, M2, M3, M, M, M3, M, M, M, M3, M, M3, M, M4, M3,
1144 /*1b*/M2, 70, 107, M4, 65, M2, M2, M, 127, M, M, M, M2, M3, M3, M,
1145 /*1c*/80, M2, M2, 72, M, 119, 118, M, M2, 126, 76, M, 125, M, M4, M3,
1146 /*1d*/M2, 115, 124, M, 75, M, M, M3, 61, M, M4, M, M4, M, M, M,
1147 /*1e*/M, 123, 122, M4, 121, M4, M, M3, 117, M2, M2, M3, M4, M3, M, M,
1148 /*1f*/111, M, M, M, M4, M3, M3, M, M, M, M3, M, M3, M2, M, M
1149 };
1150 static unsigned char cheetah_mtag_syntab[] = {
1151 NONE, MTC0,
1152 MTC1, NONE,
1153 MTC2, NONE,
1154 NONE, MT0,
1155 MTC3, NONE,
1156 NONE, MT1,
1157 NONE, MT2,
1158 NONE, NONE
1159 };
1160
1161 /* Return the highest priority error conditon mentioned. */
1162 static inline unsigned long cheetah_get_hipri(unsigned long afsr)
1163 {
1164 unsigned long tmp = 0;
1165 int i;
1166
1167 for (i = 0; cheetah_error_table[i].mask; i++) {
1168 if ((tmp = (afsr & cheetah_error_table[i].mask)) != 0UL)
1169 return tmp;
1170 }
1171 return tmp;
1172 }
1173
1174 static const char *cheetah_get_string(unsigned long bit)
1175 {
1176 int i;
1177
1178 for (i = 0; cheetah_error_table[i].mask; i++) {
1179 if ((bit & cheetah_error_table[i].mask) != 0UL)
1180 return cheetah_error_table[i].name;
1181 }
1182 return "???";
1183 }
1184
1185 static void cheetah_log_errors(struct pt_regs *regs, struct cheetah_err_info *info,
1186 unsigned long afsr, unsigned long afar, int recoverable)
1187 {
1188 unsigned long hipri;
1189 char unum[256];
1190
1191 printk("%s" "ERROR(%d): Cheetah error trap taken afsr[%016lx] afar[%016lx] TL1(%d)\n",
1192 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1193 afsr, afar,
1194 (afsr & CHAFSR_TL1) ? 1 : 0);
1195 printk("%s" "ERROR(%d): TPC[%lx] TNPC[%lx] O7[%lx] TSTATE[%lx]\n",
1196 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1197 regs->tpc, regs->tnpc, regs->u_regs[UREG_I7], regs->tstate);
1198 printk("%s" "ERROR(%d): ",
1199 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id());
1200 printk("TPC<%pS>\n", (void *) regs->tpc);
1201 printk("%s" "ERROR(%d): M_SYND(%lx), E_SYND(%lx)%s%s\n",
1202 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1203 (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT,
1204 (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT,
1205 (afsr & CHAFSR_ME) ? ", Multiple Errors" : "",
1206 (afsr & CHAFSR_PRIV) ? ", Privileged" : "");
1207 hipri = cheetah_get_hipri(afsr);
1208 printk("%s" "ERROR(%d): Highest priority error (%016lx) \"%s\"\n",
1209 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1210 hipri, cheetah_get_string(hipri));
1211
1212 /* Try to get unumber if relevant. */
1213 #define ESYND_ERRORS (CHAFSR_IVC | CHAFSR_IVU | \
1214 CHAFSR_CPC | CHAFSR_CPU | \
1215 CHAFSR_UE | CHAFSR_CE | \
1216 CHAFSR_EDC | CHAFSR_EDU | \
1217 CHAFSR_UCC | CHAFSR_UCU | \
1218 CHAFSR_WDU | CHAFSR_WDC)
1219 #define MSYND_ERRORS (CHAFSR_EMC | CHAFSR_EMU)
1220 if (afsr & ESYND_ERRORS) {
1221 int syndrome;
1222 int ret;
1223
1224 syndrome = (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT;
1225 syndrome = cheetah_ecc_syntab[syndrome];
1226 ret = sprintf_dimm(syndrome, afar, unum, sizeof(unum));
1227 if (ret != -1)
1228 printk("%s" "ERROR(%d): AFAR E-syndrome [%s]\n",
1229 (recoverable ? KERN_WARNING : KERN_CRIT),
1230 smp_processor_id(), unum);
1231 } else if (afsr & MSYND_ERRORS) {
1232 int syndrome;
1233 int ret;
1234
1235 syndrome = (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT;
1236 syndrome = cheetah_mtag_syntab[syndrome];
1237 ret = sprintf_dimm(syndrome, afar, unum, sizeof(unum));
1238 if (ret != -1)
1239 printk("%s" "ERROR(%d): AFAR M-syndrome [%s]\n",
1240 (recoverable ? KERN_WARNING : KERN_CRIT),
1241 smp_processor_id(), unum);
1242 }
1243
1244 /* Now dump the cache snapshots. */
1245 printk("%s" "ERROR(%d): D-cache idx[%x] tag[%016llx] utag[%016llx] stag[%016llx]\n",
1246 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1247 (int) info->dcache_index,
1248 info->dcache_tag,
1249 info->dcache_utag,
1250 info->dcache_stag);
1251 printk("%s" "ERROR(%d): D-cache data0[%016llx] data1[%016llx] data2[%016llx] data3[%016llx]\n",
1252 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1253 info->dcache_data[0],
1254 info->dcache_data[1],
1255 info->dcache_data[2],
1256 info->dcache_data[3]);
1257 printk("%s" "ERROR(%d): I-cache idx[%x] tag[%016llx] utag[%016llx] stag[%016llx] "
1258 "u[%016llx] l[%016llx]\n",
1259 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1260 (int) info->icache_index,
1261 info->icache_tag,
1262 info->icache_utag,
1263 info->icache_stag,
1264 info->icache_upper,
1265 info->icache_lower);
1266 printk("%s" "ERROR(%d): I-cache INSN0[%016llx] INSN1[%016llx] INSN2[%016llx] INSN3[%016llx]\n",
1267 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1268 info->icache_data[0],
1269 info->icache_data[1],
1270 info->icache_data[2],
1271 info->icache_data[3]);
1272 printk("%s" "ERROR(%d): I-cache INSN4[%016llx] INSN5[%016llx] INSN6[%016llx] INSN7[%016llx]\n",
1273 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1274 info->icache_data[4],
1275 info->icache_data[5],
1276 info->icache_data[6],
1277 info->icache_data[7]);
1278 printk("%s" "ERROR(%d): E-cache idx[%x] tag[%016llx]\n",
1279 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1280 (int) info->ecache_index, info->ecache_tag);
1281 printk("%s" "ERROR(%d): E-cache data0[%016llx] data1[%016llx] data2[%016llx] data3[%016llx]\n",
1282 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1283 info->ecache_data[0],
1284 info->ecache_data[1],
1285 info->ecache_data[2],
1286 info->ecache_data[3]);
1287
1288 afsr = (afsr & ~hipri) & cheetah_afsr_errors;
1289 while (afsr != 0UL) {
1290 unsigned long bit = cheetah_get_hipri(afsr);
1291
1292 printk("%s" "ERROR: Multiple-error (%016lx) \"%s\"\n",
1293 (recoverable ? KERN_WARNING : KERN_CRIT),
1294 bit, cheetah_get_string(bit));
1295
1296 afsr &= ~bit;
1297 }
1298
1299 if (!recoverable)
1300 printk(KERN_CRIT "ERROR: This condition is not recoverable.\n");
1301 }
1302
1303 static int cheetah_recheck_errors(struct cheetah_err_info *logp)
1304 {
1305 unsigned long afsr, afar;
1306 int ret = 0;
1307
1308 __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
1309 : "=r" (afsr)
1310 : "i" (ASI_AFSR));
1311 if ((afsr & cheetah_afsr_errors) != 0) {
1312 if (logp != NULL) {
1313 __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
1314 : "=r" (afar)
1315 : "i" (ASI_AFAR));
1316 logp->afsr = afsr;
1317 logp->afar = afar;
1318 }
1319 ret = 1;
1320 }
1321 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
1322 "membar #Sync\n\t"
1323 : : "r" (afsr), "i" (ASI_AFSR));
1324
1325 return ret;
1326 }
1327
1328 void cheetah_fecc_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1329 {
1330 struct cheetah_err_info local_snapshot, *p;
1331 int recoverable;
1332
1333 /* Flush E-cache */
1334 cheetah_flush_ecache();
1335
1336 p = cheetah_get_error_log(afsr);
1337 if (!p) {
1338 prom_printf("ERROR: Early Fast-ECC error afsr[%016lx] afar[%016lx]\n",
1339 afsr, afar);
1340 prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1341 smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1342 prom_halt();
1343 }
1344
1345 /* Grab snapshot of logged error. */
1346 memcpy(&local_snapshot, p, sizeof(local_snapshot));
1347
1348 /* If the current trap snapshot does not match what the
1349 * trap handler passed along into our args, big trouble.
1350 * In such a case, mark the local copy as invalid.
1351 *
1352 * Else, it matches and we mark the afsr in the non-local
1353 * copy as invalid so we may log new error traps there.
1354 */
1355 if (p->afsr != afsr || p->afar != afar)
1356 local_snapshot.afsr = CHAFSR_INVALID;
1357 else
1358 p->afsr = CHAFSR_INVALID;
1359
1360 cheetah_flush_icache();
1361 cheetah_flush_dcache();
1362
1363 /* Re-enable I-cache/D-cache */
1364 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1365 "or %%g1, %1, %%g1\n\t"
1366 "stxa %%g1, [%%g0] %0\n\t"
1367 "membar #Sync"
1368 : /* no outputs */
1369 : "i" (ASI_DCU_CONTROL_REG),
1370 "i" (DCU_DC | DCU_IC)
1371 : "g1");
1372
1373 /* Re-enable error reporting */
1374 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1375 "or %%g1, %1, %%g1\n\t"
1376 "stxa %%g1, [%%g0] %0\n\t"
1377 "membar #Sync"
1378 : /* no outputs */
1379 : "i" (ASI_ESTATE_ERROR_EN),
1380 "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1381 : "g1");
1382
1383 /* Decide if we can continue after handling this trap and
1384 * logging the error.
1385 */
1386 recoverable = 1;
1387 if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1388 recoverable = 0;
1389
1390 /* Re-check AFSR/AFAR. What we are looking for here is whether a new
1391 * error was logged while we had error reporting traps disabled.
1392 */
1393 if (cheetah_recheck_errors(&local_snapshot)) {
1394 unsigned long new_afsr = local_snapshot.afsr;
1395
1396 /* If we got a new asynchronous error, die... */
1397 if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU |
1398 CHAFSR_WDU | CHAFSR_CPU |
1399 CHAFSR_IVU | CHAFSR_UE |
1400 CHAFSR_BERR | CHAFSR_TO))
1401 recoverable = 0;
1402 }
1403
1404 /* Log errors. */
1405 cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1406
1407 if (!recoverable)
1408 panic("Irrecoverable Fast-ECC error trap.\n");
1409
1410 /* Flush E-cache to kick the error trap handlers out. */
1411 cheetah_flush_ecache();
1412 }
1413
1414 /* Try to fix a correctable error by pushing the line out from
1415 * the E-cache. Recheck error reporting registers to see if the
1416 * problem is intermittent.
1417 */
1418 static int cheetah_fix_ce(unsigned long physaddr)
1419 {
1420 unsigned long orig_estate;
1421 unsigned long alias1, alias2;
1422 int ret;
1423
1424 /* Make sure correctable error traps are disabled. */
1425 __asm__ __volatile__("ldxa [%%g0] %2, %0\n\t"
1426 "andn %0, %1, %%g1\n\t"
1427 "stxa %%g1, [%%g0] %2\n\t"
1428 "membar #Sync"
1429 : "=&r" (orig_estate)
1430 : "i" (ESTATE_ERROR_CEEN),
1431 "i" (ASI_ESTATE_ERROR_EN)
1432 : "g1");
1433
1434 /* We calculate alias addresses that will force the
1435 * cache line in question out of the E-cache. Then
1436 * we bring it back in with an atomic instruction so
1437 * that we get it in some modified/exclusive state,
1438 * then we displace it again to try and get proper ECC
1439 * pushed back into the system.
1440 */
1441 physaddr &= ~(8UL - 1UL);
1442 alias1 = (ecache_flush_physbase +
1443 (physaddr & ((ecache_flush_size >> 1) - 1)));
1444 alias2 = alias1 + (ecache_flush_size >> 1);
1445 __asm__ __volatile__("ldxa [%0] %3, %%g0\n\t"
1446 "ldxa [%1] %3, %%g0\n\t"
1447 "casxa [%2] %3, %%g0, %%g0\n\t"
1448 "ldxa [%0] %3, %%g0\n\t"
1449 "ldxa [%1] %3, %%g0\n\t"
1450 "membar #Sync"
1451 : /* no outputs */
1452 : "r" (alias1), "r" (alias2),
1453 "r" (physaddr), "i" (ASI_PHYS_USE_EC));
1454
1455 /* Did that trigger another error? */
1456 if (cheetah_recheck_errors(NULL)) {
1457 /* Try one more time. */
1458 __asm__ __volatile__("ldxa [%0] %1, %%g0\n\t"
1459 "membar #Sync"
1460 : : "r" (physaddr), "i" (ASI_PHYS_USE_EC));
1461 if (cheetah_recheck_errors(NULL))
1462 ret = 2;
1463 else
1464 ret = 1;
1465 } else {
1466 /* No new error, intermittent problem. */
1467 ret = 0;
1468 }
1469
1470 /* Restore error enables. */
1471 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
1472 "membar #Sync"
1473 : : "r" (orig_estate), "i" (ASI_ESTATE_ERROR_EN));
1474
1475 return ret;
1476 }
1477
1478 /* Return non-zero if PADDR is a valid physical memory address. */
1479 static int cheetah_check_main_memory(unsigned long paddr)
1480 {
1481 unsigned long vaddr = PAGE_OFFSET + paddr;
1482
1483 if (vaddr > (unsigned long) high_memory)
1484 return 0;
1485
1486 return kern_addr_valid(vaddr);
1487 }
1488
1489 void cheetah_cee_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1490 {
1491 struct cheetah_err_info local_snapshot, *p;
1492 int recoverable, is_memory;
1493
1494 p = cheetah_get_error_log(afsr);
1495 if (!p) {
1496 prom_printf("ERROR: Early CEE error afsr[%016lx] afar[%016lx]\n",
1497 afsr, afar);
1498 prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1499 smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1500 prom_halt();
1501 }
1502
1503 /* Grab snapshot of logged error. */
1504 memcpy(&local_snapshot, p, sizeof(local_snapshot));
1505
1506 /* If the current trap snapshot does not match what the
1507 * trap handler passed along into our args, big trouble.
1508 * In such a case, mark the local copy as invalid.
1509 *
1510 * Else, it matches and we mark the afsr in the non-local
1511 * copy as invalid so we may log new error traps there.
1512 */
1513 if (p->afsr != afsr || p->afar != afar)
1514 local_snapshot.afsr = CHAFSR_INVALID;
1515 else
1516 p->afsr = CHAFSR_INVALID;
1517
1518 is_memory = cheetah_check_main_memory(afar);
1519
1520 if (is_memory && (afsr & CHAFSR_CE) != 0UL) {
1521 /* XXX Might want to log the results of this operation
1522 * XXX somewhere... -DaveM
1523 */
1524 cheetah_fix_ce(afar);
1525 }
1526
1527 {
1528 int flush_all, flush_line;
1529
1530 flush_all = flush_line = 0;
1531 if ((afsr & CHAFSR_EDC) != 0UL) {
1532 if ((afsr & cheetah_afsr_errors) == CHAFSR_EDC)
1533 flush_line = 1;
1534 else
1535 flush_all = 1;
1536 } else if ((afsr & CHAFSR_CPC) != 0UL) {
1537 if ((afsr & cheetah_afsr_errors) == CHAFSR_CPC)
1538 flush_line = 1;
1539 else
1540 flush_all = 1;
1541 }
1542
1543 /* Trap handler only disabled I-cache, flush it. */
1544 cheetah_flush_icache();
1545
1546 /* Re-enable I-cache */
1547 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1548 "or %%g1, %1, %%g1\n\t"
1549 "stxa %%g1, [%%g0] %0\n\t"
1550 "membar #Sync"
1551 : /* no outputs */
1552 : "i" (ASI_DCU_CONTROL_REG),
1553 "i" (DCU_IC)
1554 : "g1");
1555
1556 if (flush_all)
1557 cheetah_flush_ecache();
1558 else if (flush_line)
1559 cheetah_flush_ecache_line(afar);
1560 }
1561
1562 /* Re-enable error reporting */
1563 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1564 "or %%g1, %1, %%g1\n\t"
1565 "stxa %%g1, [%%g0] %0\n\t"
1566 "membar #Sync"
1567 : /* no outputs */
1568 : "i" (ASI_ESTATE_ERROR_EN),
1569 "i" (ESTATE_ERROR_CEEN)
1570 : "g1");
1571
1572 /* Decide if we can continue after handling this trap and
1573 * logging the error.
1574 */
1575 recoverable = 1;
1576 if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1577 recoverable = 0;
1578
1579 /* Re-check AFSR/AFAR */
1580 (void) cheetah_recheck_errors(&local_snapshot);
1581
1582 /* Log errors. */
1583 cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1584
1585 if (!recoverable)
1586 panic("Irrecoverable Correctable-ECC error trap.\n");
1587 }
1588
1589 void cheetah_deferred_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1590 {
1591 struct cheetah_err_info local_snapshot, *p;
1592 int recoverable, is_memory;
1593
1594 #ifdef CONFIG_PCI
1595 /* Check for the special PCI poke sequence. */
1596 if (pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) {
1597 cheetah_flush_icache();
1598 cheetah_flush_dcache();
1599
1600 /* Re-enable I-cache/D-cache */
1601 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1602 "or %%g1, %1, %%g1\n\t"
1603 "stxa %%g1, [%%g0] %0\n\t"
1604 "membar #Sync"
1605 : /* no outputs */
1606 : "i" (ASI_DCU_CONTROL_REG),
1607 "i" (DCU_DC | DCU_IC)
1608 : "g1");
1609
1610 /* Re-enable error reporting */
1611 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1612 "or %%g1, %1, %%g1\n\t"
1613 "stxa %%g1, [%%g0] %0\n\t"
1614 "membar #Sync"
1615 : /* no outputs */
1616 : "i" (ASI_ESTATE_ERROR_EN),
1617 "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1618 : "g1");
1619
1620 (void) cheetah_recheck_errors(NULL);
1621
1622 pci_poke_faulted = 1;
1623 regs->tpc += 4;
1624 regs->tnpc = regs->tpc + 4;
1625 return;
1626 }
1627 #endif
1628
1629 p = cheetah_get_error_log(afsr);
1630 if (!p) {
1631 prom_printf("ERROR: Early deferred error afsr[%016lx] afar[%016lx]\n",
1632 afsr, afar);
1633 prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1634 smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1635 prom_halt();
1636 }
1637
1638 /* Grab snapshot of logged error. */
1639 memcpy(&local_snapshot, p, sizeof(local_snapshot));
1640
1641 /* If the current trap snapshot does not match what the
1642 * trap handler passed along into our args, big trouble.
1643 * In such a case, mark the local copy as invalid.
1644 *
1645 * Else, it matches and we mark the afsr in the non-local
1646 * copy as invalid so we may log new error traps there.
1647 */
1648 if (p->afsr != afsr || p->afar != afar)
1649 local_snapshot.afsr = CHAFSR_INVALID;
1650 else
1651 p->afsr = CHAFSR_INVALID;
1652
1653 is_memory = cheetah_check_main_memory(afar);
1654
1655 {
1656 int flush_all, flush_line;
1657
1658 flush_all = flush_line = 0;
1659 if ((afsr & CHAFSR_EDU) != 0UL) {
1660 if ((afsr & cheetah_afsr_errors) == CHAFSR_EDU)
1661 flush_line = 1;
1662 else
1663 flush_all = 1;
1664 } else if ((afsr & CHAFSR_BERR) != 0UL) {
1665 if ((afsr & cheetah_afsr_errors) == CHAFSR_BERR)
1666 flush_line = 1;
1667 else
1668 flush_all = 1;
1669 }
1670
1671 cheetah_flush_icache();
1672 cheetah_flush_dcache();
1673
1674 /* Re-enable I/D caches */
1675 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1676 "or %%g1, %1, %%g1\n\t"
1677 "stxa %%g1, [%%g0] %0\n\t"
1678 "membar #Sync"
1679 : /* no outputs */
1680 : "i" (ASI_DCU_CONTROL_REG),
1681 "i" (DCU_IC | DCU_DC)
1682 : "g1");
1683
1684 if (flush_all)
1685 cheetah_flush_ecache();
1686 else if (flush_line)
1687 cheetah_flush_ecache_line(afar);
1688 }
1689
1690 /* Re-enable error reporting */
1691 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1692 "or %%g1, %1, %%g1\n\t"
1693 "stxa %%g1, [%%g0] %0\n\t"
1694 "membar #Sync"
1695 : /* no outputs */
1696 : "i" (ASI_ESTATE_ERROR_EN),
1697 "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1698 : "g1");
1699
1700 /* Decide if we can continue after handling this trap and
1701 * logging the error.
1702 */
1703 recoverable = 1;
1704 if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1705 recoverable = 0;
1706
1707 /* Re-check AFSR/AFAR. What we are looking for here is whether a new
1708 * error was logged while we had error reporting traps disabled.
1709 */
1710 if (cheetah_recheck_errors(&local_snapshot)) {
1711 unsigned long new_afsr = local_snapshot.afsr;
1712
1713 /* If we got a new asynchronous error, die... */
1714 if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU |
1715 CHAFSR_WDU | CHAFSR_CPU |
1716 CHAFSR_IVU | CHAFSR_UE |
1717 CHAFSR_BERR | CHAFSR_TO))
1718 recoverable = 0;
1719 }
1720
1721 /* Log errors. */
1722 cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1723
1724 /* "Recoverable" here means we try to yank the page from ever
1725 * being newly used again. This depends upon a few things:
1726 * 1) Must be main memory, and AFAR must be valid.
1727 * 2) If we trapped from user, OK.
1728 * 3) Else, if we trapped from kernel we must find exception
1729 * table entry (ie. we have to have been accessing user
1730 * space).
1731 *
1732 * If AFAR is not in main memory, or we trapped from kernel
1733 * and cannot find an exception table entry, it is unacceptable
1734 * to try and continue.
1735 */
1736 if (recoverable && is_memory) {
1737 if ((regs->tstate & TSTATE_PRIV) == 0UL) {
1738 /* OK, usermode access. */
1739 recoverable = 1;
1740 } else {
1741 const struct exception_table_entry *entry;
1742
1743 entry = search_exception_tables(regs->tpc);
1744 if (entry) {
1745 /* OK, kernel access to userspace. */
1746 recoverable = 1;
1747
1748 } else {
1749 /* BAD, privileged state is corrupted. */
1750 recoverable = 0;
1751 }
1752
1753 if (recoverable) {
1754 if (pfn_valid(afar >> PAGE_SHIFT))
1755 get_page(pfn_to_page(afar >> PAGE_SHIFT));
1756 else
1757 recoverable = 0;
1758
1759 /* Only perform fixup if we still have a
1760 * recoverable condition.
1761 */
1762 if (recoverable) {
1763 regs->tpc = entry->fixup;
1764 regs->tnpc = regs->tpc + 4;
1765 }
1766 }
1767 }
1768 } else {
1769 recoverable = 0;
1770 }
1771
1772 if (!recoverable)
1773 panic("Irrecoverable deferred error trap.\n");
1774 }
1775
1776 /* Handle a D/I cache parity error trap. TYPE is encoded as:
1777 *
1778 * Bit0: 0=dcache,1=icache
1779 * Bit1: 0=recoverable,1=unrecoverable
1780 *
1781 * The hardware has disabled both the I-cache and D-cache in
1782 * the %dcr register.
1783 */
1784 void cheetah_plus_parity_error(int type, struct pt_regs *regs)
1785 {
1786 if (type & 0x1)
1787 __cheetah_flush_icache();
1788 else
1789 cheetah_plus_zap_dcache_parity();
1790 cheetah_flush_dcache();
1791
1792 /* Re-enable I-cache/D-cache */
1793 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1794 "or %%g1, %1, %%g1\n\t"
1795 "stxa %%g1, [%%g0] %0\n\t"
1796 "membar #Sync"
1797 : /* no outputs */
1798 : "i" (ASI_DCU_CONTROL_REG),
1799 "i" (DCU_DC | DCU_IC)
1800 : "g1");
1801
1802 if (type & 0x2) {
1803 printk(KERN_EMERG "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
1804 smp_processor_id(),
1805 (type & 0x1) ? 'I' : 'D',
1806 regs->tpc);
1807 printk(KERN_EMERG "TPC<%pS>\n", (void *) regs->tpc);
1808 panic("Irrecoverable Cheetah+ parity error.");
1809 }
1810
1811 printk(KERN_WARNING "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
1812 smp_processor_id(),
1813 (type & 0x1) ? 'I' : 'D',
1814 regs->tpc);
1815 printk(KERN_WARNING "TPC<%pS>\n", (void *) regs->tpc);
1816 }
1817
1818 struct sun4v_error_entry {
1819 /* Unique error handle */
1820 /*0x00*/u64 err_handle;
1821
1822 /* %stick value at the time of the error */
1823 /*0x08*/u64 err_stick;
1824
1825 /*0x10*/u8 reserved_1[3];
1826
1827 /* Error type */
1828 /*0x13*/u8 err_type;
1829 #define SUN4V_ERR_TYPE_UNDEFINED 0
1830 #define SUN4V_ERR_TYPE_UNCORRECTED_RES 1
1831 #define SUN4V_ERR_TYPE_PRECISE_NONRES 2
1832 #define SUN4V_ERR_TYPE_DEFERRED_NONRES 3
1833 #define SUN4V_ERR_TYPE_SHUTDOWN_RQST 4
1834 #define SUN4V_ERR_TYPE_DUMP_CORE 5
1835 #define SUN4V_ERR_TYPE_SP_STATE_CHANGE 6
1836 #define SUN4V_ERR_TYPE_NUM 7
1837
1838 /* Error attributes */
1839 /*0x14*/u32 err_attrs;
1840 #define SUN4V_ERR_ATTRS_PROCESSOR 0x00000001
1841 #define SUN4V_ERR_ATTRS_MEMORY 0x00000002
1842 #define SUN4V_ERR_ATTRS_PIO 0x00000004
1843 #define SUN4V_ERR_ATTRS_INT_REGISTERS 0x00000008
1844 #define SUN4V_ERR_ATTRS_FPU_REGISTERS 0x00000010
1845 #define SUN4V_ERR_ATTRS_SHUTDOWN_RQST 0x00000020
1846 #define SUN4V_ERR_ATTRS_ASR 0x00000040
1847 #define SUN4V_ERR_ATTRS_ASI 0x00000080
1848 #define SUN4V_ERR_ATTRS_PRIV_REG 0x00000100
1849 #define SUN4V_ERR_ATTRS_SPSTATE_MSK 0x00000600
1850 #define SUN4V_ERR_ATTRS_SPSTATE_SHFT 9
1851 #define SUN4V_ERR_ATTRS_MODE_MSK 0x03000000
1852 #define SUN4V_ERR_ATTRS_MODE_SHFT 24
1853 #define SUN4V_ERR_ATTRS_RES_QUEUE_FULL 0x80000000
1854
1855 #define SUN4V_ERR_SPSTATE_FAULTED 0
1856 #define SUN4V_ERR_SPSTATE_AVAILABLE 1
1857 #define SUN4V_ERR_SPSTATE_NOT_PRESENT 2
1858
1859 #define SUN4V_ERR_MODE_USER 1
1860 #define SUN4V_ERR_MODE_PRIV 2
1861
1862 /* Real address of the memory region or PIO transaction */
1863 /*0x18*/u64 err_raddr;
1864
1865 /* Size of the operation triggering the error, in bytes */
1866 /*0x20*/u32 err_size;
1867
1868 /* ID of the CPU */
1869 /*0x24*/u16 err_cpu;
1870
1871 /* Grace periof for shutdown, in seconds */
1872 /*0x26*/u16 err_secs;
1873
1874 /* Value of the %asi register */
1875 /*0x28*/u8 err_asi;
1876
1877 /*0x29*/u8 reserved_2;
1878
1879 /* Value of the ASR register number */
1880 /*0x2a*/u16 err_asr;
1881 #define SUN4V_ERR_ASR_VALID 0x8000
1882
1883 /*0x2c*/u32 reserved_3;
1884 /*0x30*/u64 reserved_4;
1885 /*0x38*/u64 reserved_5;
1886 };
1887
1888 static atomic_t sun4v_resum_oflow_cnt = ATOMIC_INIT(0);
1889 static atomic_t sun4v_nonresum_oflow_cnt = ATOMIC_INIT(0);
1890
1891 static const char *sun4v_err_type_to_str(u8 type)
1892 {
1893 static const char *types[SUN4V_ERR_TYPE_NUM] = {
1894 "undefined",
1895 "uncorrected resumable",
1896 "precise nonresumable",
1897 "deferred nonresumable",
1898 "shutdown request",
1899 "dump core",
1900 "SP state change",
1901 };
1902
1903 if (type < SUN4V_ERR_TYPE_NUM)
1904 return types[type];
1905
1906 return "unknown";
1907 }
1908
1909 static void sun4v_emit_err_attr_strings(u32 attrs)
1910 {
1911 static const char *attr_names[] = {
1912 "processor",
1913 "memory",
1914 "PIO",
1915 "int-registers",
1916 "fpu-registers",
1917 "shutdown-request",
1918 "ASR",
1919 "ASI",
1920 "priv-reg",
1921 };
1922 static const char *sp_states[] = {
1923 "sp-faulted",
1924 "sp-available",
1925 "sp-not-present",
1926 "sp-state-reserved",
1927 };
1928 static const char *modes[] = {
1929 "mode-reserved0",
1930 "user",
1931 "priv",
1932 "mode-reserved1",
1933 };
1934 u32 sp_state, mode;
1935 int i;
1936
1937 for (i = 0; i < ARRAY_SIZE(attr_names); i++) {
1938 if (attrs & (1U << i)) {
1939 const char *s = attr_names[i];
1940
1941 pr_cont("%s ", s);
1942 }
1943 }
1944
1945 sp_state = ((attrs & SUN4V_ERR_ATTRS_SPSTATE_MSK) >>
1946 SUN4V_ERR_ATTRS_SPSTATE_SHFT);
1947 pr_cont("%s ", sp_states[sp_state]);
1948
1949 mode = ((attrs & SUN4V_ERR_ATTRS_MODE_MSK) >>
1950 SUN4V_ERR_ATTRS_MODE_SHFT);
1951 pr_cont("%s ", modes[mode]);
1952
1953 if (attrs & SUN4V_ERR_ATTRS_RES_QUEUE_FULL)
1954 pr_cont("res-queue-full ");
1955 }
1956
1957 /* When the report contains a real-address of "-1" it means that the
1958 * hardware did not provide the address. So we compute the effective
1959 * address of the load or store instruction at regs->tpc and report
1960 * that. Usually when this happens it's a PIO and in such a case we
1961 * are using physical addresses with bypass ASIs anyways, so what we
1962 * report here is exactly what we want.
1963 */
1964 static void sun4v_report_real_raddr(const char *pfx, struct pt_regs *regs)
1965 {
1966 unsigned int insn;
1967 u64 addr;
1968
1969 if (!(regs->tstate & TSTATE_PRIV))
1970 return;
1971
1972 insn = *(unsigned int *) regs->tpc;
1973
1974 addr = compute_effective_address(regs, insn, 0);
1975
1976 printk("%s: insn effective address [0x%016llx]\n",
1977 pfx, addr);
1978 }
1979
1980 static void sun4v_log_error(struct pt_regs *regs, struct sun4v_error_entry *ent,
1981 int cpu, const char *pfx, atomic_t *ocnt)
1982 {
1983 u64 *raw_ptr = (u64 *) ent;
1984 u32 attrs;
1985 int cnt;
1986
1987 printk("%s: Reporting on cpu %d\n", pfx, cpu);
1988 printk("%s: TPC [0x%016lx] <%pS>\n",
1989 pfx, regs->tpc, (void *) regs->tpc);
1990
1991 printk("%s: RAW [%016llx:%016llx:%016llx:%016llx\n",
1992 pfx, raw_ptr[0], raw_ptr[1], raw_ptr[2], raw_ptr[3]);
1993 printk("%s: %016llx:%016llx:%016llx:%016llx]\n",
1994 pfx, raw_ptr[4], raw_ptr[5], raw_ptr[6], raw_ptr[7]);
1995
1996 printk("%s: handle [0x%016llx] stick [0x%016llx]\n",
1997 pfx, ent->err_handle, ent->err_stick);
1998
1999 printk("%s: type [%s]\n", pfx, sun4v_err_type_to_str(ent->err_type));
2000
2001 attrs = ent->err_attrs;
2002 printk("%s: attrs [0x%08x] < ", pfx, attrs);
2003 sun4v_emit_err_attr_strings(attrs);
2004 pr_cont(">\n");
2005
2006 /* Various fields in the error report are only valid if
2007 * certain attribute bits are set.
2008 */
2009 if (attrs & (SUN4V_ERR_ATTRS_MEMORY |
2010 SUN4V_ERR_ATTRS_PIO |
2011 SUN4V_ERR_ATTRS_ASI)) {
2012 printk("%s: raddr [0x%016llx]\n", pfx, ent->err_raddr);
2013
2014 if (ent->err_raddr == ~(u64)0)
2015 sun4v_report_real_raddr(pfx, regs);
2016 }
2017
2018 if (attrs & (SUN4V_ERR_ATTRS_MEMORY | SUN4V_ERR_ATTRS_ASI))
2019 printk("%s: size [0x%x]\n", pfx, ent->err_size);
2020
2021 if (attrs & (SUN4V_ERR_ATTRS_PROCESSOR |
2022 SUN4V_ERR_ATTRS_INT_REGISTERS |
2023 SUN4V_ERR_ATTRS_FPU_REGISTERS |
2024 SUN4V_ERR_ATTRS_PRIV_REG))
2025 printk("%s: cpu[%u]\n", pfx, ent->err_cpu);
2026
2027 if (attrs & SUN4V_ERR_ATTRS_ASI)
2028 printk("%s: asi [0x%02x]\n", pfx, ent->err_asi);
2029
2030 if ((attrs & (SUN4V_ERR_ATTRS_INT_REGISTERS |
2031 SUN4V_ERR_ATTRS_FPU_REGISTERS |
2032 SUN4V_ERR_ATTRS_PRIV_REG)) &&
2033 (ent->err_asr & SUN4V_ERR_ASR_VALID) != 0)
2034 printk("%s: reg [0x%04x]\n",
2035 pfx, ent->err_asr & ~SUN4V_ERR_ASR_VALID);
2036
2037 show_regs(regs);
2038
2039 if ((cnt = atomic_read(ocnt)) != 0) {
2040 atomic_set(ocnt, 0);
2041 wmb();
2042 printk("%s: Queue overflowed %d times.\n",
2043 pfx, cnt);
2044 }
2045 }
2046
2047 /* We run with %pil set to PIL_NORMAL_MAX and PSTATE_IE enabled in %pstate.
2048 * Log the event and clear the first word of the entry.
2049 */
2050 void sun4v_resum_error(struct pt_regs *regs, unsigned long offset)
2051 {
2052 enum ctx_state prev_state = exception_enter();
2053 struct sun4v_error_entry *ent, local_copy;
2054 struct trap_per_cpu *tb;
2055 unsigned long paddr;
2056 int cpu;
2057
2058 cpu = get_cpu();
2059
2060 tb = &trap_block[cpu];
2061 paddr = tb->resum_kernel_buf_pa + offset;
2062 ent = __va(paddr);
2063
2064 memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry));
2065
2066 /* We have a local copy now, so release the entry. */
2067 ent->err_handle = 0;
2068 wmb();
2069
2070 put_cpu();
2071
2072 if (local_copy.err_type == SUN4V_ERR_TYPE_SHUTDOWN_RQST) {
2073 /* We should really take the seconds field of
2074 * the error report and use it for the shutdown
2075 * invocation, but for now do the same thing we
2076 * do for a DS shutdown request.
2077 */
2078 pr_info("Shutdown request, %u seconds...\n",
2079 local_copy.err_secs);
2080 orderly_poweroff(true);
2081 goto out;
2082 }
2083
2084 sun4v_log_error(regs, &local_copy, cpu,
2085 KERN_ERR "RESUMABLE ERROR",
2086 &sun4v_resum_oflow_cnt);
2087 out:
2088 exception_exit(prev_state);
2089 }
2090
2091 /* If we try to printk() we'll probably make matters worse, by trying
2092 * to retake locks this cpu already holds or causing more errors. So
2093 * just bump a counter, and we'll report these counter bumps above.
2094 */
2095 void sun4v_resum_overflow(struct pt_regs *regs)
2096 {
2097 atomic_inc(&sun4v_resum_oflow_cnt);
2098 }
2099
2100 /* Given a set of registers, get the virtual addressi that was being accessed
2101 * by the faulting instructions at tpc.
2102 */
2103 static unsigned long sun4v_get_vaddr(struct pt_regs *regs)
2104 {
2105 unsigned int insn;
2106
2107 if (!copy_from_user(&insn, (void __user *)regs->tpc, 4)) {
2108 return compute_effective_address(regs, insn,
2109 (insn >> 25) & 0x1f);
2110 }
2111 return 0;
2112 }
2113
2114 /* Attempt to handle non-resumable errors generated from userspace.
2115 * Returns true if the signal was handled, false otherwise.
2116 */
2117 bool sun4v_nonresum_error_user_handled(struct pt_regs *regs,
2118 struct sun4v_error_entry *ent) {
2119
2120 unsigned int attrs = ent->err_attrs;
2121
2122 if (attrs & SUN4V_ERR_ATTRS_MEMORY) {
2123 unsigned long addr = ent->err_raddr;
2124 siginfo_t info;
2125
2126 if (addr == ~(u64)0) {
2127 /* This seems highly unlikely to ever occur */
2128 pr_emerg("SUN4V NON-RECOVERABLE ERROR: Memory error detected in unknown location!\n");
2129 } else {
2130 unsigned long page_cnt = DIV_ROUND_UP(ent->err_size,
2131 PAGE_SIZE);
2132
2133 /* Break the unfortunate news. */
2134 pr_emerg("SUN4V NON-RECOVERABLE ERROR: Memory failed at %016lX\n",
2135 addr);
2136 pr_emerg("SUN4V NON-RECOVERABLE ERROR: Claiming %lu ages.\n",
2137 page_cnt);
2138
2139 while (page_cnt-- > 0) {
2140 if (pfn_valid(addr >> PAGE_SHIFT))
2141 get_page(pfn_to_page(addr >> PAGE_SHIFT));
2142 addr += PAGE_SIZE;
2143 }
2144 }
2145 info.si_signo = SIGKILL;
2146 info.si_errno = 0;
2147 info.si_trapno = 0;
2148 force_sig_info(info.si_signo, &info, current);
2149
2150 return true;
2151 }
2152 if (attrs & SUN4V_ERR_ATTRS_PIO) {
2153 siginfo_t info;
2154
2155 info.si_signo = SIGBUS;
2156 info.si_code = BUS_ADRERR;
2157 info.si_addr = (void __user *)sun4v_get_vaddr(regs);
2158 force_sig_info(info.si_signo, &info, current);
2159
2160 return true;
2161 }
2162
2163 /* Default to doing nothing */
2164 return false;
2165 }
2166
2167 /* We run with %pil set to PIL_NORMAL_MAX and PSTATE_IE enabled in %pstate.
2168 * Log the event, clear the first word of the entry, and die.
2169 */
2170 void sun4v_nonresum_error(struct pt_regs *regs, unsigned long offset)
2171 {
2172 struct sun4v_error_entry *ent, local_copy;
2173 struct trap_per_cpu *tb;
2174 unsigned long paddr;
2175 int cpu;
2176
2177 cpu = get_cpu();
2178
2179 tb = &trap_block[cpu];
2180 paddr = tb->nonresum_kernel_buf_pa + offset;
2181 ent = __va(paddr);
2182
2183 memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry));
2184
2185 /* We have a local copy now, so release the entry. */
2186 ent->err_handle = 0;
2187 wmb();
2188
2189 put_cpu();
2190
2191 if (!(regs->tstate & TSTATE_PRIV) &&
2192 sun4v_nonresum_error_user_handled(regs, &local_copy)) {
2193 /* DON'T PANIC: This userspace error was handled. */
2194 return;
2195 }
2196
2197 #ifdef CONFIG_PCI
2198 /* Check for the special PCI poke sequence. */
2199 if (pci_poke_in_progress && pci_poke_cpu == cpu) {
2200 pci_poke_faulted = 1;
2201 regs->tpc += 4;
2202 regs->tnpc = regs->tpc + 4;
2203 return;
2204 }
2205 #endif
2206
2207 sun4v_log_error(regs, &local_copy, cpu,
2208 KERN_EMERG "NON-RESUMABLE ERROR",
2209 &sun4v_nonresum_oflow_cnt);
2210
2211 panic("Non-resumable error.");
2212 }
2213
2214 /* If we try to printk() we'll probably make matters worse, by trying
2215 * to retake locks this cpu already holds or causing more errors. So
2216 * just bump a counter, and we'll report these counter bumps above.
2217 */
2218 void sun4v_nonresum_overflow(struct pt_regs *regs)
2219 {
2220 /* XXX Actually even this can make not that much sense. Perhaps
2221 * XXX we should just pull the plug and panic directly from here?
2222 */
2223 atomic_inc(&sun4v_nonresum_oflow_cnt);
2224 }
2225
2226 static void sun4v_tlb_error(struct pt_regs *regs)
2227 {
2228 die_if_kernel("TLB/TSB error", regs);
2229 }
2230
2231 unsigned long sun4v_err_itlb_vaddr;
2232 unsigned long sun4v_err_itlb_ctx;
2233 unsigned long sun4v_err_itlb_pte;
2234 unsigned long sun4v_err_itlb_error;
2235
2236 void sun4v_itlb_error_report(struct pt_regs *regs, int tl)
2237 {
2238 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2239
2240 printk(KERN_EMERG "SUN4V-ITLB: Error at TPC[%lx], tl %d\n",
2241 regs->tpc, tl);
2242 printk(KERN_EMERG "SUN4V-ITLB: TPC<%pS>\n", (void *) regs->tpc);
2243 printk(KERN_EMERG "SUN4V-ITLB: O7[%lx]\n", regs->u_regs[UREG_I7]);
2244 printk(KERN_EMERG "SUN4V-ITLB: O7<%pS>\n",
2245 (void *) regs->u_regs[UREG_I7]);
2246 printk(KERN_EMERG "SUN4V-ITLB: vaddr[%lx] ctx[%lx] "
2247 "pte[%lx] error[%lx]\n",
2248 sun4v_err_itlb_vaddr, sun4v_err_itlb_ctx,
2249 sun4v_err_itlb_pte, sun4v_err_itlb_error);
2250
2251 sun4v_tlb_error(regs);
2252 }
2253
2254 unsigned long sun4v_err_dtlb_vaddr;
2255 unsigned long sun4v_err_dtlb_ctx;
2256 unsigned long sun4v_err_dtlb_pte;
2257 unsigned long sun4v_err_dtlb_error;
2258
2259 void sun4v_dtlb_error_report(struct pt_regs *regs, int tl)
2260 {
2261 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2262
2263 printk(KERN_EMERG "SUN4V-DTLB: Error at TPC[%lx], tl %d\n",
2264 regs->tpc, tl);
2265 printk(KERN_EMERG "SUN4V-DTLB: TPC<%pS>\n", (void *) regs->tpc);
2266 printk(KERN_EMERG "SUN4V-DTLB: O7[%lx]\n", regs->u_regs[UREG_I7]);
2267 printk(KERN_EMERG "SUN4V-DTLB: O7<%pS>\n",
2268 (void *) regs->u_regs[UREG_I7]);
2269 printk(KERN_EMERG "SUN4V-DTLB: vaddr[%lx] ctx[%lx] "
2270 "pte[%lx] error[%lx]\n",
2271 sun4v_err_dtlb_vaddr, sun4v_err_dtlb_ctx,
2272 sun4v_err_dtlb_pte, sun4v_err_dtlb_error);
2273
2274 sun4v_tlb_error(regs);
2275 }
2276
2277 void hypervisor_tlbop_error(unsigned long err, unsigned long op)
2278 {
2279 printk(KERN_CRIT "SUN4V: TLB hv call error %lu for op %lu\n",
2280 err, op);
2281 }
2282
2283 void hypervisor_tlbop_error_xcall(unsigned long err, unsigned long op)
2284 {
2285 printk(KERN_CRIT "SUN4V: XCALL TLB hv call error %lu for op %lu\n",
2286 err, op);
2287 }
2288
2289 static void do_fpe_common(struct pt_regs *regs)
2290 {
2291 if (regs->tstate & TSTATE_PRIV) {
2292 regs->tpc = regs->tnpc;
2293 regs->tnpc += 4;
2294 } else {
2295 unsigned long fsr = current_thread_info()->xfsr[0];
2296 siginfo_t info;
2297
2298 if (test_thread_flag(TIF_32BIT)) {
2299 regs->tpc &= 0xffffffff;
2300 regs->tnpc &= 0xffffffff;
2301 }
2302 info.si_signo = SIGFPE;
2303 info.si_errno = 0;
2304 info.si_addr = (void __user *)regs->tpc;
2305 info.si_trapno = 0;
2306 info.si_code = FPE_FIXME;
2307 if ((fsr & 0x1c000) == (1 << 14)) {
2308 if (fsr & 0x10)
2309 info.si_code = FPE_FLTINV;
2310 else if (fsr & 0x08)
2311 info.si_code = FPE_FLTOVF;
2312 else if (fsr & 0x04)
2313 info.si_code = FPE_FLTUND;
2314 else if (fsr & 0x02)
2315 info.si_code = FPE_FLTDIV;
2316 else if (fsr & 0x01)
2317 info.si_code = FPE_FLTRES;
2318 }
2319 force_sig_info(SIGFPE, &info, current);
2320 }
2321 }
2322
2323 void do_fpieee(struct pt_regs *regs)
2324 {
2325 enum ctx_state prev_state = exception_enter();
2326
2327 if (notify_die(DIE_TRAP, "fpu exception ieee", regs,
2328 0, 0x24, SIGFPE) == NOTIFY_STOP)
2329 goto out;
2330
2331 do_fpe_common(regs);
2332 out:
2333 exception_exit(prev_state);
2334 }
2335
2336 void do_fpother(struct pt_regs *regs)
2337 {
2338 enum ctx_state prev_state = exception_enter();
2339 struct fpustate *f = FPUSTATE;
2340 int ret = 0;
2341
2342 if (notify_die(DIE_TRAP, "fpu exception other", regs,
2343 0, 0x25, SIGFPE) == NOTIFY_STOP)
2344 goto out;
2345
2346 switch ((current_thread_info()->xfsr[0] & 0x1c000)) {
2347 case (2 << 14): /* unfinished_FPop */
2348 case (3 << 14): /* unimplemented_FPop */
2349 ret = do_mathemu(regs, f, false);
2350 break;
2351 }
2352 if (ret)
2353 goto out;
2354 do_fpe_common(regs);
2355 out:
2356 exception_exit(prev_state);
2357 }
2358
2359 void do_tof(struct pt_regs *regs)
2360 {
2361 enum ctx_state prev_state = exception_enter();
2362 siginfo_t info;
2363
2364 if (notify_die(DIE_TRAP, "tagged arithmetic overflow", regs,
2365 0, 0x26, SIGEMT) == NOTIFY_STOP)
2366 goto out;
2367
2368 if (regs->tstate & TSTATE_PRIV)
2369 die_if_kernel("Penguin overflow trap from kernel mode", regs);
2370 if (test_thread_flag(TIF_32BIT)) {
2371 regs->tpc &= 0xffffffff;
2372 regs->tnpc &= 0xffffffff;
2373 }
2374 info.si_signo = SIGEMT;
2375 info.si_errno = 0;
2376 info.si_code = EMT_TAGOVF;
2377 info.si_addr = (void __user *)regs->tpc;
2378 info.si_trapno = 0;
2379 force_sig_info(SIGEMT, &info, current);
2380 out:
2381 exception_exit(prev_state);
2382 }
2383
2384 void do_div0(struct pt_regs *regs)
2385 {
2386 enum ctx_state prev_state = exception_enter();
2387 siginfo_t info;
2388
2389 if (notify_die(DIE_TRAP, "integer division by zero", regs,
2390 0, 0x28, SIGFPE) == NOTIFY_STOP)
2391 goto out;
2392
2393 if (regs->tstate & TSTATE_PRIV)
2394 die_if_kernel("TL0: Kernel divide by zero.", regs);
2395 if (test_thread_flag(TIF_32BIT)) {
2396 regs->tpc &= 0xffffffff;
2397 regs->tnpc &= 0xffffffff;
2398 }
2399 info.si_signo = SIGFPE;
2400 info.si_errno = 0;
2401 info.si_code = FPE_INTDIV;
2402 info.si_addr = (void __user *)regs->tpc;
2403 info.si_trapno = 0;
2404 force_sig_info(SIGFPE, &info, current);
2405 out:
2406 exception_exit(prev_state);
2407 }
2408
2409 static void instruction_dump(unsigned int *pc)
2410 {
2411 int i;
2412
2413 if ((((unsigned long) pc) & 3))
2414 return;
2415
2416 printk("Instruction DUMP:");
2417 for (i = -3; i < 6; i++)
2418 printk("%c%08x%c",i?' ':'<',pc[i],i?' ':'>');
2419 printk("\n");
2420 }
2421
2422 static void user_instruction_dump(unsigned int __user *pc)
2423 {
2424 int i;
2425 unsigned int buf[9];
2426
2427 if ((((unsigned long) pc) & 3))
2428 return;
2429
2430 if (copy_from_user(buf, pc - 3, sizeof(buf)))
2431 return;
2432
2433 printk("Instruction DUMP:");
2434 for (i = 0; i < 9; i++)
2435 printk("%c%08x%c",i==3?' ':'<',buf[i],i==3?' ':'>');
2436 printk("\n");
2437 }
2438
2439 void show_stack(struct task_struct *tsk, unsigned long *_ksp)
2440 {
2441 unsigned long fp, ksp;
2442 struct thread_info *tp;
2443 int count = 0;
2444 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2445 int graph = 0;
2446 #endif
2447
2448 ksp = (unsigned long) _ksp;
2449 if (!tsk)
2450 tsk = current;
2451 tp = task_thread_info(tsk);
2452 if (ksp == 0UL) {
2453 if (tsk == current)
2454 asm("mov %%fp, %0" : "=r" (ksp));
2455 else
2456 ksp = tp->ksp;
2457 }
2458 if (tp == current_thread_info())
2459 flushw_all();
2460
2461 fp = ksp + STACK_BIAS;
2462
2463 printk("Call Trace:\n");
2464 do {
2465 struct sparc_stackf *sf;
2466 struct pt_regs *regs;
2467 unsigned long pc;
2468
2469 if (!kstack_valid(tp, fp))
2470 break;
2471 sf = (struct sparc_stackf *) fp;
2472 regs = (struct pt_regs *) (sf + 1);
2473
2474 if (kstack_is_trap_frame(tp, regs)) {
2475 if (!(regs->tstate & TSTATE_PRIV))
2476 break;
2477 pc = regs->tpc;
2478 fp = regs->u_regs[UREG_I6] + STACK_BIAS;
2479 } else {
2480 pc = sf->callers_pc;
2481 fp = (unsigned long)sf->fp + STACK_BIAS;
2482 }
2483
2484 printk(" [%016lx] %pS\n", pc, (void *) pc);
2485 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2486 if ((pc + 8UL) == (unsigned long) &return_to_handler) {
2487 int index = tsk->curr_ret_stack;
2488 if (tsk->ret_stack && index >= graph) {
2489 pc = tsk->ret_stack[index - graph].ret;
2490 printk(" [%016lx] %pS\n", pc, (void *) pc);
2491 graph++;
2492 }
2493 }
2494 #endif
2495 } while (++count < 16);
2496 }
2497
2498 static inline struct reg_window *kernel_stack_up(struct reg_window *rw)
2499 {
2500 unsigned long fp = rw->ins[6];
2501
2502 if (!fp)
2503 return NULL;
2504
2505 return (struct reg_window *) (fp + STACK_BIAS);
2506 }
2507
2508 void __noreturn die_if_kernel(char *str, struct pt_regs *regs)
2509 {
2510 static int die_counter;
2511 int count = 0;
2512
2513 /* Amuse the user. */
2514 printk(
2515 " \\|/ ____ \\|/\n"
2516 " \"@'/ .. \\`@\"\n"
2517 " /_| \\__/ |_\\\n"
2518 " \\__U_/\n");
2519
2520 printk("%s(%d): %s [#%d]\n", current->comm, task_pid_nr(current), str, ++die_counter);
2521 notify_die(DIE_OOPS, str, regs, 0, 255, SIGSEGV);
2522 __asm__ __volatile__("flushw");
2523 show_regs(regs);
2524 add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
2525 if (regs->tstate & TSTATE_PRIV) {
2526 struct thread_info *tp = current_thread_info();
2527 struct reg_window *rw = (struct reg_window *)
2528 (regs->u_regs[UREG_FP] + STACK_BIAS);
2529
2530 /* Stop the back trace when we hit userland or we
2531 * find some badly aligned kernel stack.
2532 */
2533 while (rw &&
2534 count++ < 30 &&
2535 kstack_valid(tp, (unsigned long) rw)) {
2536 printk("Caller[%016lx]: %pS\n", rw->ins[7],
2537 (void *) rw->ins[7]);
2538
2539 rw = kernel_stack_up(rw);
2540 }
2541 instruction_dump ((unsigned int *) regs->tpc);
2542 } else {
2543 if (test_thread_flag(TIF_32BIT)) {
2544 regs->tpc &= 0xffffffff;
2545 regs->tnpc &= 0xffffffff;
2546 }
2547 user_instruction_dump ((unsigned int __user *) regs->tpc);
2548 }
2549 if (panic_on_oops)
2550 panic("Fatal exception");
2551 if (regs->tstate & TSTATE_PRIV)
2552 do_exit(SIGKILL);
2553 do_exit(SIGSEGV);
2554 }
2555 EXPORT_SYMBOL(die_if_kernel);
2556
2557 #define VIS_OPCODE_MASK ((0x3 << 30) | (0x3f << 19))
2558 #define VIS_OPCODE_VAL ((0x2 << 30) | (0x36 << 19))
2559
2560 void do_illegal_instruction(struct pt_regs *regs)
2561 {
2562 enum ctx_state prev_state = exception_enter();
2563 unsigned long pc = regs->tpc;
2564 unsigned long tstate = regs->tstate;
2565 u32 insn;
2566 siginfo_t info;
2567
2568 if (notify_die(DIE_TRAP, "illegal instruction", regs,
2569 0, 0x10, SIGILL) == NOTIFY_STOP)
2570 goto out;
2571
2572 if (tstate & TSTATE_PRIV)
2573 die_if_kernel("Kernel illegal instruction", regs);
2574 if (test_thread_flag(TIF_32BIT))
2575 pc = (u32)pc;
2576 if (get_user(insn, (u32 __user *) pc) != -EFAULT) {
2577 if ((insn & 0xc1ffc000) == 0x81700000) /* POPC */ {
2578 if (handle_popc(insn, regs))
2579 goto out;
2580 } else if ((insn & 0xc1580000) == 0xc1100000) /* LDQ/STQ */ {
2581 if (handle_ldf_stq(insn, regs))
2582 goto out;
2583 } else if (tlb_type == hypervisor) {
2584 if ((insn & VIS_OPCODE_MASK) == VIS_OPCODE_VAL) {
2585 if (!vis_emul(regs, insn))
2586 goto out;
2587 } else {
2588 struct fpustate *f = FPUSTATE;
2589
2590 /* On UltraSPARC T2 and later, FPU insns which
2591 * are not implemented in HW signal an illegal
2592 * instruction trap and do not set the FP Trap
2593 * Trap in the %fsr to unimplemented_FPop.
2594 */
2595 if (do_mathemu(regs, f, true))
2596 goto out;
2597 }
2598 }
2599 }
2600 info.si_signo = SIGILL;
2601 info.si_errno = 0;
2602 info.si_code = ILL_ILLOPC;
2603 info.si_addr = (void __user *)pc;
2604 info.si_trapno = 0;
2605 force_sig_info(SIGILL, &info, current);
2606 out:
2607 exception_exit(prev_state);
2608 }
2609
2610 void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr)
2611 {
2612 enum ctx_state prev_state = exception_enter();
2613 siginfo_t info;
2614
2615 if (notify_die(DIE_TRAP, "memory address unaligned", regs,
2616 0, 0x34, SIGSEGV) == NOTIFY_STOP)
2617 goto out;
2618
2619 if (regs->tstate & TSTATE_PRIV) {
2620 kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc));
2621 goto out;
2622 }
2623 if (is_no_fault_exception(regs))
2624 return;
2625
2626 info.si_signo = SIGBUS;
2627 info.si_errno = 0;
2628 info.si_code = BUS_ADRALN;
2629 info.si_addr = (void __user *)sfar;
2630 info.si_trapno = 0;
2631 force_sig_info(SIGBUS, &info, current);
2632 out:
2633 exception_exit(prev_state);
2634 }
2635
2636 void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
2637 {
2638 siginfo_t info;
2639
2640 if (notify_die(DIE_TRAP, "memory address unaligned", regs,
2641 0, 0x34, SIGSEGV) == NOTIFY_STOP)
2642 return;
2643
2644 if (regs->tstate & TSTATE_PRIV) {
2645 kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc));
2646 return;
2647 }
2648 if (is_no_fault_exception(regs))
2649 return;
2650
2651 info.si_signo = SIGBUS;
2652 info.si_errno = 0;
2653 info.si_code = BUS_ADRALN;
2654 info.si_addr = (void __user *) addr;
2655 info.si_trapno = 0;
2656 force_sig_info(SIGBUS, &info, current);
2657 }
2658
2659 void do_privop(struct pt_regs *regs)
2660 {
2661 enum ctx_state prev_state = exception_enter();
2662 siginfo_t info;
2663
2664 if (notify_die(DIE_TRAP, "privileged operation", regs,
2665 0, 0x11, SIGILL) == NOTIFY_STOP)
2666 goto out;
2667
2668 if (test_thread_flag(TIF_32BIT)) {
2669 regs->tpc &= 0xffffffff;
2670 regs->tnpc &= 0xffffffff;
2671 }
2672 info.si_signo = SIGILL;
2673 info.si_errno = 0;
2674 info.si_code = ILL_PRVOPC;
2675 info.si_addr = (void __user *)regs->tpc;
2676 info.si_trapno = 0;
2677 force_sig_info(SIGILL, &info, current);
2678 out:
2679 exception_exit(prev_state);
2680 }
2681
2682 void do_privact(struct pt_regs *regs)
2683 {
2684 do_privop(regs);
2685 }
2686
2687 /* Trap level 1 stuff or other traps we should never see... */
2688 void do_cee(struct pt_regs *regs)
2689 {
2690 exception_enter();
2691 die_if_kernel("TL0: Cache Error Exception", regs);
2692 }
2693
2694 void do_div0_tl1(struct pt_regs *regs)
2695 {
2696 exception_enter();
2697 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2698 die_if_kernel("TL1: DIV0 Exception", regs);
2699 }
2700
2701 void do_fpieee_tl1(struct pt_regs *regs)
2702 {
2703 exception_enter();
2704 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2705 die_if_kernel("TL1: FPU IEEE Exception", regs);
2706 }
2707
2708 void do_fpother_tl1(struct pt_regs *regs)
2709 {
2710 exception_enter();
2711 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2712 die_if_kernel("TL1: FPU Other Exception", regs);
2713 }
2714
2715 void do_ill_tl1(struct pt_regs *regs)
2716 {
2717 exception_enter();
2718 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2719 die_if_kernel("TL1: Illegal Instruction Exception", regs);
2720 }
2721
2722 void do_irq_tl1(struct pt_regs *regs)
2723 {
2724 exception_enter();
2725 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2726 die_if_kernel("TL1: IRQ Exception", regs);
2727 }
2728
2729 void do_lddfmna_tl1(struct pt_regs *regs)
2730 {
2731 exception_enter();
2732 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2733 die_if_kernel("TL1: LDDF Exception", regs);
2734 }
2735
2736 void do_stdfmna_tl1(struct pt_regs *regs)
2737 {
2738 exception_enter();
2739 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2740 die_if_kernel("TL1: STDF Exception", regs);
2741 }
2742
2743 void do_paw(struct pt_regs *regs)
2744 {
2745 exception_enter();
2746 die_if_kernel("TL0: Phys Watchpoint Exception", regs);
2747 }
2748
2749 void do_paw_tl1(struct pt_regs *regs)
2750 {
2751 exception_enter();
2752 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2753 die_if_kernel("TL1: Phys Watchpoint Exception", regs);
2754 }
2755
2756 void do_vaw(struct pt_regs *regs)
2757 {
2758 exception_enter();
2759 die_if_kernel("TL0: Virt Watchpoint Exception", regs);
2760 }
2761
2762 void do_vaw_tl1(struct pt_regs *regs)
2763 {
2764 exception_enter();
2765 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2766 die_if_kernel("TL1: Virt Watchpoint Exception", regs);
2767 }
2768
2769 void do_tof_tl1(struct pt_regs *regs)
2770 {
2771 exception_enter();
2772 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2773 die_if_kernel("TL1: Tag Overflow Exception", regs);
2774 }
2775
2776 void do_getpsr(struct pt_regs *regs)
2777 {
2778 regs->u_regs[UREG_I0] = tstate_to_psr(regs->tstate);
2779 regs->tpc = regs->tnpc;
2780 regs->tnpc += 4;
2781 if (test_thread_flag(TIF_32BIT)) {
2782 regs->tpc &= 0xffffffff;
2783 regs->tnpc &= 0xffffffff;
2784 }
2785 }
2786
2787 u64 cpu_mondo_counter[NR_CPUS] = {0};
2788 struct trap_per_cpu trap_block[NR_CPUS];
2789 EXPORT_SYMBOL(trap_block);
2790
2791 /* This can get invoked before sched_init() so play it super safe
2792 * and use hard_smp_processor_id().
2793 */
2794 void notrace init_cur_cpu_trap(struct thread_info *t)
2795 {
2796 int cpu = hard_smp_processor_id();
2797 struct trap_per_cpu *p = &trap_block[cpu];
2798
2799 p->thread = t;
2800 p->pgd_paddr = 0;
2801 }
2802
2803 extern void thread_info_offsets_are_bolixed_dave(void);
2804 extern void trap_per_cpu_offsets_are_bolixed_dave(void);
2805 extern void tsb_config_offsets_are_bolixed_dave(void);
2806
2807 /* Only invoked on boot processor. */
2808 void __init trap_init(void)
2809 {
2810 /* Compile time sanity check. */
2811 BUILD_BUG_ON(TI_TASK != offsetof(struct thread_info, task) ||
2812 TI_FLAGS != offsetof(struct thread_info, flags) ||
2813 TI_CPU != offsetof(struct thread_info, cpu) ||
2814 TI_FPSAVED != offsetof(struct thread_info, fpsaved) ||
2815 TI_KSP != offsetof(struct thread_info, ksp) ||
2816 TI_FAULT_ADDR != offsetof(struct thread_info,
2817 fault_address) ||
2818 TI_KREGS != offsetof(struct thread_info, kregs) ||
2819 TI_UTRAPS != offsetof(struct thread_info, utraps) ||
2820 TI_REG_WINDOW != offsetof(struct thread_info,
2821 reg_window) ||
2822 TI_RWIN_SPTRS != offsetof(struct thread_info,
2823 rwbuf_stkptrs) ||
2824 TI_GSR != offsetof(struct thread_info, gsr) ||
2825 TI_XFSR != offsetof(struct thread_info, xfsr) ||
2826 TI_PRE_COUNT != offsetof(struct thread_info,
2827 preempt_count) ||
2828 TI_NEW_CHILD != offsetof(struct thread_info, new_child) ||
2829 TI_CURRENT_DS != offsetof(struct thread_info,
2830 current_ds) ||
2831 TI_KUNA_REGS != offsetof(struct thread_info,
2832 kern_una_regs) ||
2833 TI_KUNA_INSN != offsetof(struct thread_info,
2834 kern_una_insn) ||
2835 TI_FPREGS != offsetof(struct thread_info, fpregs) ||
2836 (TI_FPREGS & (64 - 1)));
2837
2838 BUILD_BUG_ON(TRAP_PER_CPU_THREAD != offsetof(struct trap_per_cpu,
2839 thread) ||
2840 (TRAP_PER_CPU_PGD_PADDR !=
2841 offsetof(struct trap_per_cpu, pgd_paddr)) ||
2842 (TRAP_PER_CPU_CPU_MONDO_PA !=
2843 offsetof(struct trap_per_cpu, cpu_mondo_pa)) ||
2844 (TRAP_PER_CPU_DEV_MONDO_PA !=
2845 offsetof(struct trap_per_cpu, dev_mondo_pa)) ||
2846 (TRAP_PER_CPU_RESUM_MONDO_PA !=
2847 offsetof(struct trap_per_cpu, resum_mondo_pa)) ||
2848 (TRAP_PER_CPU_RESUM_KBUF_PA !=
2849 offsetof(struct trap_per_cpu, resum_kernel_buf_pa)) ||
2850 (TRAP_PER_CPU_NONRESUM_MONDO_PA !=
2851 offsetof(struct trap_per_cpu, nonresum_mondo_pa)) ||
2852 (TRAP_PER_CPU_NONRESUM_KBUF_PA !=
2853 offsetof(struct trap_per_cpu, nonresum_kernel_buf_pa)) ||
2854 (TRAP_PER_CPU_FAULT_INFO !=
2855 offsetof(struct trap_per_cpu, fault_info)) ||
2856 (TRAP_PER_CPU_CPU_MONDO_BLOCK_PA !=
2857 offsetof(struct trap_per_cpu, cpu_mondo_block_pa)) ||
2858 (TRAP_PER_CPU_CPU_LIST_PA !=
2859 offsetof(struct trap_per_cpu, cpu_list_pa)) ||
2860 (TRAP_PER_CPU_TSB_HUGE !=
2861 offsetof(struct trap_per_cpu, tsb_huge)) ||
2862 (TRAP_PER_CPU_TSB_HUGE_TEMP !=
2863 offsetof(struct trap_per_cpu, tsb_huge_temp)) ||
2864 (TRAP_PER_CPU_IRQ_WORKLIST_PA !=
2865 offsetof(struct trap_per_cpu, irq_worklist_pa)) ||
2866 (TRAP_PER_CPU_CPU_MONDO_QMASK !=
2867 offsetof(struct trap_per_cpu, cpu_mondo_qmask)) ||
2868 (TRAP_PER_CPU_DEV_MONDO_QMASK !=
2869 offsetof(struct trap_per_cpu, dev_mondo_qmask)) ||
2870 (TRAP_PER_CPU_RESUM_QMASK !=
2871 offsetof(struct trap_per_cpu, resum_qmask)) ||
2872 (TRAP_PER_CPU_NONRESUM_QMASK !=
2873 offsetof(struct trap_per_cpu, nonresum_qmask)) ||
2874 (TRAP_PER_CPU_PER_CPU_BASE !=
2875 offsetof(struct trap_per_cpu, __per_cpu_base)));
2876
2877 BUILD_BUG_ON((TSB_CONFIG_TSB !=
2878 offsetof(struct tsb_config, tsb)) ||
2879 (TSB_CONFIG_RSS_LIMIT !=
2880 offsetof(struct tsb_config, tsb_rss_limit)) ||
2881 (TSB_CONFIG_NENTRIES !=
2882 offsetof(struct tsb_config, tsb_nentries)) ||
2883 (TSB_CONFIG_REG_VAL !=
2884 offsetof(struct tsb_config, tsb_reg_val)) ||
2885 (TSB_CONFIG_MAP_VADDR !=
2886 offsetof(struct tsb_config, tsb_map_vaddr)) ||
2887 (TSB_CONFIG_MAP_PTE !=
2888 offsetof(struct tsb_config, tsb_map_pte)));
2889
2890 /* Attach to the address space of init_task. On SMP we
2891 * do this in smp.c:smp_callin for other cpus.
2892 */
2893 mmgrab(&init_mm);
2894 current->active_mm = &init_mm;
2895 }
2896
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.