~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/sparc/kernel/traps_64.c

Version: ~ [ linux-4.15-rc3 ] ~ [ linux-4.14.5 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.68 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.105 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.47 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.87 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.51 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.96 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.27.62 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* arch/sparc64/kernel/traps.c
  2  *
  3  * Copyright (C) 1995,1997,2008,2009,2012 David S. Miller (davem@davemloft.net)
  4  * Copyright (C) 1997,1999,2000 Jakub Jelinek (jakub@redhat.com)
  5  */
  6 
  7 /*
  8  * I like traps on v9, :))))
  9  */
 10 
 11 #include <linux/extable.h>
 12 #include <linux/sched/mm.h>
 13 #include <linux/sched/debug.h>
 14 #include <linux/linkage.h>
 15 #include <linux/kernel.h>
 16 #include <linux/signal.h>
 17 #include <linux/smp.h>
 18 #include <linux/mm.h>
 19 #include <linux/init.h>
 20 #include <linux/kdebug.h>
 21 #include <linux/ftrace.h>
 22 #include <linux/reboot.h>
 23 #include <linux/gfp.h>
 24 #include <linux/context_tracking.h>
 25 
 26 #include <asm/smp.h>
 27 #include <asm/delay.h>
 28 #include <asm/ptrace.h>
 29 #include <asm/oplib.h>
 30 #include <asm/page.h>
 31 #include <asm/pgtable.h>
 32 #include <asm/unistd.h>
 33 #include <linux/uaccess.h>
 34 #include <asm/fpumacro.h>
 35 #include <asm/lsu.h>
 36 #include <asm/dcu.h>
 37 #include <asm/estate.h>
 38 #include <asm/chafsr.h>
 39 #include <asm/sfafsr.h>
 40 #include <asm/psrcompat.h>
 41 #include <asm/processor.h>
 42 #include <asm/timer.h>
 43 #include <asm/head.h>
 44 #include <asm/prom.h>
 45 #include <asm/memctrl.h>
 46 #include <asm/cacheflush.h>
 47 #include <asm/setup.h>
 48 
 49 #include "entry.h"
 50 #include "kernel.h"
 51 #include "kstack.h"
 52 
 53 /* When an irrecoverable trap occurs at tl > 0, the trap entry
 54  * code logs the trap state registers at every level in the trap
 55  * stack.  It is found at (pt_regs + sizeof(pt_regs)) and the layout
 56  * is as follows:
 57  */
 58 struct tl1_traplog {
 59         struct {
 60                 unsigned long tstate;
 61                 unsigned long tpc;
 62                 unsigned long tnpc;
 63                 unsigned long tt;
 64         } trapstack[4];
 65         unsigned long tl;
 66 };
 67 
 68 static void dump_tl1_traplog(struct tl1_traplog *p)
 69 {
 70         int i, limit;
 71 
 72         printk(KERN_EMERG "TRAPLOG: Error at trap level 0x%lx, "
 73                "dumping track stack.\n", p->tl);
 74 
 75         limit = (tlb_type == hypervisor) ? 2 : 4;
 76         for (i = 0; i < limit; i++) {
 77                 printk(KERN_EMERG
 78                        "TRAPLOG: Trap level %d TSTATE[%016lx] TPC[%016lx] "
 79                        "TNPC[%016lx] TT[%lx]\n",
 80                        i + 1,
 81                        p->trapstack[i].tstate, p->trapstack[i].tpc,
 82                        p->trapstack[i].tnpc, p->trapstack[i].tt);
 83                 printk("TRAPLOG: TPC<%pS>\n", (void *) p->trapstack[i].tpc);
 84         }
 85 }
 86 
 87 void bad_trap(struct pt_regs *regs, long lvl)
 88 {
 89         char buffer[36];
 90         siginfo_t info;
 91 
 92         if (notify_die(DIE_TRAP, "bad trap", regs,
 93                        0, lvl, SIGTRAP) == NOTIFY_STOP)
 94                 return;
 95 
 96         if (lvl < 0x100) {
 97                 sprintf(buffer, "Bad hw trap %lx at tl0\n", lvl);
 98                 die_if_kernel(buffer, regs);
 99         }
100 
101         lvl -= 0x100;
102         if (regs->tstate & TSTATE_PRIV) {
103                 sprintf(buffer, "Kernel bad sw trap %lx", lvl);
104                 die_if_kernel(buffer, regs);
105         }
106         if (test_thread_flag(TIF_32BIT)) {
107                 regs->tpc &= 0xffffffff;
108                 regs->tnpc &= 0xffffffff;
109         }
110         info.si_signo = SIGILL;
111         info.si_errno = 0;
112         info.si_code = ILL_ILLTRP;
113         info.si_addr = (void __user *)regs->tpc;
114         info.si_trapno = lvl;
115         force_sig_info(SIGILL, &info, current);
116 }
117 
118 void bad_trap_tl1(struct pt_regs *regs, long lvl)
119 {
120         char buffer[36];
121         
122         if (notify_die(DIE_TRAP_TL1, "bad trap tl1", regs,
123                        0, lvl, SIGTRAP) == NOTIFY_STOP)
124                 return;
125 
126         dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
127 
128         sprintf (buffer, "Bad trap %lx at tl>0", lvl);
129         die_if_kernel (buffer, regs);
130 }
131 
132 #ifdef CONFIG_DEBUG_BUGVERBOSE
133 void do_BUG(const char *file, int line)
134 {
135         bust_spinlocks(1);
136         printk("kernel BUG at %s:%d!\n", file, line);
137 }
138 EXPORT_SYMBOL(do_BUG);
139 #endif
140 
141 static DEFINE_SPINLOCK(dimm_handler_lock);
142 static dimm_printer_t dimm_handler;
143 
144 static int sprintf_dimm(int synd_code, unsigned long paddr, char *buf, int buflen)
145 {
146         unsigned long flags;
147         int ret = -ENODEV;
148 
149         spin_lock_irqsave(&dimm_handler_lock, flags);
150         if (dimm_handler) {
151                 ret = dimm_handler(synd_code, paddr, buf, buflen);
152         } else if (tlb_type == spitfire) {
153                 if (prom_getunumber(synd_code, paddr, buf, buflen) == -1)
154                         ret = -EINVAL;
155                 else
156                         ret = 0;
157         } else
158                 ret = -ENODEV;
159         spin_unlock_irqrestore(&dimm_handler_lock, flags);
160 
161         return ret;
162 }
163 
164 int register_dimm_printer(dimm_printer_t func)
165 {
166         unsigned long flags;
167         int ret = 0;
168 
169         spin_lock_irqsave(&dimm_handler_lock, flags);
170         if (!dimm_handler)
171                 dimm_handler = func;
172         else
173                 ret = -EEXIST;
174         spin_unlock_irqrestore(&dimm_handler_lock, flags);
175 
176         return ret;
177 }
178 EXPORT_SYMBOL_GPL(register_dimm_printer);
179 
180 void unregister_dimm_printer(dimm_printer_t func)
181 {
182         unsigned long flags;
183 
184         spin_lock_irqsave(&dimm_handler_lock, flags);
185         if (dimm_handler == func)
186                 dimm_handler = NULL;
187         spin_unlock_irqrestore(&dimm_handler_lock, flags);
188 }
189 EXPORT_SYMBOL_GPL(unregister_dimm_printer);
190 
191 void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
192 {
193         enum ctx_state prev_state = exception_enter();
194         siginfo_t info;
195 
196         if (notify_die(DIE_TRAP, "instruction access exception", regs,
197                        0, 0x8, SIGTRAP) == NOTIFY_STOP)
198                 goto out;
199 
200         if (regs->tstate & TSTATE_PRIV) {
201                 printk("spitfire_insn_access_exception: SFSR[%016lx] "
202                        "SFAR[%016lx], going.\n", sfsr, sfar);
203                 die_if_kernel("Iax", regs);
204         }
205         if (test_thread_flag(TIF_32BIT)) {
206                 regs->tpc &= 0xffffffff;
207                 regs->tnpc &= 0xffffffff;
208         }
209         info.si_signo = SIGSEGV;
210         info.si_errno = 0;
211         info.si_code = SEGV_MAPERR;
212         info.si_addr = (void __user *)regs->tpc;
213         info.si_trapno = 0;
214         force_sig_info(SIGSEGV, &info, current);
215 out:
216         exception_exit(prev_state);
217 }
218 
219 void spitfire_insn_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
220 {
221         if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs,
222                        0, 0x8, SIGTRAP) == NOTIFY_STOP)
223                 return;
224 
225         dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
226         spitfire_insn_access_exception(regs, sfsr, sfar);
227 }
228 
229 void sun4v_insn_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
230 {
231         unsigned short type = (type_ctx >> 16);
232         unsigned short ctx  = (type_ctx & 0xffff);
233         siginfo_t info;
234 
235         if (notify_die(DIE_TRAP, "instruction access exception", regs,
236                        0, 0x8, SIGTRAP) == NOTIFY_STOP)
237                 return;
238 
239         if (regs->tstate & TSTATE_PRIV) {
240                 printk("sun4v_insn_access_exception: ADDR[%016lx] "
241                        "CTX[%04x] TYPE[%04x], going.\n",
242                        addr, ctx, type);
243                 die_if_kernel("Iax", regs);
244         }
245 
246         if (test_thread_flag(TIF_32BIT)) {
247                 regs->tpc &= 0xffffffff;
248                 regs->tnpc &= 0xffffffff;
249         }
250         info.si_signo = SIGSEGV;
251         info.si_errno = 0;
252         info.si_code = SEGV_MAPERR;
253         info.si_addr = (void __user *) addr;
254         info.si_trapno = 0;
255         force_sig_info(SIGSEGV, &info, current);
256 }
257 
258 void sun4v_insn_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
259 {
260         if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs,
261                        0, 0x8, SIGTRAP) == NOTIFY_STOP)
262                 return;
263 
264         dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
265         sun4v_insn_access_exception(regs, addr, type_ctx);
266 }
267 
268 bool is_no_fault_exception(struct pt_regs *regs)
269 {
270         unsigned char asi;
271         u32 insn;
272 
273         if (get_user(insn, (u32 __user *)regs->tpc) == -EFAULT)
274                 return false;
275 
276         /*
277          * Must do a little instruction decoding here in order to
278          * decide on a course of action. The bits of interest are:
279          *  insn[31:30] = op, where 3 indicates the load/store group
280          *  insn[24:19] = op3, which identifies individual opcodes
281          *  insn[13] indicates an immediate offset
282          *  op3[4]=1 identifies alternate space instructions
283          *  op3[5:4]=3 identifies floating point instructions
284          *  op3[2]=1 identifies stores
285          * See "Opcode Maps" in the appendix of any Sparc V9
286          * architecture spec for full details.
287          */
288         if ((insn & 0xc0800000) == 0xc0800000) {    /* op=3, op3[4]=1   */
289                 if (insn & 0x2000)                  /* immediate offset */
290                         asi = (regs->tstate >> 24); /* saved %asi       */
291                 else
292                         asi = (insn >> 5);          /* immediate asi    */
293                 if ((asi & 0xf2) == ASI_PNF) {
294                         if (insn & 0x1000000) {     /* op3[5:4]=3       */
295                                 handle_ldf_stq(insn, regs);
296                                 return true;
297                         } else if (insn & 0x200000) { /* op3[2], stores */
298                                 return false;
299                         }
300                         handle_ld_nf(insn, regs);
301                         return true;
302                 }
303         }
304         return false;
305 }
306 
307 void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
308 {
309         enum ctx_state prev_state = exception_enter();
310         siginfo_t info;
311 
312         if (notify_die(DIE_TRAP, "data access exception", regs,
313                        0, 0x30, SIGTRAP) == NOTIFY_STOP)
314                 goto out;
315 
316         if (regs->tstate & TSTATE_PRIV) {
317                 /* Test if this comes from uaccess places. */
318                 const struct exception_table_entry *entry;
319 
320                 entry = search_exception_tables(regs->tpc);
321                 if (entry) {
322                         /* Ouch, somebody is trying VM hole tricks on us... */
323 #ifdef DEBUG_EXCEPTIONS
324                         printk("Exception: PC<%016lx> faddr<UNKNOWN>\n", regs->tpc);
325                         printk("EX_TABLE: insn<%016lx> fixup<%016lx>\n",
326                                regs->tpc, entry->fixup);
327 #endif
328                         regs->tpc = entry->fixup;
329                         regs->tnpc = regs->tpc + 4;
330                         goto out;
331                 }
332                 /* Shit... */
333                 printk("spitfire_data_access_exception: SFSR[%016lx] "
334                        "SFAR[%016lx], going.\n", sfsr, sfar);
335                 die_if_kernel("Dax", regs);
336         }
337 
338         if (is_no_fault_exception(regs))
339                 return;
340 
341         info.si_signo = SIGSEGV;
342         info.si_errno = 0;
343         info.si_code = SEGV_MAPERR;
344         info.si_addr = (void __user *)sfar;
345         info.si_trapno = 0;
346         force_sig_info(SIGSEGV, &info, current);
347 out:
348         exception_exit(prev_state);
349 }
350 
351 void spitfire_data_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
352 {
353         if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs,
354                        0, 0x30, SIGTRAP) == NOTIFY_STOP)
355                 return;
356 
357         dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
358         spitfire_data_access_exception(regs, sfsr, sfar);
359 }
360 
361 void sun4v_data_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
362 {
363         unsigned short type = (type_ctx >> 16);
364         unsigned short ctx  = (type_ctx & 0xffff);
365         siginfo_t info;
366 
367         if (notify_die(DIE_TRAP, "data access exception", regs,
368                        0, 0x8, SIGTRAP) == NOTIFY_STOP)
369                 return;
370 
371         if (regs->tstate & TSTATE_PRIV) {
372                 /* Test if this comes from uaccess places. */
373                 const struct exception_table_entry *entry;
374 
375                 entry = search_exception_tables(regs->tpc);
376                 if (entry) {
377                         /* Ouch, somebody is trying VM hole tricks on us... */
378 #ifdef DEBUG_EXCEPTIONS
379                         printk("Exception: PC<%016lx> faddr<UNKNOWN>\n", regs->tpc);
380                         printk("EX_TABLE: insn<%016lx> fixup<%016lx>\n",
381                                regs->tpc, entry->fixup);
382 #endif
383                         regs->tpc = entry->fixup;
384                         regs->tnpc = regs->tpc + 4;
385                         return;
386                 }
387                 printk("sun4v_data_access_exception: ADDR[%016lx] "
388                        "CTX[%04x] TYPE[%04x], going.\n",
389                        addr, ctx, type);
390                 die_if_kernel("Dax", regs);
391         }
392 
393         if (test_thread_flag(TIF_32BIT)) {
394                 regs->tpc &= 0xffffffff;
395                 regs->tnpc &= 0xffffffff;
396         }
397         if (is_no_fault_exception(regs))
398                 return;
399 
400         info.si_signo = SIGSEGV;
401         info.si_errno = 0;
402         info.si_code = SEGV_MAPERR;
403         info.si_addr = (void __user *) addr;
404         info.si_trapno = 0;
405         force_sig_info(SIGSEGV, &info, current);
406 }
407 
408 void sun4v_data_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
409 {
410         if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs,
411                        0, 0x8, SIGTRAP) == NOTIFY_STOP)
412                 return;
413 
414         dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
415         sun4v_data_access_exception(regs, addr, type_ctx);
416 }
417 
418 #ifdef CONFIG_PCI
419 #include "pci_impl.h"
420 #endif
421 
422 /* When access exceptions happen, we must do this. */
423 static void spitfire_clean_and_reenable_l1_caches(void)
424 {
425         unsigned long va;
426 
427         if (tlb_type != spitfire)
428                 BUG();
429 
430         /* Clean 'em. */
431         for (va =  0; va < (PAGE_SIZE << 1); va += 32) {
432                 spitfire_put_icache_tag(va, 0x0);
433                 spitfire_put_dcache_tag(va, 0x0);
434         }
435 
436         /* Re-enable in LSU. */
437         __asm__ __volatile__("flush %%g6\n\t"
438                              "membar #Sync\n\t"
439                              "stxa %0, [%%g0] %1\n\t"
440                              "membar #Sync"
441                              : /* no outputs */
442                              : "r" (LSU_CONTROL_IC | LSU_CONTROL_DC |
443                                     LSU_CONTROL_IM | LSU_CONTROL_DM),
444                              "i" (ASI_LSU_CONTROL)
445                              : "memory");
446 }
447 
448 static void spitfire_enable_estate_errors(void)
449 {
450         __asm__ __volatile__("stxa      %0, [%%g0] %1\n\t"
451                              "membar    #Sync"
452                              : /* no outputs */
453                              : "r" (ESTATE_ERR_ALL),
454                                "i" (ASI_ESTATE_ERROR_EN));
455 }
456 
457 static char ecc_syndrome_table[] = {
458         0x4c, 0x40, 0x41, 0x48, 0x42, 0x48, 0x48, 0x49,
459         0x43, 0x48, 0x48, 0x49, 0x48, 0x49, 0x49, 0x4a,
460         0x44, 0x48, 0x48, 0x20, 0x48, 0x39, 0x4b, 0x48,
461         0x48, 0x25, 0x31, 0x48, 0x28, 0x48, 0x48, 0x2c,
462         0x45, 0x48, 0x48, 0x21, 0x48, 0x3d, 0x04, 0x48,
463         0x48, 0x4b, 0x35, 0x48, 0x2d, 0x48, 0x48, 0x29,
464         0x48, 0x00, 0x01, 0x48, 0x0a, 0x48, 0x48, 0x4b,
465         0x0f, 0x48, 0x48, 0x4b, 0x48, 0x49, 0x49, 0x48,
466         0x46, 0x48, 0x48, 0x2a, 0x48, 0x3b, 0x27, 0x48,
467         0x48, 0x4b, 0x33, 0x48, 0x22, 0x48, 0x48, 0x2e,
468         0x48, 0x19, 0x1d, 0x48, 0x1b, 0x4a, 0x48, 0x4b,
469         0x1f, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48,
470         0x48, 0x4b, 0x24, 0x48, 0x07, 0x48, 0x48, 0x36,
471         0x4b, 0x48, 0x48, 0x3e, 0x48, 0x30, 0x38, 0x48,
472         0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x16, 0x48,
473         0x48, 0x12, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b,
474         0x47, 0x48, 0x48, 0x2f, 0x48, 0x3f, 0x4b, 0x48,
475         0x48, 0x06, 0x37, 0x48, 0x23, 0x48, 0x48, 0x2b,
476         0x48, 0x05, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x32,
477         0x26, 0x48, 0x48, 0x3a, 0x48, 0x34, 0x3c, 0x48,
478         0x48, 0x11, 0x15, 0x48, 0x13, 0x4a, 0x48, 0x4b,
479         0x17, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48,
480         0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x1e, 0x48,
481         0x48, 0x1a, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b,
482         0x48, 0x08, 0x0d, 0x48, 0x02, 0x48, 0x48, 0x49,
483         0x03, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x4b, 0x48,
484         0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x10, 0x48,
485         0x48, 0x14, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b,
486         0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x18, 0x48,
487         0x48, 0x1c, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b,
488         0x4a, 0x0c, 0x09, 0x48, 0x0e, 0x48, 0x48, 0x4b,
489         0x0b, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x4b, 0x4a
490 };
491 
492 static char *syndrome_unknown = "<Unknown>";
493 
494 static void spitfire_log_udb_syndrome(unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long bit)
495 {
496         unsigned short scode;
497         char memmod_str[64], *p;
498 
499         if (udbl & bit) {
500                 scode = ecc_syndrome_table[udbl & 0xff];
501                 if (sprintf_dimm(scode, afar, memmod_str, sizeof(memmod_str)) < 0)
502                         p = syndrome_unknown;
503                 else
504                         p = memmod_str;
505                 printk(KERN_WARNING "CPU[%d]: UDBL Syndrome[%x] "
506                        "Memory Module \"%s\"\n",
507                        smp_processor_id(), scode, p);
508         }
509 
510         if (udbh & bit) {
511                 scode = ecc_syndrome_table[udbh & 0xff];
512                 if (sprintf_dimm(scode, afar, memmod_str, sizeof(memmod_str)) < 0)
513                         p = syndrome_unknown;
514                 else
515                         p = memmod_str;
516                 printk(KERN_WARNING "CPU[%d]: UDBH Syndrome[%x] "
517                        "Memory Module \"%s\"\n",
518                        smp_processor_id(), scode, p);
519         }
520 
521 }
522 
523 static void spitfire_cee_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, int tl1, struct pt_regs *regs)
524 {
525 
526         printk(KERN_WARNING "CPU[%d]: Correctable ECC Error "
527                "AFSR[%lx] AFAR[%016lx] UDBL[%lx] UDBH[%lx] TL>1[%d]\n",
528                smp_processor_id(), afsr, afar, udbl, udbh, tl1);
529 
530         spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_CE);
531 
532         /* We always log it, even if someone is listening for this
533          * trap.
534          */
535         notify_die(DIE_TRAP, "Correctable ECC Error", regs,
536                    0, TRAP_TYPE_CEE, SIGTRAP);
537 
538         /* The Correctable ECC Error trap does not disable I/D caches.  So
539          * we only have to restore the ESTATE Error Enable register.
540          */
541         spitfire_enable_estate_errors();
542 }
543 
544 static void spitfire_ue_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long tt, int tl1, struct pt_regs *regs)
545 {
546         siginfo_t info;
547 
548         printk(KERN_WARNING "CPU[%d]: Uncorrectable Error AFSR[%lx] "
549                "AFAR[%lx] UDBL[%lx] UDBH[%ld] TT[%lx] TL>1[%d]\n",
550                smp_processor_id(), afsr, afar, udbl, udbh, tt, tl1);
551 
552         /* XXX add more human friendly logging of the error status
553          * XXX as is implemented for cheetah
554          */
555 
556         spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_UE);
557 
558         /* We always log it, even if someone is listening for this
559          * trap.
560          */
561         notify_die(DIE_TRAP, "Uncorrectable Error", regs,
562                    0, tt, SIGTRAP);
563 
564         if (regs->tstate & TSTATE_PRIV) {
565                 if (tl1)
566                         dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
567                 die_if_kernel("UE", regs);
568         }
569 
570         /* XXX need more intelligent processing here, such as is implemented
571          * XXX for cheetah errors, in fact if the E-cache still holds the
572          * XXX line with bad parity this will loop
573          */
574 
575         spitfire_clean_and_reenable_l1_caches();
576         spitfire_enable_estate_errors();
577 
578         if (test_thread_flag(TIF_32BIT)) {
579                 regs->tpc &= 0xffffffff;
580                 regs->tnpc &= 0xffffffff;
581         }
582         info.si_signo = SIGBUS;
583         info.si_errno = 0;
584         info.si_code = BUS_OBJERR;
585         info.si_addr = (void *)0;
586         info.si_trapno = 0;
587         force_sig_info(SIGBUS, &info, current);
588 }
589 
590 void spitfire_access_error(struct pt_regs *regs, unsigned long status_encoded, unsigned long afar)
591 {
592         unsigned long afsr, tt, udbh, udbl;
593         int tl1;
594 
595         afsr = (status_encoded & SFSTAT_AFSR_MASK) >> SFSTAT_AFSR_SHIFT;
596         tt = (status_encoded & SFSTAT_TRAP_TYPE) >> SFSTAT_TRAP_TYPE_SHIFT;
597         tl1 = (status_encoded & SFSTAT_TL_GT_ONE) ? 1 : 0;
598         udbl = (status_encoded & SFSTAT_UDBL_MASK) >> SFSTAT_UDBL_SHIFT;
599         udbh = (status_encoded & SFSTAT_UDBH_MASK) >> SFSTAT_UDBH_SHIFT;
600 
601 #ifdef CONFIG_PCI
602         if (tt == TRAP_TYPE_DAE &&
603             pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) {
604                 spitfire_clean_and_reenable_l1_caches();
605                 spitfire_enable_estate_errors();
606 
607                 pci_poke_faulted = 1;
608                 regs->tnpc = regs->tpc + 4;
609                 return;
610         }
611 #endif
612 
613         if (afsr & SFAFSR_UE)
614                 spitfire_ue_log(afsr, afar, udbh, udbl, tt, tl1, regs);
615 
616         if (tt == TRAP_TYPE_CEE) {
617                 /* Handle the case where we took a CEE trap, but ACK'd
618                  * only the UE state in the UDB error registers.
619                  */
620                 if (afsr & SFAFSR_UE) {
621                         if (udbh & UDBE_CE) {
622                                 __asm__ __volatile__(
623                                         "stxa   %0, [%1] %2\n\t"
624                                         "membar #Sync"
625                                         : /* no outputs */
626                                         : "r" (udbh & UDBE_CE),
627                                           "r" (0x0), "i" (ASI_UDB_ERROR_W));
628                         }
629                         if (udbl & UDBE_CE) {
630                                 __asm__ __volatile__(
631                                         "stxa   %0, [%1] %2\n\t"
632                                         "membar #Sync"
633                                         : /* no outputs */
634                                         : "r" (udbl & UDBE_CE),
635                                           "r" (0x18), "i" (ASI_UDB_ERROR_W));
636                         }
637                 }
638 
639                 spitfire_cee_log(afsr, afar, udbh, udbl, tl1, regs);
640         }
641 }
642 
643 int cheetah_pcache_forced_on;
644 
645 void cheetah_enable_pcache(void)
646 {
647         unsigned long dcr;
648 
649         printk("CHEETAH: Enabling P-Cache on cpu %d.\n",
650                smp_processor_id());
651 
652         __asm__ __volatile__("ldxa [%%g0] %1, %0"
653                              : "=r" (dcr)
654                              : "i" (ASI_DCU_CONTROL_REG));
655         dcr |= (DCU_PE | DCU_HPE | DCU_SPE | DCU_SL);
656         __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
657                              "membar #Sync"
658                              : /* no outputs */
659                              : "r" (dcr), "i" (ASI_DCU_CONTROL_REG));
660 }
661 
662 /* Cheetah error trap handling. */
663 static unsigned long ecache_flush_physbase;
664 static unsigned long ecache_flush_linesize;
665 static unsigned long ecache_flush_size;
666 
667 /* This table is ordered in priority of errors and matches the
668  * AFAR overwrite policy as well.
669  */
670 
671 struct afsr_error_table {
672         unsigned long mask;
673         const char *name;
674 };
675 
676 static const char CHAFSR_PERR_msg[] =
677         "System interface protocol error";
678 static const char CHAFSR_IERR_msg[] =
679         "Internal processor error";
680 static const char CHAFSR_ISAP_msg[] =
681         "System request parity error on incoming address";
682 static const char CHAFSR_UCU_msg[] =
683         "Uncorrectable E-cache ECC error for ifetch/data";
684 static const char CHAFSR_UCC_msg[] =
685         "SW Correctable E-cache ECC error for ifetch/data";
686 static const char CHAFSR_UE_msg[] =
687         "Uncorrectable system bus data ECC error for read";
688 static const char CHAFSR_EDU_msg[] =
689         "Uncorrectable E-cache ECC error for stmerge/blkld";
690 static const char CHAFSR_EMU_msg[] =
691         "Uncorrectable system bus MTAG error";
692 static const char CHAFSR_WDU_msg[] =
693         "Uncorrectable E-cache ECC error for writeback";
694 static const char CHAFSR_CPU_msg[] =
695         "Uncorrectable ECC error for copyout";
696 static const char CHAFSR_CE_msg[] =
697         "HW corrected system bus data ECC error for read";
698 static const char CHAFSR_EDC_msg[] =
699         "HW corrected E-cache ECC error for stmerge/blkld";
700 static const char CHAFSR_EMC_msg[] =
701         "HW corrected system bus MTAG ECC error";
702 static const char CHAFSR_WDC_msg[] =
703         "HW corrected E-cache ECC error for writeback";
704 static const char CHAFSR_CPC_msg[] =
705         "HW corrected ECC error for copyout";
706 static const char CHAFSR_TO_msg[] =
707         "Unmapped error from system bus";
708 static const char CHAFSR_BERR_msg[] =
709         "Bus error response from system bus";
710 static const char CHAFSR_IVC_msg[] =
711         "HW corrected system bus data ECC error for ivec read";
712 static const char CHAFSR_IVU_msg[] =
713         "Uncorrectable system bus data ECC error for ivec read";
714 static struct afsr_error_table __cheetah_error_table[] = {
715         {       CHAFSR_PERR,    CHAFSR_PERR_msg         },
716         {       CHAFSR_IERR,    CHAFSR_IERR_msg         },
717         {       CHAFSR_ISAP,    CHAFSR_ISAP_msg         },
718         {       CHAFSR_UCU,     CHAFSR_UCU_msg          },
719         {       CHAFSR_UCC,     CHAFSR_UCC_msg          },
720         {       CHAFSR_UE,      CHAFSR_UE_msg           },
721         {       CHAFSR_EDU,     CHAFSR_EDU_msg          },
722         {       CHAFSR_EMU,     CHAFSR_EMU_msg          },
723         {       CHAFSR_WDU,     CHAFSR_WDU_msg          },
724         {       CHAFSR_CPU,     CHAFSR_CPU_msg          },
725         {       CHAFSR_CE,      CHAFSR_CE_msg           },
726         {       CHAFSR_EDC,     CHAFSR_EDC_msg          },
727         {       CHAFSR_EMC,     CHAFSR_EMC_msg          },
728         {       CHAFSR_WDC,     CHAFSR_WDC_msg          },
729         {       CHAFSR_CPC,     CHAFSR_CPC_msg          },
730         {       CHAFSR_TO,      CHAFSR_TO_msg           },
731         {       CHAFSR_BERR,    CHAFSR_BERR_msg         },
732         /* These two do not update the AFAR. */
733         {       CHAFSR_IVC,     CHAFSR_IVC_msg          },
734         {       CHAFSR_IVU,     CHAFSR_IVU_msg          },
735         {       0,              NULL                    },
736 };
737 static const char CHPAFSR_DTO_msg[] =
738         "System bus unmapped error for prefetch/storequeue-read";
739 static const char CHPAFSR_DBERR_msg[] =
740         "System bus error for prefetch/storequeue-read";
741 static const char CHPAFSR_THCE_msg[] =
742         "Hardware corrected E-cache Tag ECC error";
743 static const char CHPAFSR_TSCE_msg[] =
744         "SW handled correctable E-cache Tag ECC error";
745 static const char CHPAFSR_TUE_msg[] =
746         "Uncorrectable E-cache Tag ECC error";
747 static const char CHPAFSR_DUE_msg[] =
748         "System bus uncorrectable data ECC error due to prefetch/store-fill";
749 static struct afsr_error_table __cheetah_plus_error_table[] = {
750         {       CHAFSR_PERR,    CHAFSR_PERR_msg         },
751         {       CHAFSR_IERR,    CHAFSR_IERR_msg         },
752         {       CHAFSR_ISAP,    CHAFSR_ISAP_msg         },
753         {       CHAFSR_UCU,     CHAFSR_UCU_msg          },
754         {       CHAFSR_UCC,     CHAFSR_UCC_msg          },
755         {       CHAFSR_UE,      CHAFSR_UE_msg           },
756         {       CHAFSR_EDU,     CHAFSR_EDU_msg          },
757         {       CHAFSR_EMU,     CHAFSR_EMU_msg          },
758         {       CHAFSR_WDU,     CHAFSR_WDU_msg          },
759         {       CHAFSR_CPU,     CHAFSR_CPU_msg          },
760         {       CHAFSR_CE,      CHAFSR_CE_msg           },
761         {       CHAFSR_EDC,     CHAFSR_EDC_msg          },
762         {       CHAFSR_EMC,     CHAFSR_EMC_msg          },
763         {       CHAFSR_WDC,     CHAFSR_WDC_msg          },
764         {       CHAFSR_CPC,     CHAFSR_CPC_msg          },
765         {       CHAFSR_TO,      CHAFSR_TO_msg           },
766         {       CHAFSR_BERR,    CHAFSR_BERR_msg         },
767         {       CHPAFSR_DTO,    CHPAFSR_DTO_msg         },
768         {       CHPAFSR_DBERR,  CHPAFSR_DBERR_msg       },
769         {       CHPAFSR_THCE,   CHPAFSR_THCE_msg        },
770         {       CHPAFSR_TSCE,   CHPAFSR_TSCE_msg        },
771         {       CHPAFSR_TUE,    CHPAFSR_TUE_msg         },
772         {       CHPAFSR_DUE,    CHPAFSR_DUE_msg         },
773         /* These two do not update the AFAR. */
774         {       CHAFSR_IVC,     CHAFSR_IVC_msg          },
775         {       CHAFSR_IVU,     CHAFSR_IVU_msg          },
776         {       0,              NULL                    },
777 };
778 static const char JPAFSR_JETO_msg[] =
779         "System interface protocol error, hw timeout caused";
780 static const char JPAFSR_SCE_msg[] =
781         "Parity error on system snoop results";
782 static const char JPAFSR_JEIC_msg[] =
783         "System interface protocol error, illegal command detected";
784 static const char JPAFSR_JEIT_msg[] =
785         "System interface protocol error, illegal ADTYPE detected";
786 static const char JPAFSR_OM_msg[] =
787         "Out of range memory error has occurred";
788 static const char JPAFSR_ETP_msg[] =
789         "Parity error on L2 cache tag SRAM";
790 static const char JPAFSR_UMS_msg[] =
791         "Error due to unsupported store";
792 static const char JPAFSR_RUE_msg[] =
793         "Uncorrectable ECC error from remote cache/memory";
794 static const char JPAFSR_RCE_msg[] =
795         "Correctable ECC error from remote cache/memory";
796 static const char JPAFSR_BP_msg[] =
797         "JBUS parity error on returned read data";
798 static const char JPAFSR_WBP_msg[] =
799         "JBUS parity error on data for writeback or block store";
800 static const char JPAFSR_FRC_msg[] =
801         "Foreign read to DRAM incurring correctable ECC error";
802 static const char JPAFSR_FRU_msg[] =
803         "Foreign read to DRAM incurring uncorrectable ECC error";
804 static struct afsr_error_table __jalapeno_error_table[] = {
805         {       JPAFSR_JETO,    JPAFSR_JETO_msg         },
806         {       JPAFSR_SCE,     JPAFSR_SCE_msg          },
807         {       JPAFSR_JEIC,    JPAFSR_JEIC_msg         },
808         {       JPAFSR_JEIT,    JPAFSR_JEIT_msg         },
809         {       CHAFSR_PERR,    CHAFSR_PERR_msg         },
810         {       CHAFSR_IERR,    CHAFSR_IERR_msg         },
811         {       CHAFSR_ISAP,    CHAFSR_ISAP_msg         },
812         {       CHAFSR_UCU,     CHAFSR_UCU_msg          },
813         {       CHAFSR_UCC,     CHAFSR_UCC_msg          },
814         {       CHAFSR_UE,      CHAFSR_UE_msg           },
815         {       CHAFSR_EDU,     CHAFSR_EDU_msg          },
816         {       JPAFSR_OM,      JPAFSR_OM_msg           },
817         {       CHAFSR_WDU,     CHAFSR_WDU_msg          },
818         {       CHAFSR_CPU,     CHAFSR_CPU_msg          },
819         {       CHAFSR_CE,      CHAFSR_CE_msg           },
820         {       CHAFSR_EDC,     CHAFSR_EDC_msg          },
821         {       JPAFSR_ETP,     JPAFSR_ETP_msg          },
822         {       CHAFSR_WDC,     CHAFSR_WDC_msg          },
823         {       CHAFSR_CPC,     CHAFSR_CPC_msg          },
824         {       CHAFSR_TO,      CHAFSR_TO_msg           },
825         {       CHAFSR_BERR,    CHAFSR_BERR_msg         },
826         {       JPAFSR_UMS,     JPAFSR_UMS_msg          },
827         {       JPAFSR_RUE,     JPAFSR_RUE_msg          },
828         {       JPAFSR_RCE,     JPAFSR_RCE_msg          },
829         {       JPAFSR_BP,      JPAFSR_BP_msg           },
830         {       JPAFSR_WBP,     JPAFSR_WBP_msg          },
831         {       JPAFSR_FRC,     JPAFSR_FRC_msg          },
832         {       JPAFSR_FRU,     JPAFSR_FRU_msg          },
833         /* These two do not update the AFAR. */
834         {       CHAFSR_IVU,     CHAFSR_IVU_msg          },
835         {       0,              NULL                    },
836 };
837 static struct afsr_error_table *cheetah_error_table;
838 static unsigned long cheetah_afsr_errors;
839 
840 struct cheetah_err_info *cheetah_error_log;
841 
842 static inline struct cheetah_err_info *cheetah_get_error_log(unsigned long afsr)
843 {
844         struct cheetah_err_info *p;
845         int cpu = smp_processor_id();
846 
847         if (!cheetah_error_log)
848                 return NULL;
849 
850         p = cheetah_error_log + (cpu * 2);
851         if ((afsr & CHAFSR_TL1) != 0UL)
852                 p++;
853 
854         return p;
855 }
856 
857 extern unsigned int tl0_icpe[], tl1_icpe[];
858 extern unsigned int tl0_dcpe[], tl1_dcpe[];
859 extern unsigned int tl0_fecc[], tl1_fecc[];
860 extern unsigned int tl0_cee[], tl1_cee[];
861 extern unsigned int tl0_iae[], tl1_iae[];
862 extern unsigned int tl0_dae[], tl1_dae[];
863 extern unsigned int cheetah_plus_icpe_trap_vector[], cheetah_plus_icpe_trap_vector_tl1[];
864 extern unsigned int cheetah_plus_dcpe_trap_vector[], cheetah_plus_dcpe_trap_vector_tl1[];
865 extern unsigned int cheetah_fecc_trap_vector[], cheetah_fecc_trap_vector_tl1[];
866 extern unsigned int cheetah_cee_trap_vector[], cheetah_cee_trap_vector_tl1[];
867 extern unsigned int cheetah_deferred_trap_vector[], cheetah_deferred_trap_vector_tl1[];
868 
869 void __init cheetah_ecache_flush_init(void)
870 {
871         unsigned long largest_size, smallest_linesize, order, ver;
872         int i, sz;
873 
874         /* Scan all cpu device tree nodes, note two values:
875          * 1) largest E-cache size
876          * 2) smallest E-cache line size
877          */
878         largest_size = 0UL;
879         smallest_linesize = ~0UL;
880 
881         for (i = 0; i < NR_CPUS; i++) {
882                 unsigned long val;
883 
884                 val = cpu_data(i).ecache_size;
885                 if (!val)
886                         continue;
887 
888                 if (val > largest_size)
889                         largest_size = val;
890 
891                 val = cpu_data(i).ecache_line_size;
892                 if (val < smallest_linesize)
893                         smallest_linesize = val;
894 
895         }
896 
897         if (largest_size == 0UL || smallest_linesize == ~0UL) {
898                 prom_printf("cheetah_ecache_flush_init: Cannot probe cpu E-cache "
899                             "parameters.\n");
900                 prom_halt();
901         }
902 
903         ecache_flush_size = (2 * largest_size);
904         ecache_flush_linesize = smallest_linesize;
905 
906         ecache_flush_physbase = find_ecache_flush_span(ecache_flush_size);
907 
908         if (ecache_flush_physbase == ~0UL) {
909                 prom_printf("cheetah_ecache_flush_init: Cannot find %ld byte "
910                             "contiguous physical memory.\n",
911                             ecache_flush_size);
912                 prom_halt();
913         }
914 
915         /* Now allocate error trap reporting scoreboard. */
916         sz = NR_CPUS * (2 * sizeof(struct cheetah_err_info));
917         for (order = 0; order < MAX_ORDER; order++) {
918                 if ((PAGE_SIZE << order) >= sz)
919                         break;
920         }
921         cheetah_error_log = (struct cheetah_err_info *)
922                 __get_free_pages(GFP_KERNEL, order);
923         if (!cheetah_error_log) {
924                 prom_printf("cheetah_ecache_flush_init: Failed to allocate "
925                             "error logging scoreboard (%d bytes).\n", sz);
926                 prom_halt();
927         }
928         memset(cheetah_error_log, 0, PAGE_SIZE << order);
929 
930         /* Mark all AFSRs as invalid so that the trap handler will
931          * log new new information there.
932          */
933         for (i = 0; i < 2 * NR_CPUS; i++)
934                 cheetah_error_log[i].afsr = CHAFSR_INVALID;
935 
936         __asm__ ("rdpr %%ver, %0" : "=r" (ver));
937         if ((ver >> 32) == __JALAPENO_ID ||
938             (ver >> 32) == __SERRANO_ID) {
939                 cheetah_error_table = &__jalapeno_error_table[0];
940                 cheetah_afsr_errors = JPAFSR_ERRORS;
941         } else if ((ver >> 32) == 0x003e0015) {
942                 cheetah_error_table = &__cheetah_plus_error_table[0];
943                 cheetah_afsr_errors = CHPAFSR_ERRORS;
944         } else {
945                 cheetah_error_table = &__cheetah_error_table[0];
946                 cheetah_afsr_errors = CHAFSR_ERRORS;
947         }
948 
949         /* Now patch trap tables. */
950         memcpy(tl0_fecc, cheetah_fecc_trap_vector, (8 * 4));
951         memcpy(tl1_fecc, cheetah_fecc_trap_vector_tl1, (8 * 4));
952         memcpy(tl0_cee, cheetah_cee_trap_vector, (8 * 4));
953         memcpy(tl1_cee, cheetah_cee_trap_vector_tl1, (8 * 4));
954         memcpy(tl0_iae, cheetah_deferred_trap_vector, (8 * 4));
955         memcpy(tl1_iae, cheetah_deferred_trap_vector_tl1, (8 * 4));
956         memcpy(tl0_dae, cheetah_deferred_trap_vector, (8 * 4));
957         memcpy(tl1_dae, cheetah_deferred_trap_vector_tl1, (8 * 4));
958         if (tlb_type == cheetah_plus) {
959                 memcpy(tl0_dcpe, cheetah_plus_dcpe_trap_vector, (8 * 4));
960                 memcpy(tl1_dcpe, cheetah_plus_dcpe_trap_vector_tl1, (8 * 4));
961                 memcpy(tl0_icpe, cheetah_plus_icpe_trap_vector, (8 * 4));
962                 memcpy(tl1_icpe, cheetah_plus_icpe_trap_vector_tl1, (8 * 4));
963         }
964         flushi(PAGE_OFFSET);
965 }
966 
967 static void cheetah_flush_ecache(void)
968 {
969         unsigned long flush_base = ecache_flush_physbase;
970         unsigned long flush_linesize = ecache_flush_linesize;
971         unsigned long flush_size = ecache_flush_size;
972 
973         __asm__ __volatile__("1: subcc  %0, %4, %0\n\t"
974                              "   bne,pt %%xcc, 1b\n\t"
975                              "    ldxa  [%2 + %0] %3, %%g0\n\t"
976                              : "=&r" (flush_size)
977                              : "" (flush_size), "r" (flush_base),
978                                "i" (ASI_PHYS_USE_EC), "r" (flush_linesize));
979 }
980 
981 static void cheetah_flush_ecache_line(unsigned long physaddr)
982 {
983         unsigned long alias;
984 
985         physaddr &= ~(8UL - 1UL);
986         physaddr = (ecache_flush_physbase +
987                     (physaddr & ((ecache_flush_size>>1UL) - 1UL)));
988         alias = physaddr + (ecache_flush_size >> 1UL);
989         __asm__ __volatile__("ldxa [%0] %2, %%g0\n\t"
990                              "ldxa [%1] %2, %%g0\n\t"
991                              "membar #Sync"
992                              : /* no outputs */
993                              : "r" (physaddr), "r" (alias),
994                                "i" (ASI_PHYS_USE_EC));
995 }
996 
997 /* Unfortunately, the diagnostic access to the I-cache tags we need to
998  * use to clear the thing interferes with I-cache coherency transactions.
999  *
1000  * So we must only flush the I-cache when it is disabled.
1001  */
1002 static void __cheetah_flush_icache(void)
1003 {
1004         unsigned int icache_size, icache_line_size;
1005         unsigned long addr;
1006 
1007         icache_size = local_cpu_data().icache_size;
1008         icache_line_size = local_cpu_data().icache_line_size;
1009 
1010         /* Clear the valid bits in all the tags. */
1011         for (addr = 0; addr < icache_size; addr += icache_line_size) {
1012                 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
1013                                      "membar #Sync"
1014                                      : /* no outputs */
1015                                      : "r" (addr | (2 << 3)),
1016                                        "i" (ASI_IC_TAG));
1017         }
1018 }
1019 
1020 static void cheetah_flush_icache(void)
1021 {
1022         unsigned long dcu_save;
1023 
1024         /* Save current DCU, disable I-cache. */
1025         __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
1026                              "or %0, %2, %%g1\n\t"
1027                              "stxa %%g1, [%%g0] %1\n\t"
1028                              "membar #Sync"
1029                              : "=r" (dcu_save)
1030                              : "i" (ASI_DCU_CONTROL_REG), "i" (DCU_IC)
1031                              : "g1");
1032 
1033         __cheetah_flush_icache();
1034 
1035         /* Restore DCU register */
1036         __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
1037                              "membar #Sync"
1038                              : /* no outputs */
1039                              : "r" (dcu_save), "i" (ASI_DCU_CONTROL_REG));
1040 }
1041 
1042 static void cheetah_flush_dcache(void)
1043 {
1044         unsigned int dcache_size, dcache_line_size;
1045         unsigned long addr;
1046 
1047         dcache_size = local_cpu_data().dcache_size;
1048         dcache_line_size = local_cpu_data().dcache_line_size;
1049 
1050         for (addr = 0; addr < dcache_size; addr += dcache_line_size) {
1051                 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
1052                                      "membar #Sync"
1053                                      : /* no outputs */
1054                                      : "r" (addr), "i" (ASI_DCACHE_TAG));
1055         }
1056 }
1057 
1058 /* In order to make the even parity correct we must do two things.
1059  * First, we clear DC_data_parity and set DC_utag to an appropriate value.
1060  * Next, we clear out all 32-bytes of data for that line.  Data of
1061  * all-zero + tag parity value of zero == correct parity.
1062  */
1063 static void cheetah_plus_zap_dcache_parity(void)
1064 {
1065         unsigned int dcache_size, dcache_line_size;
1066         unsigned long addr;
1067 
1068         dcache_size = local_cpu_data().dcache_size;
1069         dcache_line_size = local_cpu_data().dcache_line_size;
1070 
1071         for (addr = 0; addr < dcache_size; addr += dcache_line_size) {
1072                 unsigned long tag = (addr >> 14);
1073                 unsigned long line;
1074 
1075                 __asm__ __volatile__("membar    #Sync\n\t"
1076                                      "stxa      %0, [%1] %2\n\t"
1077                                      "membar    #Sync"
1078                                      : /* no outputs */
1079                                      : "r" (tag), "r" (addr),
1080                                        "i" (ASI_DCACHE_UTAG));
1081                 for (line = addr; line < addr + dcache_line_size; line += 8)
1082                         __asm__ __volatile__("membar    #Sync\n\t"
1083                                              "stxa      %%g0, [%0] %1\n\t"
1084                                              "membar    #Sync"
1085                                              : /* no outputs */
1086                                              : "r" (line),
1087                                                "i" (ASI_DCACHE_DATA));
1088         }
1089 }
1090 
1091 /* Conversion tables used to frob Cheetah AFSR syndrome values into
1092  * something palatable to the memory controller driver get_unumber
1093  * routine.
1094  */
1095 #define MT0     137
1096 #define MT1     138
1097 #define MT2     139
1098 #define NONE    254
1099 #define MTC0    140
1100 #define MTC1    141
1101 #define MTC2    142
1102 #define MTC3    143
1103 #define C0      128
1104 #define C1      129
1105 #define C2      130
1106 #define C3      131
1107 #define C4      132
1108 #define C5      133
1109 #define C6      134
1110 #define C7      135
1111 #define C8      136
1112 #define M2      144
1113 #define M3      145
1114 #define M4      146
1115 #define M       147
1116 static unsigned char cheetah_ecc_syntab[] = {
1117 /*00*/NONE, C0, C1, M2, C2, M2, M3, 47, C3, M2, M2, 53, M2, 41, 29, M,
1118 /*01*/C4, M, M, 50, M2, 38, 25, M2, M2, 33, 24, M2, 11, M, M2, 16,
1119 /*02*/C5, M, M, 46, M2, 37, 19, M2, M, 31, 32, M, 7, M2, M2, 10,
1120 /*03*/M2, 40, 13, M2, 59, M, M2, 66, M, M2, M2, 0, M2, 67, 71, M,
1121 /*04*/C6, M, M, 43, M, 36, 18, M, M2, 49, 15, M, 63, M2, M2, 6,
1122 /*05*/M2, 44, 28, M2, M, M2, M2, 52, 68, M2, M2, 62, M2, M3, M3, M4,
1123 /*06*/M2, 26, 106, M2, 64, M, M2, 2, 120, M, M2, M3, M, M3, M3, M4,
1124 /*07*/116, M2, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3,
1125 /*08*/C7, M2, M, 42, M, 35, 17, M2, M, 45, 14, M2, 21, M2, M2, 5,
1126 /*09*/M, 27, M, M, 99, M, M, 3, 114, M2, M2, 20, M2, M3, M3, M,
1127 /*0a*/M2, 23, 113, M2, 112, M2, M, 51, 95, M, M2, M3, M2, M3, M3, M2,
1128 /*0b*/103, M, M2, M3, M2, M3, M3, M4, M2, 48, M, M, 73, M2, M, M3,
1129 /*0c*/M2, 22, 110, M2, 109, M2, M, 9, 108, M2, M, M3, M2, M3, M3, M,
1130 /*0d*/102, M2, M, M, M2, M3, M3, M, M2, M3, M3, M2, M, M4, M, M3,
1131 /*0e*/98, M, M2, M3, M2, M, M3, M4, M2, M3, M3, M4, M3, M, M, M,
1132 /*0f*/M2, M3, M3, M, M3, M, M, M, 56, M4, M, M3, M4, M, M, M,
1133 /*10*/C8, M, M2, 39, M, 34, 105, M2, M, 30, 104, M, 101, M, M, 4,
1134 /*11*/M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, M2, M, M3, M,
1135 /*12*/M2, 97, 82, M2, 78, M2, M2, 1, 96, M, M, M, M, M, M3, M2,
1136 /*13*/94, M, M2, M3, M2, M, M3, M, M2, M, 79, M, 69, M, M4, M,
1137 /*14*/M2, 93, 92, M, 91, M, M2, 8, 90, M2, M2, M, M, M, M, M4,
1138 /*15*/89, M, M, M3, M2, M3, M3, M, M, M, M3, M2, M3, M2, M, M3,
1139 /*16*/86, M, M2, M3, M2, M, M3, M, M2, M, M3, M, M3, M, M, M3,
1140 /*17*/M, M, M3, M2, M3, M2, M4, M, 60, M, M2, M3, M4, M, M, M2,
1141 /*18*/M2, 88, 85, M2, 84, M, M2, 55, 81, M2, M2, M3, M2, M3, M3, M4,
1142 /*19*/77, M, M, M, M2, M3, M, M, M2, M3, M3, M4, M3, M2, M, M,
1143 /*1a*/74, M, M2, M3, M, M, M3, M, M, M, M3, M, M3, M, M4, M3,
1144 /*1b*/M2, 70, 107, M4, 65, M2, M2, M, 127, M, M, M, M2, M3, M3, M,
1145 /*1c*/80, M2, M2, 72, M, 119, 118, M, M2, 126, 76, M, 125, M, M4, M3,
1146 /*1d*/M2, 115, 124, M, 75, M, M, M3, 61, M, M4, M, M4, M, M, M,
1147 /*1e*/M, 123, 122, M4, 121, M4, M, M3, 117, M2, M2, M3, M4, M3, M, M,
1148 /*1f*/111, M, M, M, M4, M3, M3, M, M, M, M3, M, M3, M2, M, M
1149 };
1150 static unsigned char cheetah_mtag_syntab[] = {
1151        NONE, MTC0,
1152        MTC1, NONE,
1153        MTC2, NONE,
1154        NONE, MT0,
1155        MTC3, NONE,
1156        NONE, MT1,
1157        NONE, MT2,
1158        NONE, NONE
1159 };
1160 
1161 /* Return the highest priority error conditon mentioned. */
1162 static inline unsigned long cheetah_get_hipri(unsigned long afsr)
1163 {
1164         unsigned long tmp = 0;
1165         int i;
1166 
1167         for (i = 0; cheetah_error_table[i].mask; i++) {
1168                 if ((tmp = (afsr & cheetah_error_table[i].mask)) != 0UL)
1169                         return tmp;
1170         }
1171         return tmp;
1172 }
1173 
1174 static const char *cheetah_get_string(unsigned long bit)
1175 {
1176         int i;
1177 
1178         for (i = 0; cheetah_error_table[i].mask; i++) {
1179                 if ((bit & cheetah_error_table[i].mask) != 0UL)
1180                         return cheetah_error_table[i].name;
1181         }
1182         return "???";
1183 }
1184 
1185 static void cheetah_log_errors(struct pt_regs *regs, struct cheetah_err_info *info,
1186                                unsigned long afsr, unsigned long afar, int recoverable)
1187 {
1188         unsigned long hipri;
1189         char unum[256];
1190 
1191         printk("%s" "ERROR(%d): Cheetah error trap taken afsr[%016lx] afar[%016lx] TL1(%d)\n",
1192                (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1193                afsr, afar,
1194                (afsr & CHAFSR_TL1) ? 1 : 0);
1195         printk("%s" "ERROR(%d): TPC[%lx] TNPC[%lx] O7[%lx] TSTATE[%lx]\n",
1196                (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1197                regs->tpc, regs->tnpc, regs->u_regs[UREG_I7], regs->tstate);
1198         printk("%s" "ERROR(%d): ",
1199                (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id());
1200         printk("TPC<%pS>\n", (void *) regs->tpc);
1201         printk("%s" "ERROR(%d): M_SYND(%lx),  E_SYND(%lx)%s%s\n",
1202                (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1203                (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT,
1204                (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT,
1205                (afsr & CHAFSR_ME) ? ", Multiple Errors" : "",
1206                (afsr & CHAFSR_PRIV) ? ", Privileged" : "");
1207         hipri = cheetah_get_hipri(afsr);
1208         printk("%s" "ERROR(%d): Highest priority error (%016lx) \"%s\"\n",
1209                (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1210                hipri, cheetah_get_string(hipri));
1211 
1212         /* Try to get unumber if relevant. */
1213 #define ESYND_ERRORS    (CHAFSR_IVC | CHAFSR_IVU | \
1214                          CHAFSR_CPC | CHAFSR_CPU | \
1215                          CHAFSR_UE  | CHAFSR_CE  | \
1216                          CHAFSR_EDC | CHAFSR_EDU  | \
1217                          CHAFSR_UCC | CHAFSR_UCU  | \
1218                          CHAFSR_WDU | CHAFSR_WDC)
1219 #define MSYND_ERRORS    (CHAFSR_EMC | CHAFSR_EMU)
1220         if (afsr & ESYND_ERRORS) {
1221                 int syndrome;
1222                 int ret;
1223 
1224                 syndrome = (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT;
1225                 syndrome = cheetah_ecc_syntab[syndrome];
1226                 ret = sprintf_dimm(syndrome, afar, unum, sizeof(unum));
1227                 if (ret != -1)
1228                         printk("%s" "ERROR(%d): AFAR E-syndrome [%s]\n",
1229                                (recoverable ? KERN_WARNING : KERN_CRIT),
1230                                smp_processor_id(), unum);
1231         } else if (afsr & MSYND_ERRORS) {
1232                 int syndrome;
1233                 int ret;
1234 
1235                 syndrome = (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT;
1236                 syndrome = cheetah_mtag_syntab[syndrome];
1237                 ret = sprintf_dimm(syndrome, afar, unum, sizeof(unum));
1238                 if (ret != -1)
1239                         printk("%s" "ERROR(%d): AFAR M-syndrome [%s]\n",
1240                                (recoverable ? KERN_WARNING : KERN_CRIT),
1241                                smp_processor_id(), unum);
1242         }
1243 
1244         /* Now dump the cache snapshots. */
1245         printk("%s" "ERROR(%d): D-cache idx[%x] tag[%016llx] utag[%016llx] stag[%016llx]\n",
1246                (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1247                (int) info->dcache_index,
1248                info->dcache_tag,
1249                info->dcache_utag,
1250                info->dcache_stag);
1251         printk("%s" "ERROR(%d): D-cache data0[%016llx] data1[%016llx] data2[%016llx] data3[%016llx]\n",
1252                (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1253                info->dcache_data[0],
1254                info->dcache_data[1],
1255                info->dcache_data[2],
1256                info->dcache_data[3]);
1257         printk("%s" "ERROR(%d): I-cache idx[%x] tag[%016llx] utag[%016llx] stag[%016llx] "
1258                "u[%016llx] l[%016llx]\n",
1259                (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1260                (int) info->icache_index,
1261                info->icache_tag,
1262                info->icache_utag,
1263                info->icache_stag,
1264                info->icache_upper,
1265                info->icache_lower);
1266         printk("%s" "ERROR(%d): I-cache INSN0[%016llx] INSN1[%016llx] INSN2[%016llx] INSN3[%016llx]\n",
1267                (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1268                info->icache_data[0],
1269                info->icache_data[1],
1270                info->icache_data[2],
1271                info->icache_data[3]);
1272         printk("%s" "ERROR(%d): I-cache INSN4[%016llx] INSN5[%016llx] INSN6[%016llx] INSN7[%016llx]\n",
1273                (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1274                info->icache_data[4],
1275                info->icache_data[5],
1276                info->icache_data[6],
1277                info->icache_data[7]);
1278         printk("%s" "ERROR(%d): E-cache idx[%x] tag[%016llx]\n",
1279                (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1280                (int) info->ecache_index, info->ecache_tag);
1281         printk("%s" "ERROR(%d): E-cache data0[%016llx] data1[%016llx] data2[%016llx] data3[%016llx]\n",
1282                (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1283                info->ecache_data[0],
1284                info->ecache_data[1],
1285                info->ecache_data[2],
1286                info->ecache_data[3]);
1287 
1288         afsr = (afsr & ~hipri) & cheetah_afsr_errors;
1289         while (afsr != 0UL) {
1290                 unsigned long bit = cheetah_get_hipri(afsr);
1291 
1292                 printk("%s" "ERROR: Multiple-error (%016lx) \"%s\"\n",
1293                        (recoverable ? KERN_WARNING : KERN_CRIT),
1294                        bit, cheetah_get_string(bit));
1295 
1296                 afsr &= ~bit;
1297         }
1298 
1299         if (!recoverable)
1300                 printk(KERN_CRIT "ERROR: This condition is not recoverable.\n");
1301 }
1302 
1303 static int cheetah_recheck_errors(struct cheetah_err_info *logp)
1304 {
1305         unsigned long afsr, afar;
1306         int ret = 0;
1307 
1308         __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
1309                              : "=r" (afsr)
1310                              : "i" (ASI_AFSR));
1311         if ((afsr & cheetah_afsr_errors) != 0) {
1312                 if (logp != NULL) {
1313                         __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
1314                                              : "=r" (afar)
1315                                              : "i" (ASI_AFAR));
1316                         logp->afsr = afsr;
1317                         logp->afar = afar;
1318                 }
1319                 ret = 1;
1320         }
1321         __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
1322                              "membar #Sync\n\t"
1323                              : : "r" (afsr), "i" (ASI_AFSR));
1324 
1325         return ret;
1326 }
1327 
1328 void cheetah_fecc_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1329 {
1330         struct cheetah_err_info local_snapshot, *p;
1331         int recoverable;
1332 
1333         /* Flush E-cache */
1334         cheetah_flush_ecache();
1335 
1336         p = cheetah_get_error_log(afsr);
1337         if (!p) {
1338                 prom_printf("ERROR: Early Fast-ECC error afsr[%016lx] afar[%016lx]\n",
1339                             afsr, afar);
1340                 prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1341                             smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1342                 prom_halt();
1343         }
1344 
1345         /* Grab snapshot of logged error. */
1346         memcpy(&local_snapshot, p, sizeof(local_snapshot));
1347 
1348         /* If the current trap snapshot does not match what the
1349          * trap handler passed along into our args, big trouble.
1350          * In such a case, mark the local copy as invalid.
1351          *
1352          * Else, it matches and we mark the afsr in the non-local
1353          * copy as invalid so we may log new error traps there.
1354          */
1355         if (p->afsr != afsr || p->afar != afar)
1356                 local_snapshot.afsr = CHAFSR_INVALID;
1357         else
1358                 p->afsr = CHAFSR_INVALID;
1359 
1360         cheetah_flush_icache();
1361         cheetah_flush_dcache();
1362 
1363         /* Re-enable I-cache/D-cache */
1364         __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1365                              "or %%g1, %1, %%g1\n\t"
1366                              "stxa %%g1, [%%g0] %0\n\t"
1367                              "membar #Sync"
1368                              : /* no outputs */
1369                              : "i" (ASI_DCU_CONTROL_REG),
1370                                "i" (DCU_DC | DCU_IC)
1371                              : "g1");
1372 
1373         /* Re-enable error reporting */
1374         __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1375                              "or %%g1, %1, %%g1\n\t"
1376                              "stxa %%g1, [%%g0] %0\n\t"
1377                              "membar #Sync"
1378                              : /* no outputs */
1379                              : "i" (ASI_ESTATE_ERROR_EN),
1380                                "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1381                              : "g1");
1382 
1383         /* Decide if we can continue after handling this trap and
1384          * logging the error.
1385          */
1386         recoverable = 1;
1387         if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1388                 recoverable = 0;
1389 
1390         /* Re-check AFSR/AFAR.  What we are looking for here is whether a new
1391          * error was logged while we had error reporting traps disabled.
1392          */
1393         if (cheetah_recheck_errors(&local_snapshot)) {
1394                 unsigned long new_afsr = local_snapshot.afsr;
1395 
1396                 /* If we got a new asynchronous error, die... */
1397                 if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU |
1398                                 CHAFSR_WDU | CHAFSR_CPU |
1399                                 CHAFSR_IVU | CHAFSR_UE |
1400                                 CHAFSR_BERR | CHAFSR_TO))
1401                         recoverable = 0;
1402         }
1403 
1404         /* Log errors. */
1405         cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1406 
1407         if (!recoverable)
1408                 panic("Irrecoverable Fast-ECC error trap.\n");
1409 
1410         /* Flush E-cache to kick the error trap handlers out. */
1411         cheetah_flush_ecache();
1412 }
1413 
1414 /* Try to fix a correctable error by pushing the line out from
1415  * the E-cache.  Recheck error reporting registers to see if the
1416  * problem is intermittent.
1417  */
1418 static int cheetah_fix_ce(unsigned long physaddr)
1419 {
1420         unsigned long orig_estate;
1421         unsigned long alias1, alias2;
1422         int ret;
1423 
1424         /* Make sure correctable error traps are disabled. */
1425         __asm__ __volatile__("ldxa      [%%g0] %2, %0\n\t"
1426                              "andn      %0, %1, %%g1\n\t"
1427                              "stxa      %%g1, [%%g0] %2\n\t"
1428                              "membar    #Sync"
1429                              : "=&r" (orig_estate)
1430                              : "i" (ESTATE_ERROR_CEEN),
1431                                "i" (ASI_ESTATE_ERROR_EN)
1432                              : "g1");
1433 
1434         /* We calculate alias addresses that will force the
1435          * cache line in question out of the E-cache.  Then
1436          * we bring it back in with an atomic instruction so
1437          * that we get it in some modified/exclusive state,
1438          * then we displace it again to try and get proper ECC
1439          * pushed back into the system.
1440          */
1441         physaddr &= ~(8UL - 1UL);
1442         alias1 = (ecache_flush_physbase +
1443                   (physaddr & ((ecache_flush_size >> 1) - 1)));
1444         alias2 = alias1 + (ecache_flush_size >> 1);
1445         __asm__ __volatile__("ldxa      [%0] %3, %%g0\n\t"
1446                              "ldxa      [%1] %3, %%g0\n\t"
1447                              "casxa     [%2] %3, %%g0, %%g0\n\t"
1448                              "ldxa      [%0] %3, %%g0\n\t"
1449                              "ldxa      [%1] %3, %%g0\n\t"
1450                              "membar    #Sync"
1451                              : /* no outputs */
1452                              : "r" (alias1), "r" (alias2),
1453                                "r" (physaddr), "i" (ASI_PHYS_USE_EC));
1454 
1455         /* Did that trigger another error? */
1456         if (cheetah_recheck_errors(NULL)) {
1457                 /* Try one more time. */
1458                 __asm__ __volatile__("ldxa [%0] %1, %%g0\n\t"
1459                                      "membar #Sync"
1460                                      : : "r" (physaddr), "i" (ASI_PHYS_USE_EC));
1461                 if (cheetah_recheck_errors(NULL))
1462                         ret = 2;
1463                 else
1464                         ret = 1;
1465         } else {
1466                 /* No new error, intermittent problem. */
1467                 ret = 0;
1468         }
1469 
1470         /* Restore error enables. */
1471         __asm__ __volatile__("stxa      %0, [%%g0] %1\n\t"
1472                              "membar    #Sync"
1473                              : : "r" (orig_estate), "i" (ASI_ESTATE_ERROR_EN));
1474 
1475         return ret;
1476 }
1477 
1478 /* Return non-zero if PADDR is a valid physical memory address. */
1479 static int cheetah_check_main_memory(unsigned long paddr)
1480 {
1481         unsigned long vaddr = PAGE_OFFSET + paddr;
1482 
1483         if (vaddr > (unsigned long) high_memory)
1484                 return 0;
1485 
1486         return kern_addr_valid(vaddr);
1487 }
1488 
1489 void cheetah_cee_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1490 {
1491         struct cheetah_err_info local_snapshot, *p;
1492         int recoverable, is_memory;
1493 
1494         p = cheetah_get_error_log(afsr);
1495         if (!p) {
1496                 prom_printf("ERROR: Early CEE error afsr[%016lx] afar[%016lx]\n",
1497                             afsr, afar);
1498                 prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1499                             smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1500                 prom_halt();
1501         }
1502 
1503         /* Grab snapshot of logged error. */
1504         memcpy(&local_snapshot, p, sizeof(local_snapshot));
1505 
1506         /* If the current trap snapshot does not match what the
1507          * trap handler passed along into our args, big trouble.
1508          * In such a case, mark the local copy as invalid.
1509          *
1510          * Else, it matches and we mark the afsr in the non-local
1511          * copy as invalid so we may log new error traps there.
1512          */
1513         if (p->afsr != afsr || p->afar != afar)
1514                 local_snapshot.afsr = CHAFSR_INVALID;
1515         else
1516                 p->afsr = CHAFSR_INVALID;
1517 
1518         is_memory = cheetah_check_main_memory(afar);
1519 
1520         if (is_memory && (afsr & CHAFSR_CE) != 0UL) {
1521                 /* XXX Might want to log the results of this operation
1522                  * XXX somewhere... -DaveM
1523                  */
1524                 cheetah_fix_ce(afar);
1525         }
1526 
1527         {
1528                 int flush_all, flush_line;
1529 
1530                 flush_all = flush_line = 0;
1531                 if ((afsr & CHAFSR_EDC) != 0UL) {
1532                         if ((afsr & cheetah_afsr_errors) == CHAFSR_EDC)
1533                                 flush_line = 1;
1534                         else
1535                                 flush_all = 1;
1536                 } else if ((afsr & CHAFSR_CPC) != 0UL) {
1537                         if ((afsr & cheetah_afsr_errors) == CHAFSR_CPC)
1538                                 flush_line = 1;
1539                         else
1540                                 flush_all = 1;
1541                 }
1542 
1543                 /* Trap handler only disabled I-cache, flush it. */
1544                 cheetah_flush_icache();
1545 
1546                 /* Re-enable I-cache */
1547                 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1548                                      "or %%g1, %1, %%g1\n\t"
1549                                      "stxa %%g1, [%%g0] %0\n\t"
1550                                      "membar #Sync"
1551                                      : /* no outputs */
1552                                      : "i" (ASI_DCU_CONTROL_REG),
1553                                      "i" (DCU_IC)
1554                                      : "g1");
1555 
1556                 if (flush_all)
1557                         cheetah_flush_ecache();
1558                 else if (flush_line)
1559                         cheetah_flush_ecache_line(afar);
1560         }
1561 
1562         /* Re-enable error reporting */
1563         __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1564                              "or %%g1, %1, %%g1\n\t"
1565                              "stxa %%g1, [%%g0] %0\n\t"
1566                              "membar #Sync"
1567                              : /* no outputs */
1568                              : "i" (ASI_ESTATE_ERROR_EN),
1569                                "i" (ESTATE_ERROR_CEEN)
1570                              : "g1");
1571 
1572         /* Decide if we can continue after handling this trap and
1573          * logging the error.
1574          */
1575         recoverable = 1;
1576         if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1577                 recoverable = 0;
1578 
1579         /* Re-check AFSR/AFAR */
1580         (void) cheetah_recheck_errors(&local_snapshot);
1581 
1582         /* Log errors. */
1583         cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1584 
1585         if (!recoverable)
1586                 panic("Irrecoverable Correctable-ECC error trap.\n");
1587 }
1588 
1589 void cheetah_deferred_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1590 {
1591         struct cheetah_err_info local_snapshot, *p;
1592         int recoverable, is_memory;
1593 
1594 #ifdef CONFIG_PCI
1595         /* Check for the special PCI poke sequence. */
1596         if (pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) {
1597                 cheetah_flush_icache();
1598                 cheetah_flush_dcache();
1599 
1600                 /* Re-enable I-cache/D-cache */
1601                 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1602                                      "or %%g1, %1, %%g1\n\t"
1603                                      "stxa %%g1, [%%g0] %0\n\t"
1604                                      "membar #Sync"
1605                                      : /* no outputs */
1606                                      : "i" (ASI_DCU_CONTROL_REG),
1607                                        "i" (DCU_DC | DCU_IC)
1608                                      : "g1");
1609 
1610                 /* Re-enable error reporting */
1611                 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1612                                      "or %%g1, %1, %%g1\n\t"
1613                                      "stxa %%g1, [%%g0] %0\n\t"
1614                                      "membar #Sync"
1615                                      : /* no outputs */
1616                                      : "i" (ASI_ESTATE_ERROR_EN),
1617                                        "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1618                                      : "g1");
1619 
1620                 (void) cheetah_recheck_errors(NULL);
1621 
1622                 pci_poke_faulted = 1;
1623                 regs->tpc += 4;
1624                 regs->tnpc = regs->tpc + 4;
1625                 return;
1626         }
1627 #endif
1628 
1629         p = cheetah_get_error_log(afsr);
1630         if (!p) {
1631                 prom_printf("ERROR: Early deferred error afsr[%016lx] afar[%016lx]\n",
1632                             afsr, afar);
1633                 prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1634                             smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1635                 prom_halt();
1636         }
1637 
1638         /* Grab snapshot of logged error. */
1639         memcpy(&local_snapshot, p, sizeof(local_snapshot));
1640 
1641         /* If the current trap snapshot does not match what the
1642          * trap handler passed along into our args, big trouble.
1643          * In such a case, mark the local copy as invalid.
1644          *
1645          * Else, it matches and we mark the afsr in the non-local
1646          * copy as invalid so we may log new error traps there.
1647          */
1648         if (p->afsr != afsr || p->afar != afar)
1649                 local_snapshot.afsr = CHAFSR_INVALID;
1650         else
1651                 p->afsr = CHAFSR_INVALID;
1652 
1653         is_memory = cheetah_check_main_memory(afar);
1654 
1655         {
1656                 int flush_all, flush_line;
1657 
1658                 flush_all = flush_line = 0;
1659                 if ((afsr & CHAFSR_EDU) != 0UL) {
1660                         if ((afsr & cheetah_afsr_errors) == CHAFSR_EDU)
1661                                 flush_line = 1;
1662                         else
1663                                 flush_all = 1;
1664                 } else if ((afsr & CHAFSR_BERR) != 0UL) {
1665                         if ((afsr & cheetah_afsr_errors) == CHAFSR_BERR)
1666                                 flush_line = 1;
1667                         else
1668                                 flush_all = 1;
1669                 }
1670 
1671                 cheetah_flush_icache();
1672                 cheetah_flush_dcache();
1673 
1674                 /* Re-enable I/D caches */
1675                 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1676                                      "or %%g1, %1, %%g1\n\t"
1677                                      "stxa %%g1, [%%g0] %0\n\t"
1678                                      "membar #Sync"
1679                                      : /* no outputs */
1680                                      : "i" (ASI_DCU_CONTROL_REG),
1681                                      "i" (DCU_IC | DCU_DC)
1682                                      : "g1");
1683 
1684                 if (flush_all)
1685                         cheetah_flush_ecache();
1686                 else if (flush_line)
1687                         cheetah_flush_ecache_line(afar);
1688         }
1689 
1690         /* Re-enable error reporting */
1691         __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1692                              "or %%g1, %1, %%g1\n\t"
1693                              "stxa %%g1, [%%g0] %0\n\t"
1694                              "membar #Sync"
1695                              : /* no outputs */
1696                              : "i" (ASI_ESTATE_ERROR_EN),
1697                              "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1698                              : "g1");
1699 
1700         /* Decide if we can continue after handling this trap and
1701          * logging the error.
1702          */
1703         recoverable = 1;
1704         if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1705                 recoverable = 0;
1706 
1707         /* Re-check AFSR/AFAR.  What we are looking for here is whether a new
1708          * error was logged while we had error reporting traps disabled.
1709          */
1710         if (cheetah_recheck_errors(&local_snapshot)) {
1711                 unsigned long new_afsr = local_snapshot.afsr;
1712 
1713                 /* If we got a new asynchronous error, die... */
1714                 if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU |
1715                                 CHAFSR_WDU | CHAFSR_CPU |
1716                                 CHAFSR_IVU | CHAFSR_UE |
1717                                 CHAFSR_BERR | CHAFSR_TO))
1718                         recoverable = 0;
1719         }
1720 
1721         /* Log errors. */
1722         cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1723 
1724         /* "Recoverable" here means we try to yank the page from ever
1725          * being newly used again.  This depends upon a few things:
1726          * 1) Must be main memory, and AFAR must be valid.
1727          * 2) If we trapped from user, OK.
1728          * 3) Else, if we trapped from kernel we must find exception
1729          *    table entry (ie. we have to have been accessing user
1730          *    space).
1731          *
1732          * If AFAR is not in main memory, or we trapped from kernel
1733          * and cannot find an exception table entry, it is unacceptable
1734          * to try and continue.
1735          */
1736         if (recoverable && is_memory) {
1737                 if ((regs->tstate & TSTATE_PRIV) == 0UL) {
1738                         /* OK, usermode access. */
1739                         recoverable = 1;
1740                 } else {
1741                         const struct exception_table_entry *entry;
1742 
1743                         entry = search_exception_tables(regs->tpc);
1744                         if (entry) {
1745                                 /* OK, kernel access to userspace. */
1746                                 recoverable = 1;
1747 
1748                         } else {
1749                                 /* BAD, privileged state is corrupted. */
1750                                 recoverable = 0;
1751                         }
1752 
1753                         if (recoverable) {
1754                                 if (pfn_valid(afar >> PAGE_SHIFT))
1755                                         get_page(pfn_to_page(afar >> PAGE_SHIFT));
1756                                 else
1757                                         recoverable = 0;
1758 
1759                                 /* Only perform fixup if we still have a
1760                                  * recoverable condition.
1761                                  */
1762                                 if (recoverable) {
1763                                         regs->tpc = entry->fixup;
1764                                         regs->tnpc = regs->tpc + 4;
1765                                 }
1766                         }
1767                 }
1768         } else {
1769                 recoverable = 0;
1770         }
1771 
1772         if (!recoverable)
1773                 panic("Irrecoverable deferred error trap.\n");
1774 }
1775 
1776 /* Handle a D/I cache parity error trap.  TYPE is encoded as:
1777  *
1778  * Bit0:        0=dcache,1=icache
1779  * Bit1:        0=recoverable,1=unrecoverable
1780  *
1781  * The hardware has disabled both the I-cache and D-cache in
1782  * the %dcr register.  
1783  */
1784 void cheetah_plus_parity_error(int type, struct pt_regs *regs)
1785 {
1786         if (type & 0x1)
1787                 __cheetah_flush_icache();
1788         else
1789                 cheetah_plus_zap_dcache_parity();
1790         cheetah_flush_dcache();
1791 
1792         /* Re-enable I-cache/D-cache */
1793         __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1794                              "or %%g1, %1, %%g1\n\t"
1795                              "stxa %%g1, [%%g0] %0\n\t"
1796                              "membar #Sync"
1797                              : /* no outputs */
1798                              : "i" (ASI_DCU_CONTROL_REG),
1799                                "i" (DCU_DC | DCU_IC)
1800                              : "g1");
1801 
1802         if (type & 0x2) {
1803                 printk(KERN_EMERG "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
1804                        smp_processor_id(),
1805                        (type & 0x1) ? 'I' : 'D',
1806                        regs->tpc);
1807                 printk(KERN_EMERG "TPC<%pS>\n", (void *) regs->tpc);
1808                 panic("Irrecoverable Cheetah+ parity error.");
1809         }
1810 
1811         printk(KERN_WARNING "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
1812                smp_processor_id(),
1813                (type & 0x1) ? 'I' : 'D',
1814                regs->tpc);
1815         printk(KERN_WARNING "TPC<%pS>\n", (void *) regs->tpc);
1816 }
1817 
1818 struct sun4v_error_entry {
1819         /* Unique error handle */
1820 /*0x00*/u64             err_handle;
1821 
1822         /* %stick value at the time of the error */
1823 /*0x08*/u64             err_stick;
1824 
1825 /*0x10*/u8              reserved_1[3];
1826 
1827         /* Error type */
1828 /*0x13*/u8              err_type;
1829 #define SUN4V_ERR_TYPE_UNDEFINED        0
1830 #define SUN4V_ERR_TYPE_UNCORRECTED_RES  1
1831 #define SUN4V_ERR_TYPE_PRECISE_NONRES   2
1832 #define SUN4V_ERR_TYPE_DEFERRED_NONRES  3
1833 #define SUN4V_ERR_TYPE_SHUTDOWN_RQST    4
1834 #define SUN4V_ERR_TYPE_DUMP_CORE        5
1835 #define SUN4V_ERR_TYPE_SP_STATE_CHANGE  6
1836 #define SUN4V_ERR_TYPE_NUM              7
1837 
1838         /* Error attributes */
1839 /*0x14*/u32             err_attrs;
1840 #define SUN4V_ERR_ATTRS_PROCESSOR       0x00000001
1841 #define SUN4V_ERR_ATTRS_MEMORY          0x00000002
1842 #define SUN4V_ERR_ATTRS_PIO             0x00000004
1843 #define SUN4V_ERR_ATTRS_INT_REGISTERS   0x00000008
1844 #define SUN4V_ERR_ATTRS_FPU_REGISTERS   0x00000010
1845 #define SUN4V_ERR_ATTRS_SHUTDOWN_RQST   0x00000020
1846 #define SUN4V_ERR_ATTRS_ASR             0x00000040
1847 #define SUN4V_ERR_ATTRS_ASI             0x00000080
1848 #define SUN4V_ERR_ATTRS_PRIV_REG        0x00000100
1849 #define SUN4V_ERR_ATTRS_SPSTATE_MSK     0x00000600
1850 #define SUN4V_ERR_ATTRS_SPSTATE_SHFT    9
1851 #define SUN4V_ERR_ATTRS_MODE_MSK        0x03000000
1852 #define SUN4V_ERR_ATTRS_MODE_SHFT       24
1853 #define SUN4V_ERR_ATTRS_RES_QUEUE_FULL  0x80000000
1854 
1855 #define SUN4V_ERR_SPSTATE_FAULTED       0
1856 #define SUN4V_ERR_SPSTATE_AVAILABLE     1
1857 #define SUN4V_ERR_SPSTATE_NOT_PRESENT   2
1858 
1859 #define SUN4V_ERR_MODE_USER             1
1860 #define SUN4V_ERR_MODE_PRIV             2
1861 
1862         /* Real address of the memory region or PIO transaction */
1863 /*0x18*/u64             err_raddr;
1864 
1865         /* Size of the operation triggering the error, in bytes */
1866 /*0x20*/u32             err_size;
1867 
1868         /* ID of the CPU */
1869 /*0x24*/u16             err_cpu;
1870 
1871         /* Grace periof for shutdown, in seconds */
1872 /*0x26*/u16             err_secs;
1873 
1874         /* Value of the %asi register */
1875 /*0x28*/u8              err_asi;
1876 
1877 /*0x29*/u8              reserved_2;
1878 
1879         /* Value of the ASR register number */
1880 /*0x2a*/u16             err_asr;
1881 #define SUN4V_ERR_ASR_VALID             0x8000
1882 
1883 /*0x2c*/u32             reserved_3;
1884 /*0x30*/u64             reserved_4;
1885 /*0x38*/u64             reserved_5;
1886 };
1887 
1888 static atomic_t sun4v_resum_oflow_cnt = ATOMIC_INIT(0);
1889 static atomic_t sun4v_nonresum_oflow_cnt = ATOMIC_INIT(0);
1890 
1891 static const char *sun4v_err_type_to_str(u8 type)
1892 {
1893         static const char *types[SUN4V_ERR_TYPE_NUM] = {
1894                 "undefined",
1895                 "uncorrected resumable",
1896                 "precise nonresumable",
1897                 "deferred nonresumable",
1898                 "shutdown request",
1899                 "dump core",
1900                 "SP state change",
1901         };
1902 
1903         if (type < SUN4V_ERR_TYPE_NUM)
1904                 return types[type];
1905 
1906         return "unknown";
1907 }
1908 
1909 static void sun4v_emit_err_attr_strings(u32 attrs)
1910 {
1911         static const char *attr_names[] = {
1912                 "processor",
1913                 "memory",
1914                 "PIO",
1915                 "int-registers",
1916                 "fpu-registers",
1917                 "shutdown-request",
1918                 "ASR",
1919                 "ASI",
1920                 "priv-reg",
1921         };
1922         static const char *sp_states[] = {
1923                 "sp-faulted",
1924                 "sp-available",
1925                 "sp-not-present",
1926                 "sp-state-reserved",
1927         };
1928         static const char *modes[] = {
1929                 "mode-reserved0",
1930                 "user",
1931                 "priv",
1932                 "mode-reserved1",
1933         };
1934         u32 sp_state, mode;
1935         int i;
1936 
1937         for (i = 0; i < ARRAY_SIZE(attr_names); i++) {
1938                 if (attrs & (1U << i)) {
1939                         const char *s = attr_names[i];
1940 
1941                         pr_cont("%s ", s);
1942                 }
1943         }
1944 
1945         sp_state = ((attrs & SUN4V_ERR_ATTRS_SPSTATE_MSK) >>
1946                     SUN4V_ERR_ATTRS_SPSTATE_SHFT);
1947         pr_cont("%s ", sp_states[sp_state]);
1948 
1949         mode = ((attrs & SUN4V_ERR_ATTRS_MODE_MSK) >>
1950                 SUN4V_ERR_ATTRS_MODE_SHFT);
1951         pr_cont("%s ", modes[mode]);
1952 
1953         if (attrs & SUN4V_ERR_ATTRS_RES_QUEUE_FULL)
1954                 pr_cont("res-queue-full ");
1955 }
1956 
1957 /* When the report contains a real-address of "-1" it means that the
1958  * hardware did not provide the address.  So we compute the effective
1959  * address of the load or store instruction at regs->tpc and report
1960  * that.  Usually when this happens it's a PIO and in such a case we
1961  * are using physical addresses with bypass ASIs anyways, so what we
1962  * report here is exactly what we want.
1963  */
1964 static void sun4v_report_real_raddr(const char *pfx, struct pt_regs *regs)
1965 {
1966         unsigned int insn;
1967         u64 addr;
1968 
1969         if (!(regs->tstate & TSTATE_PRIV))
1970                 return;
1971 
1972         insn = *(unsigned int *) regs->tpc;
1973 
1974         addr = compute_effective_address(regs, insn, 0);
1975 
1976         printk("%s: insn effective address [0x%016llx]\n",
1977                pfx, addr);
1978 }
1979 
1980 static void sun4v_log_error(struct pt_regs *regs, struct sun4v_error_entry *ent,
1981                             int cpu, const char *pfx, atomic_t *ocnt)
1982 {
1983         u64 *raw_ptr = (u64 *) ent;
1984         u32 attrs;
1985         int cnt;
1986 
1987         printk("%s: Reporting on cpu %d\n", pfx, cpu);
1988         printk("%s: TPC [0x%016lx] <%pS>\n",
1989                pfx, regs->tpc, (void *) regs->tpc);
1990 
1991         printk("%s: RAW [%016llx:%016llx:%016llx:%016llx\n",
1992                pfx, raw_ptr[0], raw_ptr[1], raw_ptr[2], raw_ptr[3]);
1993         printk("%s:      %016llx:%016llx:%016llx:%016llx]\n",
1994                pfx, raw_ptr[4], raw_ptr[5], raw_ptr[6], raw_ptr[7]);
1995 
1996         printk("%s: handle [0x%016llx] stick [0x%016llx]\n",
1997                pfx, ent->err_handle, ent->err_stick);
1998 
1999         printk("%s: type [%s]\n", pfx, sun4v_err_type_to_str(ent->err_type));
2000 
2001         attrs = ent->err_attrs;
2002         printk("%s: attrs [0x%08x] < ", pfx, attrs);
2003         sun4v_emit_err_attr_strings(attrs);
2004         pr_cont(">\n");
2005 
2006         /* Various fields in the error report are only valid if
2007          * certain attribute bits are set.
2008          */
2009         if (attrs & (SUN4V_ERR_ATTRS_MEMORY |
2010                      SUN4V_ERR_ATTRS_PIO |
2011                      SUN4V_ERR_ATTRS_ASI)) {
2012                 printk("%s: raddr [0x%016llx]\n", pfx, ent->err_raddr);
2013 
2014                 if (ent->err_raddr == ~(u64)0)
2015                         sun4v_report_real_raddr(pfx, regs);
2016         }
2017 
2018         if (attrs & (SUN4V_ERR_ATTRS_MEMORY | SUN4V_ERR_ATTRS_ASI))
2019                 printk("%s: size [0x%x]\n", pfx, ent->err_size);
2020 
2021         if (attrs & (SUN4V_ERR_ATTRS_PROCESSOR |
2022                      SUN4V_ERR_ATTRS_INT_REGISTERS |
2023                      SUN4V_ERR_ATTRS_FPU_REGISTERS |
2024                      SUN4V_ERR_ATTRS_PRIV_REG))
2025                 printk("%s: cpu[%u]\n", pfx, ent->err_cpu);
2026 
2027         if (attrs & SUN4V_ERR_ATTRS_ASI)
2028                 printk("%s: asi [0x%02x]\n", pfx, ent->err_asi);
2029 
2030         if ((attrs & (SUN4V_ERR_ATTRS_INT_REGISTERS |
2031                       SUN4V_ERR_ATTRS_FPU_REGISTERS |
2032                       SUN4V_ERR_ATTRS_PRIV_REG)) &&
2033             (ent->err_asr & SUN4V_ERR_ASR_VALID) != 0)
2034                 printk("%s: reg [0x%04x]\n",
2035                        pfx, ent->err_asr & ~SUN4V_ERR_ASR_VALID);
2036 
2037         show_regs(regs);
2038 
2039         if ((cnt = atomic_read(ocnt)) != 0) {
2040                 atomic_set(ocnt, 0);
2041                 wmb();
2042                 printk("%s: Queue overflowed %d times.\n",
2043                        pfx, cnt);
2044         }
2045 }
2046 
2047 /* We run with %pil set to PIL_NORMAL_MAX and PSTATE_IE enabled in %pstate.
2048  * Log the event and clear the first word of the entry.
2049  */
2050 void sun4v_resum_error(struct pt_regs *regs, unsigned long offset)
2051 {
2052         enum ctx_state prev_state = exception_enter();
2053         struct sun4v_error_entry *ent, local_copy;
2054         struct trap_per_cpu *tb;
2055         unsigned long paddr;
2056         int cpu;
2057 
2058         cpu = get_cpu();
2059 
2060         tb = &trap_block[cpu];
2061         paddr = tb->resum_kernel_buf_pa + offset;
2062         ent = __va(paddr);
2063 
2064         memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry));
2065 
2066         /* We have a local copy now, so release the entry.  */
2067         ent->err_handle = 0;
2068         wmb();
2069 
2070         put_cpu();
2071 
2072         if (local_copy.err_type == SUN4V_ERR_TYPE_SHUTDOWN_RQST) {
2073                 /* We should really take the seconds field of
2074                  * the error report and use it for the shutdown
2075                  * invocation, but for now do the same thing we
2076                  * do for a DS shutdown request.
2077                  */
2078                 pr_info("Shutdown request, %u seconds...\n",
2079                         local_copy.err_secs);
2080                 orderly_poweroff(true);
2081                 goto out;
2082         }
2083 
2084         sun4v_log_error(regs, &local_copy, cpu,
2085                         KERN_ERR "RESUMABLE ERROR",
2086                         &sun4v_resum_oflow_cnt);
2087 out:
2088         exception_exit(prev_state);
2089 }
2090 
2091 /* If we try to printk() we'll probably make matters worse, by trying
2092  * to retake locks this cpu already holds or causing more errors. So
2093  * just bump a counter, and we'll report these counter bumps above.
2094  */
2095 void sun4v_resum_overflow(struct pt_regs *regs)
2096 {
2097         atomic_inc(&sun4v_resum_oflow_cnt);
2098 }
2099 
2100 /* Given a set of registers, get the virtual addressi that was being accessed
2101  * by the faulting instructions at tpc.
2102  */
2103 static unsigned long sun4v_get_vaddr(struct pt_regs *regs)
2104 {
2105         unsigned int insn;
2106 
2107         if (!copy_from_user(&insn, (void __user *)regs->tpc, 4)) {
2108                 return compute_effective_address(regs, insn,
2109                                                  (insn >> 25) & 0x1f);
2110         }
2111         return 0;
2112 }
2113 
2114 /* Attempt to handle non-resumable errors generated from userspace.
2115  * Returns true if the signal was handled, false otherwise.
2116  */
2117 bool sun4v_nonresum_error_user_handled(struct pt_regs *regs,
2118                                   struct sun4v_error_entry *ent) {
2119 
2120         unsigned int attrs = ent->err_attrs;
2121 
2122         if (attrs & SUN4V_ERR_ATTRS_MEMORY) {
2123                 unsigned long addr = ent->err_raddr;
2124                 siginfo_t info;
2125 
2126                 if (addr == ~(u64)0) {
2127                         /* This seems highly unlikely to ever occur */
2128                         pr_emerg("SUN4V NON-RECOVERABLE ERROR: Memory error detected in unknown location!\n");
2129                 } else {
2130                         unsigned long page_cnt = DIV_ROUND_UP(ent->err_size,
2131                                                               PAGE_SIZE);
2132 
2133                         /* Break the unfortunate news. */
2134                         pr_emerg("SUN4V NON-RECOVERABLE ERROR: Memory failed at %016lX\n",
2135                                  addr);
2136                         pr_emerg("SUN4V NON-RECOVERABLE ERROR:   Claiming %lu ages.\n",
2137                                  page_cnt);
2138 
2139                         while (page_cnt-- > 0) {
2140                                 if (pfn_valid(addr >> PAGE_SHIFT))
2141                                         get_page(pfn_to_page(addr >> PAGE_SHIFT));
2142                                 addr += PAGE_SIZE;
2143                         }
2144                 }
2145                 info.si_signo = SIGKILL;
2146                 info.si_errno = 0;
2147                 info.si_trapno = 0;
2148                 force_sig_info(info.si_signo, &info, current);
2149 
2150                 return true;
2151         }
2152         if (attrs & SUN4V_ERR_ATTRS_PIO) {
2153                 siginfo_t info;
2154 
2155                 info.si_signo = SIGBUS;
2156                 info.si_code = BUS_ADRERR;
2157                 info.si_addr = (void __user *)sun4v_get_vaddr(regs);
2158                 force_sig_info(info.si_signo, &info, current);
2159 
2160                 return true;
2161         }
2162 
2163         /* Default to doing nothing */
2164         return false;
2165 }
2166 
2167 /* We run with %pil set to PIL_NORMAL_MAX and PSTATE_IE enabled in %pstate.
2168  * Log the event, clear the first word of the entry, and die.
2169  */
2170 void sun4v_nonresum_error(struct pt_regs *regs, unsigned long offset)
2171 {
2172         struct sun4v_error_entry *ent, local_copy;
2173         struct trap_per_cpu *tb;
2174         unsigned long paddr;
2175         int cpu;
2176 
2177         cpu = get_cpu();
2178 
2179         tb = &trap_block[cpu];
2180         paddr = tb->nonresum_kernel_buf_pa + offset;
2181         ent = __va(paddr);
2182 
2183         memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry));
2184 
2185         /* We have a local copy now, so release the entry.  */
2186         ent->err_handle = 0;
2187         wmb();
2188 
2189         put_cpu();
2190 
2191         if (!(regs->tstate & TSTATE_PRIV) &&
2192             sun4v_nonresum_error_user_handled(regs, &local_copy)) {
2193                 /* DON'T PANIC: This userspace error was handled. */
2194                 return;
2195         }
2196 
2197 #ifdef CONFIG_PCI
2198         /* Check for the special PCI poke sequence. */
2199         if (pci_poke_in_progress && pci_poke_cpu == cpu) {
2200                 pci_poke_faulted = 1;
2201                 regs->tpc += 4;
2202                 regs->tnpc = regs->tpc + 4;
2203                 return;
2204         }
2205 #endif
2206 
2207         sun4v_log_error(regs, &local_copy, cpu,
2208                         KERN_EMERG "NON-RESUMABLE ERROR",
2209                         &sun4v_nonresum_oflow_cnt);
2210 
2211         panic("Non-resumable error.");
2212 }
2213 
2214 /* If we try to printk() we'll probably make matters worse, by trying
2215  * to retake locks this cpu already holds or causing more errors. So
2216  * just bump a counter, and we'll report these counter bumps above.
2217  */
2218 void sun4v_nonresum_overflow(struct pt_regs *regs)
2219 {
2220         /* XXX Actually even this can make not that much sense.  Perhaps
2221          * XXX we should just pull the plug and panic directly from here?
2222          */
2223         atomic_inc(&sun4v_nonresum_oflow_cnt);
2224 }
2225 
2226 static void sun4v_tlb_error(struct pt_regs *regs)
2227 {
2228         die_if_kernel("TLB/TSB error", regs);
2229 }
2230 
2231 unsigned long sun4v_err_itlb_vaddr;
2232 unsigned long sun4v_err_itlb_ctx;
2233 unsigned long sun4v_err_itlb_pte;
2234 unsigned long sun4v_err_itlb_error;
2235 
2236 void sun4v_itlb_error_report(struct pt_regs *regs, int tl)
2237 {
2238         dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2239 
2240         printk(KERN_EMERG "SUN4V-ITLB: Error at TPC[%lx], tl %d\n",
2241                regs->tpc, tl);
2242         printk(KERN_EMERG "SUN4V-ITLB: TPC<%pS>\n", (void *) regs->tpc);
2243         printk(KERN_EMERG "SUN4V-ITLB: O7[%lx]\n", regs->u_regs[UREG_I7]);
2244         printk(KERN_EMERG "SUN4V-ITLB: O7<%pS>\n",
2245                (void *) regs->u_regs[UREG_I7]);
2246         printk(KERN_EMERG "SUN4V-ITLB: vaddr[%lx] ctx[%lx] "
2247                "pte[%lx] error[%lx]\n",
2248                sun4v_err_itlb_vaddr, sun4v_err_itlb_ctx,
2249                sun4v_err_itlb_pte, sun4v_err_itlb_error);
2250 
2251         sun4v_tlb_error(regs);
2252 }
2253 
2254 unsigned long sun4v_err_dtlb_vaddr;
2255 unsigned long sun4v_err_dtlb_ctx;
2256 unsigned long sun4v_err_dtlb_pte;
2257 unsigned long sun4v_err_dtlb_error;
2258 
2259 void sun4v_dtlb_error_report(struct pt_regs *regs, int tl)
2260 {
2261         dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2262 
2263         printk(KERN_EMERG "SUN4V-DTLB: Error at TPC[%lx], tl %d\n",
2264                regs->tpc, tl);
2265         printk(KERN_EMERG "SUN4V-DTLB: TPC<%pS>\n", (void *) regs->tpc);
2266         printk(KERN_EMERG "SUN4V-DTLB: O7[%lx]\n", regs->u_regs[UREG_I7]);
2267         printk(KERN_EMERG "SUN4V-DTLB: O7<%pS>\n",
2268                (void *) regs->u_regs[UREG_I7]);
2269         printk(KERN_EMERG "SUN4V-DTLB: vaddr[%lx] ctx[%lx] "
2270                "pte[%lx] error[%lx]\n",
2271                sun4v_err_dtlb_vaddr, sun4v_err_dtlb_ctx,
2272                sun4v_err_dtlb_pte, sun4v_err_dtlb_error);
2273 
2274         sun4v_tlb_error(regs);
2275 }
2276 
2277 void hypervisor_tlbop_error(unsigned long err, unsigned long op)
2278 {
2279         printk(KERN_CRIT "SUN4V: TLB hv call error %lu for op %lu\n",
2280                err, op);
2281 }
2282 
2283 void hypervisor_tlbop_error_xcall(unsigned long err, unsigned long op)
2284 {
2285         printk(KERN_CRIT "SUN4V: XCALL TLB hv call error %lu for op %lu\n",
2286                err, op);
2287 }
2288 
2289 static void do_fpe_common(struct pt_regs *regs)
2290 {
2291         if (regs->tstate & TSTATE_PRIV) {
2292                 regs->tpc = regs->tnpc;
2293                 regs->tnpc += 4;
2294         } else {
2295                 unsigned long fsr = current_thread_info()->xfsr[0];
2296                 siginfo_t info;
2297 
2298                 if (test_thread_flag(TIF_32BIT)) {
2299                         regs->tpc &= 0xffffffff;
2300                         regs->tnpc &= 0xffffffff;
2301                 }
2302                 info.si_signo = SIGFPE;
2303                 info.si_errno = 0;
2304                 info.si_addr = (void __user *)regs->tpc;
2305                 info.si_trapno = 0;
2306                 info.si_code = FPE_FIXME;
2307                 if ((fsr & 0x1c000) == (1 << 14)) {
2308                         if (fsr & 0x10)
2309                                 info.si_code = FPE_FLTINV;
2310                         else if (fsr & 0x08)
2311                                 info.si_code = FPE_FLTOVF;
2312                         else if (fsr & 0x04)
2313                                 info.si_code = FPE_FLTUND;
2314                         else if (fsr & 0x02)
2315                                 info.si_code = FPE_FLTDIV;
2316                         else if (fsr & 0x01)
2317                                 info.si_code = FPE_FLTRES;
2318                 }
2319                 force_sig_info(SIGFPE, &info, current);
2320         }
2321 }
2322 
2323 void do_fpieee(struct pt_regs *regs)
2324 {
2325         enum ctx_state prev_state = exception_enter();
2326 
2327         if (notify_die(DIE_TRAP, "fpu exception ieee", regs,
2328                        0, 0x24, SIGFPE) == NOTIFY_STOP)
2329                 goto out;
2330 
2331         do_fpe_common(regs);
2332 out:
2333         exception_exit(prev_state);
2334 }
2335 
2336 void do_fpother(struct pt_regs *regs)
2337 {
2338         enum ctx_state prev_state = exception_enter();
2339         struct fpustate *f = FPUSTATE;
2340         int ret = 0;
2341 
2342         if (notify_die(DIE_TRAP, "fpu exception other", regs,
2343                        0, 0x25, SIGFPE) == NOTIFY_STOP)
2344                 goto out;
2345 
2346         switch ((current_thread_info()->xfsr[0] & 0x1c000)) {
2347         case (2 << 14): /* unfinished_FPop */
2348         case (3 << 14): /* unimplemented_FPop */
2349                 ret = do_mathemu(regs, f, false);
2350                 break;
2351         }
2352         if (ret)
2353                 goto out;
2354         do_fpe_common(regs);
2355 out:
2356         exception_exit(prev_state);
2357 }
2358 
2359 void do_tof(struct pt_regs *regs)
2360 {
2361         enum ctx_state prev_state = exception_enter();
2362         siginfo_t info;
2363 
2364         if (notify_die(DIE_TRAP, "tagged arithmetic overflow", regs,
2365                        0, 0x26, SIGEMT) == NOTIFY_STOP)
2366                 goto out;
2367 
2368         if (regs->tstate & TSTATE_PRIV)
2369                 die_if_kernel("Penguin overflow trap from kernel mode", regs);
2370         if (test_thread_flag(TIF_32BIT)) {
2371                 regs->tpc &= 0xffffffff;
2372                 regs->tnpc &= 0xffffffff;
2373         }
2374         info.si_signo = SIGEMT;
2375         info.si_errno = 0;
2376         info.si_code = EMT_TAGOVF;
2377         info.si_addr = (void __user *)regs->tpc;
2378         info.si_trapno = 0;
2379         force_sig_info(SIGEMT, &info, current);
2380 out:
2381         exception_exit(prev_state);
2382 }
2383 
2384 void do_div0(struct pt_regs *regs)
2385 {
2386         enum ctx_state prev_state = exception_enter();
2387         siginfo_t info;
2388 
2389         if (notify_die(DIE_TRAP, "integer division by zero", regs,
2390                        0, 0x28, SIGFPE) == NOTIFY_STOP)
2391                 goto out;
2392 
2393         if (regs->tstate & TSTATE_PRIV)
2394                 die_if_kernel("TL0: Kernel divide by zero.", regs);
2395         if (test_thread_flag(TIF_32BIT)) {
2396                 regs->tpc &= 0xffffffff;
2397                 regs->tnpc &= 0xffffffff;
2398         }
2399         info.si_signo = SIGFPE;
2400         info.si_errno = 0;
2401         info.si_code = FPE_INTDIV;
2402         info.si_addr = (void __user *)regs->tpc;
2403         info.si_trapno = 0;
2404         force_sig_info(SIGFPE, &info, current);
2405 out:
2406         exception_exit(prev_state);
2407 }
2408 
2409 static void instruction_dump(unsigned int *pc)
2410 {
2411         int i;
2412 
2413         if ((((unsigned long) pc) & 3))
2414                 return;
2415 
2416         printk("Instruction DUMP:");
2417         for (i = -3; i < 6; i++)
2418                 printk("%c%08x%c",i?' ':'<',pc[i],i?' ':'>');
2419         printk("\n");
2420 }
2421 
2422 static void user_instruction_dump(unsigned int __user *pc)
2423 {
2424         int i;
2425         unsigned int buf[9];
2426         
2427         if ((((unsigned long) pc) & 3))
2428                 return;
2429                 
2430         if (copy_from_user(buf, pc - 3, sizeof(buf)))
2431                 return;
2432 
2433         printk("Instruction DUMP:");
2434         for (i = 0; i < 9; i++)
2435                 printk("%c%08x%c",i==3?' ':'<',buf[i],i==3?' ':'>');
2436         printk("\n");
2437 }
2438 
2439 void show_stack(struct task_struct *tsk, unsigned long *_ksp)
2440 {
2441         unsigned long fp, ksp;
2442         struct thread_info *tp;
2443         int count = 0;
2444 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2445         int graph = 0;
2446 #endif
2447 
2448         ksp = (unsigned long) _ksp;
2449         if (!tsk)
2450                 tsk = current;
2451         tp = task_thread_info(tsk);
2452         if (ksp == 0UL) {
2453                 if (tsk == current)
2454                         asm("mov %%fp, %0" : "=r" (ksp));
2455                 else
2456                         ksp = tp->ksp;
2457         }
2458         if (tp == current_thread_info())
2459                 flushw_all();
2460 
2461         fp = ksp + STACK_BIAS;
2462 
2463         printk("Call Trace:\n");
2464         do {
2465                 struct sparc_stackf *sf;
2466                 struct pt_regs *regs;
2467                 unsigned long pc;
2468 
2469                 if (!kstack_valid(tp, fp))
2470                         break;
2471                 sf = (struct sparc_stackf *) fp;
2472                 regs = (struct pt_regs *) (sf + 1);
2473 
2474                 if (kstack_is_trap_frame(tp, regs)) {
2475                         if (!(regs->tstate & TSTATE_PRIV))
2476                                 break;
2477                         pc = regs->tpc;
2478                         fp = regs->u_regs[UREG_I6] + STACK_BIAS;
2479                 } else {
2480                         pc = sf->callers_pc;
2481                         fp = (unsigned long)sf->fp + STACK_BIAS;
2482                 }
2483 
2484                 printk(" [%016lx] %pS\n", pc, (void *) pc);
2485 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2486                 if ((pc + 8UL) == (unsigned long) &return_to_handler) {
2487                         int index = tsk->curr_ret_stack;
2488                         if (tsk->ret_stack && index >= graph) {
2489                                 pc = tsk->ret_stack[index - graph].ret;
2490                                 printk(" [%016lx] %pS\n", pc, (void *) pc);
2491                                 graph++;
2492                         }
2493                 }
2494 #endif
2495         } while (++count < 16);
2496 }
2497 
2498 static inline struct reg_window *kernel_stack_up(struct reg_window *rw)
2499 {
2500         unsigned long fp = rw->ins[6];
2501 
2502         if (!fp)
2503                 return NULL;
2504 
2505         return (struct reg_window *) (fp + STACK_BIAS);
2506 }
2507 
2508 void __noreturn die_if_kernel(char *str, struct pt_regs *regs)
2509 {
2510         static int die_counter;
2511         int count = 0;
2512         
2513         /* Amuse the user. */
2514         printk(
2515 "              \\|/ ____ \\|/\n"
2516 "              \"@'/ .. \\`@\"\n"
2517 "              /_| \\__/ |_\\\n"
2518 "                 \\__U_/\n");
2519 
2520         printk("%s(%d): %s [#%d]\n", current->comm, task_pid_nr(current), str, ++die_counter);
2521         notify_die(DIE_OOPS, str, regs, 0, 255, SIGSEGV);
2522         __asm__ __volatile__("flushw");
2523         show_regs(regs);
2524         add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
2525         if (regs->tstate & TSTATE_PRIV) {
2526                 struct thread_info *tp = current_thread_info();
2527                 struct reg_window *rw = (struct reg_window *)
2528                         (regs->u_regs[UREG_FP] + STACK_BIAS);
2529 
2530                 /* Stop the back trace when we hit userland or we
2531                  * find some badly aligned kernel stack.
2532                  */
2533                 while (rw &&
2534                        count++ < 30 &&
2535                        kstack_valid(tp, (unsigned long) rw)) {
2536                         printk("Caller[%016lx]: %pS\n", rw->ins[7],
2537                                (void *) rw->ins[7]);
2538 
2539                         rw = kernel_stack_up(rw);
2540                 }
2541                 instruction_dump ((unsigned int *) regs->tpc);
2542         } else {
2543                 if (test_thread_flag(TIF_32BIT)) {
2544                         regs->tpc &= 0xffffffff;
2545                         regs->tnpc &= 0xffffffff;
2546                 }
2547                 user_instruction_dump ((unsigned int __user *) regs->tpc);
2548         }
2549         if (panic_on_oops)
2550                 panic("Fatal exception");
2551         if (regs->tstate & TSTATE_PRIV)
2552                 do_exit(SIGKILL);
2553         do_exit(SIGSEGV);
2554 }
2555 EXPORT_SYMBOL(die_if_kernel);
2556 
2557 #define VIS_OPCODE_MASK ((0x3 << 30) | (0x3f << 19))
2558 #define VIS_OPCODE_VAL  ((0x2 << 30) | (0x36 << 19))
2559 
2560 void do_illegal_instruction(struct pt_regs *regs)
2561 {
2562         enum ctx_state prev_state = exception_enter();
2563         unsigned long pc = regs->tpc;
2564         unsigned long tstate = regs->tstate;
2565         u32 insn;
2566         siginfo_t info;
2567 
2568         if (notify_die(DIE_TRAP, "illegal instruction", regs,
2569                        0, 0x10, SIGILL) == NOTIFY_STOP)
2570                 goto out;
2571 
2572         if (tstate & TSTATE_PRIV)
2573                 die_if_kernel("Kernel illegal instruction", regs);
2574         if (test_thread_flag(TIF_32BIT))
2575                 pc = (u32)pc;
2576         if (get_user(insn, (u32 __user *) pc) != -EFAULT) {
2577                 if ((insn & 0xc1ffc000) == 0x81700000) /* POPC */ {
2578                         if (handle_popc(insn, regs))
2579                                 goto out;
2580                 } else if ((insn & 0xc1580000) == 0xc1100000) /* LDQ/STQ */ {
2581                         if (handle_ldf_stq(insn, regs))
2582                                 goto out;
2583                 } else if (tlb_type == hypervisor) {
2584                         if ((insn & VIS_OPCODE_MASK) == VIS_OPCODE_VAL) {
2585                                 if (!vis_emul(regs, insn))
2586                                         goto out;
2587                         } else {
2588                                 struct fpustate *f = FPUSTATE;
2589 
2590                                 /* On UltraSPARC T2 and later, FPU insns which
2591                                  * are not implemented in HW signal an illegal
2592                                  * instruction trap and do not set the FP Trap
2593                                  * Trap in the %fsr to unimplemented_FPop.
2594                                  */
2595                                 if (do_mathemu(regs, f, true))
2596                                         goto out;
2597                         }
2598                 }
2599         }
2600         info.si_signo = SIGILL;
2601         info.si_errno = 0;
2602         info.si_code = ILL_ILLOPC;
2603         info.si_addr = (void __user *)pc;
2604         info.si_trapno = 0;
2605         force_sig_info(SIGILL, &info, current);
2606 out:
2607         exception_exit(prev_state);
2608 }
2609 
2610 void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr)
2611 {
2612         enum ctx_state prev_state = exception_enter();
2613         siginfo_t info;
2614 
2615         if (notify_die(DIE_TRAP, "memory address unaligned", regs,
2616                        0, 0x34, SIGSEGV) == NOTIFY_STOP)
2617                 goto out;
2618 
2619         if (regs->tstate & TSTATE_PRIV) {
2620                 kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc));
2621                 goto out;
2622         }
2623         if (is_no_fault_exception(regs))
2624                 return;
2625 
2626         info.si_signo = SIGBUS;
2627         info.si_errno = 0;
2628         info.si_code = BUS_ADRALN;
2629         info.si_addr = (void __user *)sfar;
2630         info.si_trapno = 0;
2631         force_sig_info(SIGBUS, &info, current);
2632 out:
2633         exception_exit(prev_state);
2634 }
2635 
2636 void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
2637 {
2638         siginfo_t info;
2639 
2640         if (notify_die(DIE_TRAP, "memory address unaligned", regs,
2641                        0, 0x34, SIGSEGV) == NOTIFY_STOP)
2642                 return;
2643 
2644         if (regs->tstate & TSTATE_PRIV) {
2645                 kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc));
2646                 return;
2647         }
2648         if (is_no_fault_exception(regs))
2649                 return;
2650 
2651         info.si_signo = SIGBUS;
2652         info.si_errno = 0;
2653         info.si_code = BUS_ADRALN;
2654         info.si_addr = (void __user *) addr;
2655         info.si_trapno = 0;
2656         force_sig_info(SIGBUS, &info, current);
2657 }
2658 
2659 void do_privop(struct pt_regs *regs)
2660 {
2661         enum ctx_state prev_state = exception_enter();
2662         siginfo_t info;
2663 
2664         if (notify_die(DIE_TRAP, "privileged operation", regs,
2665                        0, 0x11, SIGILL) == NOTIFY_STOP)
2666                 goto out;
2667 
2668         if (test_thread_flag(TIF_32BIT)) {
2669                 regs->tpc &= 0xffffffff;
2670                 regs->tnpc &= 0xffffffff;
2671         }
2672         info.si_signo = SIGILL;
2673         info.si_errno = 0;
2674         info.si_code = ILL_PRVOPC;
2675         info.si_addr = (void __user *)regs->tpc;
2676         info.si_trapno = 0;
2677         force_sig_info(SIGILL, &info, current);
2678 out:
2679         exception_exit(prev_state);
2680 }
2681 
2682 void do_privact(struct pt_regs *regs)
2683 {
2684         do_privop(regs);
2685 }
2686 
2687 /* Trap level 1 stuff or other traps we should never see... */
2688 void do_cee(struct pt_regs *regs)
2689 {
2690         exception_enter();
2691         die_if_kernel("TL0: Cache Error Exception", regs);
2692 }
2693 
2694 void do_div0_tl1(struct pt_regs *regs)
2695 {
2696         exception_enter();
2697         dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2698         die_if_kernel("TL1: DIV0 Exception", regs);
2699 }
2700 
2701 void do_fpieee_tl1(struct pt_regs *regs)
2702 {
2703         exception_enter();
2704         dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2705         die_if_kernel("TL1: FPU IEEE Exception", regs);
2706 }
2707 
2708 void do_fpother_tl1(struct pt_regs *regs)
2709 {
2710         exception_enter();
2711         dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2712         die_if_kernel("TL1: FPU Other Exception", regs);
2713 }
2714 
2715 void do_ill_tl1(struct pt_regs *regs)
2716 {
2717         exception_enter();
2718         dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2719         die_if_kernel("TL1: Illegal Instruction Exception", regs);
2720 }
2721 
2722 void do_irq_tl1(struct pt_regs *regs)
2723 {
2724         exception_enter();
2725         dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2726         die_if_kernel("TL1: IRQ Exception", regs);
2727 }
2728 
2729 void do_lddfmna_tl1(struct pt_regs *regs)
2730 {
2731         exception_enter();
2732         dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2733         die_if_kernel("TL1: LDDF Exception", regs);
2734 }
2735 
2736 void do_stdfmna_tl1(struct pt_regs *regs)
2737 {
2738         exception_enter();
2739         dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2740         die_if_kernel("TL1: STDF Exception", regs);
2741 }
2742 
2743 void do_paw(struct pt_regs *regs)
2744 {
2745         exception_enter();
2746         die_if_kernel("TL0: Phys Watchpoint Exception", regs);
2747 }
2748 
2749 void do_paw_tl1(struct pt_regs *regs)
2750 {
2751         exception_enter();
2752         dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2753         die_if_kernel("TL1: Phys Watchpoint Exception", regs);
2754 }
2755 
2756 void do_vaw(struct pt_regs *regs)
2757 {
2758         exception_enter();
2759         die_if_kernel("TL0: Virt Watchpoint Exception", regs);
2760 }
2761 
2762 void do_vaw_tl1(struct pt_regs *regs)
2763 {
2764         exception_enter();
2765         dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2766         die_if_kernel("TL1: Virt Watchpoint Exception", regs);
2767 }
2768 
2769 void do_tof_tl1(struct pt_regs *regs)
2770 {
2771         exception_enter();
2772         dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2773         die_if_kernel("TL1: Tag Overflow Exception", regs);
2774 }
2775 
2776 void do_getpsr(struct pt_regs *regs)
2777 {
2778         regs->u_regs[UREG_I0] = tstate_to_psr(regs->tstate);
2779         regs->tpc   = regs->tnpc;
2780         regs->tnpc += 4;
2781         if (test_thread_flag(TIF_32BIT)) {
2782                 regs->tpc &= 0xffffffff;
2783                 regs->tnpc &= 0xffffffff;
2784         }
2785 }
2786 
2787 u64 cpu_mondo_counter[NR_CPUS] = {0};
2788 struct trap_per_cpu trap_block[NR_CPUS];
2789 EXPORT_SYMBOL(trap_block);
2790 
2791 /* This can get invoked before sched_init() so play it super safe
2792  * and use hard_smp_processor_id().
2793  */
2794 void notrace init_cur_cpu_trap(struct thread_info *t)
2795 {
2796         int cpu = hard_smp_processor_id();
2797         struct trap_per_cpu *p = &trap_block[cpu];
2798 
2799         p->thread = t;
2800         p->pgd_paddr = 0;
2801 }
2802 
2803 extern void thread_info_offsets_are_bolixed_dave(void);
2804 extern void trap_per_cpu_offsets_are_bolixed_dave(void);
2805 extern void tsb_config_offsets_are_bolixed_dave(void);
2806 
2807 /* Only invoked on boot processor. */
2808 void __init trap_init(void)
2809 {
2810         /* Compile time sanity check. */
2811         BUILD_BUG_ON(TI_TASK != offsetof(struct thread_info, task) ||
2812                      TI_FLAGS != offsetof(struct thread_info, flags) ||
2813                      TI_CPU != offsetof(struct thread_info, cpu) ||
2814                      TI_FPSAVED != offsetof(struct thread_info, fpsaved) ||
2815                      TI_KSP != offsetof(struct thread_info, ksp) ||
2816                      TI_FAULT_ADDR != offsetof(struct thread_info,
2817                                                fault_address) ||
2818                      TI_KREGS != offsetof(struct thread_info, kregs) ||
2819                      TI_UTRAPS != offsetof(struct thread_info, utraps) ||
2820                      TI_REG_WINDOW != offsetof(struct thread_info,
2821                                                reg_window) ||
2822                      TI_RWIN_SPTRS != offsetof(struct thread_info,
2823                                                rwbuf_stkptrs) ||
2824                      TI_GSR != offsetof(struct thread_info, gsr) ||
2825                      TI_XFSR != offsetof(struct thread_info, xfsr) ||
2826                      TI_PRE_COUNT != offsetof(struct thread_info,
2827                                               preempt_count) ||
2828                      TI_NEW_CHILD != offsetof(struct thread_info, new_child) ||
2829                      TI_CURRENT_DS != offsetof(struct thread_info,
2830                                                 current_ds) ||
2831                      TI_KUNA_REGS != offsetof(struct thread_info,
2832                                               kern_una_regs) ||
2833                      TI_KUNA_INSN != offsetof(struct thread_info,
2834                                               kern_una_insn) ||
2835                      TI_FPREGS != offsetof(struct thread_info, fpregs) ||
2836                      (TI_FPREGS & (64 - 1)));
2837 
2838         BUILD_BUG_ON(TRAP_PER_CPU_THREAD != offsetof(struct trap_per_cpu,
2839                                                      thread) ||
2840                      (TRAP_PER_CPU_PGD_PADDR !=
2841                       offsetof(struct trap_per_cpu, pgd_paddr)) ||
2842                      (TRAP_PER_CPU_CPU_MONDO_PA !=
2843                       offsetof(struct trap_per_cpu, cpu_mondo_pa)) ||
2844                      (TRAP_PER_CPU_DEV_MONDO_PA !=
2845                       offsetof(struct trap_per_cpu, dev_mondo_pa)) ||
2846                      (TRAP_PER_CPU_RESUM_MONDO_PA !=
2847                       offsetof(struct trap_per_cpu, resum_mondo_pa)) ||
2848                      (TRAP_PER_CPU_RESUM_KBUF_PA !=
2849                       offsetof(struct trap_per_cpu, resum_kernel_buf_pa)) ||
2850                      (TRAP_PER_CPU_NONRESUM_MONDO_PA !=
2851                       offsetof(struct trap_per_cpu, nonresum_mondo_pa)) ||
2852                      (TRAP_PER_CPU_NONRESUM_KBUF_PA !=
2853                       offsetof(struct trap_per_cpu, nonresum_kernel_buf_pa)) ||
2854                      (TRAP_PER_CPU_FAULT_INFO !=
2855                       offsetof(struct trap_per_cpu, fault_info)) ||
2856                      (TRAP_PER_CPU_CPU_MONDO_BLOCK_PA !=
2857                       offsetof(struct trap_per_cpu, cpu_mondo_block_pa)) ||
2858                      (TRAP_PER_CPU_CPU_LIST_PA !=
2859                       offsetof(struct trap_per_cpu, cpu_list_pa)) ||
2860                      (TRAP_PER_CPU_TSB_HUGE !=
2861                       offsetof(struct trap_per_cpu, tsb_huge)) ||
2862                      (TRAP_PER_CPU_TSB_HUGE_TEMP !=
2863                       offsetof(struct trap_per_cpu, tsb_huge_temp)) ||
2864                      (TRAP_PER_CPU_IRQ_WORKLIST_PA !=
2865                       offsetof(struct trap_per_cpu, irq_worklist_pa)) ||
2866                      (TRAP_PER_CPU_CPU_MONDO_QMASK !=
2867                       offsetof(struct trap_per_cpu, cpu_mondo_qmask)) ||
2868                      (TRAP_PER_CPU_DEV_MONDO_QMASK !=
2869                       offsetof(struct trap_per_cpu, dev_mondo_qmask)) ||
2870                      (TRAP_PER_CPU_RESUM_QMASK !=
2871                       offsetof(struct trap_per_cpu, resum_qmask)) ||
2872                      (TRAP_PER_CPU_NONRESUM_QMASK !=
2873                       offsetof(struct trap_per_cpu, nonresum_qmask)) ||
2874                      (TRAP_PER_CPU_PER_CPU_BASE !=
2875                       offsetof(struct trap_per_cpu, __per_cpu_base)));
2876 
2877         BUILD_BUG_ON((TSB_CONFIG_TSB !=
2878                       offsetof(struct tsb_config, tsb)) ||
2879                      (TSB_CONFIG_RSS_LIMIT !=
2880                       offsetof(struct tsb_config, tsb_rss_limit)) ||
2881                      (TSB_CONFIG_NENTRIES !=
2882                       offsetof(struct tsb_config, tsb_nentries)) ||
2883                      (TSB_CONFIG_REG_VAL !=
2884                       offsetof(struct tsb_config, tsb_reg_val)) ||
2885                      (TSB_CONFIG_MAP_VADDR !=
2886                       offsetof(struct tsb_config, tsb_map_vaddr)) ||
2887                      (TSB_CONFIG_MAP_PTE !=
2888                       offsetof(struct tsb_config, tsb_map_pte)));
2889 
2890         /* Attach to the address space of init_task.  On SMP we
2891          * do this in smp.c:smp_callin for other cpus.
2892          */
2893         mmgrab(&init_mm);
2894         current->active_mm = &init_mm;
2895 }
2896 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp