~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/mips/kernel/unaligned.c

Version: ~ [ linux-5.8-rc3 ] ~ [ linux-5.7.5 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.48 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.129 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.185 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.228 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.228 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Handle unaligned accesses by emulation.
  3  *
  4  * This file is subject to the terms and conditions of the GNU General Public
  5  * License.  See the file "COPYING" in the main directory of this archive
  6  * for more details.
  7  *
  8  * Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle
  9  * Copyright (C) 1999 Silicon Graphics, Inc.
 10  * Copyright (C) 2014 Imagination Technologies Ltd.
 11  *
 12  * This file contains exception handler for address error exception with the
 13  * special capability to execute faulting instructions in software.  The
 14  * handler does not try to handle the case when the program counter points
 15  * to an address not aligned to a word boundary.
 16  *
 17  * Putting data to unaligned addresses is a bad practice even on Intel where
 18  * only the performance is affected.  Much worse is that such code is non-
 19  * portable.  Due to several programs that die on MIPS due to alignment
 20  * problems I decided to implement this handler anyway though I originally
 21  * didn't intend to do this at all for user code.
 22  *
 23  * For now I enable fixing of address errors by default to make life easier.
 24  * I however intend to disable this somewhen in the future when the alignment
 25  * problems with user programs have been fixed.  For programmers this is the
 26  * right way to go.
 27  *
 28  * Fixing address errors is a per process option.  The option is inherited
 29  * across fork(2) and execve(2) calls.  If you really want to use the
 30  * option in your user programs - I discourage the use of the software
 31  * emulation strongly - use the following code in your userland stuff:
 32  *
 33  * #include <sys/sysmips.h>
 34  *
 35  * ...
 36  * sysmips(MIPS_FIXADE, x);
 37  * ...
 38  *
 39  * The argument x is 0 for disabling software emulation, enabled otherwise.
 40  *
 41  * Below a little program to play around with this feature.
 42  *
 43  * #include <stdio.h>
 44  * #include <sys/sysmips.h>
 45  *
 46  * struct foo {
 47  *         unsigned char bar[8];
 48  * };
 49  *
 50  * main(int argc, char *argv[])
 51  * {
 52  *         struct foo x = {0, 1, 2, 3, 4, 5, 6, 7};
 53  *         unsigned int *p = (unsigned int *) (x.bar + 3);
 54  *         int i;
 55  *
 56  *         if (argc > 1)
 57  *                 sysmips(MIPS_FIXADE, atoi(argv[1]));
 58  *
 59  *         printf("*p = %08lx\n", *p);
 60  *
 61  *         *p = 0xdeadface;
 62  *
 63  *         for(i = 0; i <= 7; i++)
 64  *         printf("%02x ", x.bar[i]);
 65  *         printf("\n");
 66  * }
 67  *
 68  * Coprocessor loads are not supported; I think this case is unimportant
 69  * in the practice.
 70  *
 71  * TODO: Handle ndc (attempted store to doubleword in uncached memory)
 72  *       exception for the R6000.
 73  *       A store crossing a page boundary might be executed only partially.
 74  *       Undo the partial store in this case.
 75  */
 76 #include <linux/context_tracking.h>
 77 #include <linux/mm.h>
 78 #include <linux/signal.h>
 79 #include <linux/smp.h>
 80 #include <linux/sched.h>
 81 #include <linux/debugfs.h>
 82 #include <linux/perf_event.h>
 83 
 84 #include <asm/asm.h>
 85 #include <asm/branch.h>
 86 #include <asm/byteorder.h>
 87 #include <asm/cop2.h>
 88 #include <asm/debug.h>
 89 #include <asm/fpu.h>
 90 #include <asm/fpu_emulator.h>
 91 #include <asm/inst.h>
 92 #include <asm/mmu_context.h>
 93 #include <linux/uaccess.h>
 94 
 95 #define STR(x)  __STR(x)
 96 #define __STR(x)  #x
 97 
 98 enum {
 99         UNALIGNED_ACTION_QUIET,
100         UNALIGNED_ACTION_SIGNAL,
101         UNALIGNED_ACTION_SHOW,
102 };
103 #ifdef CONFIG_DEBUG_FS
104 static u32 unaligned_instructions;
105 static u32 unaligned_action;
106 #else
107 #define unaligned_action UNALIGNED_ACTION_QUIET
108 #endif
109 extern void show_registers(struct pt_regs *regs);
110 
111 #ifdef __BIG_ENDIAN
112 #define     _LoadHW(addr, value, res, type)  \
113 do {                                                        \
114                 __asm__ __volatile__ (".set\tnoat\n"        \
115                         "1:\t"type##_lb("%0", "0(%2)")"\n"  \
116                         "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\
117                         "sll\t%0, 0x8\n\t"                  \
118                         "or\t%0, $1\n\t"                    \
119                         "li\t%1, 0\n"                       \
120                         "3:\t.set\tat\n\t"                  \
121                         ".insn\n\t"                         \
122                         ".section\t.fixup,\"ax\"\n\t"       \
123                         "4:\tli\t%1, %3\n\t"                \
124                         "j\t3b\n\t"                         \
125                         ".previous\n\t"                     \
126                         ".section\t__ex_table,\"a\"\n\t"    \
127                         STR(PTR)"\t1b, 4b\n\t"              \
128                         STR(PTR)"\t2b, 4b\n\t"              \
129                         ".previous"                         \
130                         : "=&r" (value), "=r" (res)         \
131                         : "r" (addr), "i" (-EFAULT));       \
132 } while(0)
133 
134 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
135 #define     _LoadW(addr, value, res, type)   \
136 do {                                                        \
137                 __asm__ __volatile__ (                      \
138                         "1:\t"type##_lwl("%0", "(%2)")"\n"   \
139                         "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\
140                         "li\t%1, 0\n"                       \
141                         "3:\n\t"                            \
142                         ".insn\n\t"                         \
143                         ".section\t.fixup,\"ax\"\n\t"       \
144                         "4:\tli\t%1, %3\n\t"                \
145                         "j\t3b\n\t"                         \
146                         ".previous\n\t"                     \
147                         ".section\t__ex_table,\"a\"\n\t"    \
148                         STR(PTR)"\t1b, 4b\n\t"              \
149                         STR(PTR)"\t2b, 4b\n\t"              \
150                         ".previous"                         \
151                         : "=&r" (value), "=r" (res)         \
152                         : "r" (addr), "i" (-EFAULT));       \
153 } while(0)
154 
155 #else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
156 /* For CPUs without lwl instruction */
157 #define     _LoadW(addr, value, res, type) \
158 do {                                                        \
159                 __asm__ __volatile__ (                      \
160                         ".set\tpush\n"                      \
161                         ".set\tnoat\n\t"                    \
162                         "1:"type##_lb("%0", "0(%2)")"\n\t"  \
163                         "2:"type##_lbu("$1", "1(%2)")"\n\t" \
164                         "sll\t%0, 0x8\n\t"                  \
165                         "or\t%0, $1\n\t"                    \
166                         "3:"type##_lbu("$1", "2(%2)")"\n\t" \
167                         "sll\t%0, 0x8\n\t"                  \
168                         "or\t%0, $1\n\t"                    \
169                         "4:"type##_lbu("$1", "3(%2)")"\n\t" \
170                         "sll\t%0, 0x8\n\t"                  \
171                         "or\t%0, $1\n\t"                    \
172                         "li\t%1, 0\n"                       \
173                         ".set\tpop\n"                       \
174                         "10:\n\t"                           \
175                         ".insn\n\t"                         \
176                         ".section\t.fixup,\"ax\"\n\t"       \
177                         "11:\tli\t%1, %3\n\t"               \
178                         "j\t10b\n\t"                        \
179                         ".previous\n\t"                     \
180                         ".section\t__ex_table,\"a\"\n\t"    \
181                         STR(PTR)"\t1b, 11b\n\t"             \
182                         STR(PTR)"\t2b, 11b\n\t"             \
183                         STR(PTR)"\t3b, 11b\n\t"             \
184                         STR(PTR)"\t4b, 11b\n\t"             \
185                         ".previous"                         \
186                         : "=&r" (value), "=r" (res)         \
187                         : "r" (addr), "i" (-EFAULT));       \
188 } while(0)
189 
190 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
191 
192 #define     _LoadHWU(addr, value, res, type) \
193 do {                                                        \
194                 __asm__ __volatile__ (                      \
195                         ".set\tnoat\n"                      \
196                         "1:\t"type##_lbu("%0", "0(%2)")"\n" \
197                         "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\
198                         "sll\t%0, 0x8\n\t"                  \
199                         "or\t%0, $1\n\t"                    \
200                         "li\t%1, 0\n"                       \
201                         "3:\n\t"                            \
202                         ".insn\n\t"                         \
203                         ".set\tat\n\t"                      \
204                         ".section\t.fixup,\"ax\"\n\t"       \
205                         "4:\tli\t%1, %3\n\t"                \
206                         "j\t3b\n\t"                         \
207                         ".previous\n\t"                     \
208                         ".section\t__ex_table,\"a\"\n\t"    \
209                         STR(PTR)"\t1b, 4b\n\t"              \
210                         STR(PTR)"\t2b, 4b\n\t"              \
211                         ".previous"                         \
212                         : "=&r" (value), "=r" (res)         \
213                         : "r" (addr), "i" (-EFAULT));       \
214 } while(0)
215 
216 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
217 #define     _LoadWU(addr, value, res, type)  \
218 do {                                                        \
219                 __asm__ __volatile__ (                      \
220                         "1:\t"type##_lwl("%0", "(%2)")"\n"  \
221                         "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\
222                         "dsll\t%0, %0, 32\n\t"              \
223                         "dsrl\t%0, %0, 32\n\t"              \
224                         "li\t%1, 0\n"                       \
225                         "3:\n\t"                            \
226                         ".insn\n\t"                         \
227                         "\t.section\t.fixup,\"ax\"\n\t"     \
228                         "4:\tli\t%1, %3\n\t"                \
229                         "j\t3b\n\t"                         \
230                         ".previous\n\t"                     \
231                         ".section\t__ex_table,\"a\"\n\t"    \
232                         STR(PTR)"\t1b, 4b\n\t"              \
233                         STR(PTR)"\t2b, 4b\n\t"              \
234                         ".previous"                         \
235                         : "=&r" (value), "=r" (res)         \
236                         : "r" (addr), "i" (-EFAULT));       \
237 } while(0)
238 
239 #define     _LoadDW(addr, value, res)  \
240 do {                                                        \
241                 __asm__ __volatile__ (                      \
242                         "1:\tldl\t%0, (%2)\n"               \
243                         "2:\tldr\t%0, 7(%2)\n\t"            \
244                         "li\t%1, 0\n"                       \
245                         "3:\n\t"                            \
246                         ".insn\n\t"                         \
247                         "\t.section\t.fixup,\"ax\"\n\t"     \
248                         "4:\tli\t%1, %3\n\t"                \
249                         "j\t3b\n\t"                         \
250                         ".previous\n\t"                     \
251                         ".section\t__ex_table,\"a\"\n\t"    \
252                         STR(PTR)"\t1b, 4b\n\t"              \
253                         STR(PTR)"\t2b, 4b\n\t"              \
254                         ".previous"                         \
255                         : "=&r" (value), "=r" (res)         \
256                         : "r" (addr), "i" (-EFAULT));       \
257 } while(0)
258 
259 #else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
260 /* For CPUs without lwl and ldl instructions */
261 #define     _LoadWU(addr, value, res, type) \
262 do {                                                        \
263                 __asm__ __volatile__ (                      \
264                         ".set\tpush\n\t"                    \
265                         ".set\tnoat\n\t"                    \
266                         "1:"type##_lbu("%0", "0(%2)")"\n\t" \
267                         "2:"type##_lbu("$1", "1(%2)")"\n\t" \
268                         "sll\t%0, 0x8\n\t"                  \
269                         "or\t%0, $1\n\t"                    \
270                         "3:"type##_lbu("$1", "2(%2)")"\n\t" \
271                         "sll\t%0, 0x8\n\t"                  \
272                         "or\t%0, $1\n\t"                    \
273                         "4:"type##_lbu("$1", "3(%2)")"\n\t" \
274                         "sll\t%0, 0x8\n\t"                  \
275                         "or\t%0, $1\n\t"                    \
276                         "li\t%1, 0\n"                       \
277                         ".set\tpop\n"                       \
278                         "10:\n\t"                           \
279                         ".insn\n\t"                         \
280                         ".section\t.fixup,\"ax\"\n\t"       \
281                         "11:\tli\t%1, %3\n\t"               \
282                         "j\t10b\n\t"                        \
283                         ".previous\n\t"                     \
284                         ".section\t__ex_table,\"a\"\n\t"    \
285                         STR(PTR)"\t1b, 11b\n\t"             \
286                         STR(PTR)"\t2b, 11b\n\t"             \
287                         STR(PTR)"\t3b, 11b\n\t"             \
288                         STR(PTR)"\t4b, 11b\n\t"             \
289                         ".previous"                         \
290                         : "=&r" (value), "=r" (res)         \
291                         : "r" (addr), "i" (-EFAULT));       \
292 } while(0)
293 
294 #define     _LoadDW(addr, value, res)  \
295 do {                                                        \
296                 __asm__ __volatile__ (                      \
297                         ".set\tpush\n\t"                    \
298                         ".set\tnoat\n\t"                    \
299                         "1:lb\t%0, 0(%2)\n\t"               \
300                         "2:lbu\t $1, 1(%2)\n\t"             \
301                         "dsll\t%0, 0x8\n\t"                 \
302                         "or\t%0, $1\n\t"                    \
303                         "3:lbu\t$1, 2(%2)\n\t"              \
304                         "dsll\t%0, 0x8\n\t"                 \
305                         "or\t%0, $1\n\t"                    \
306                         "4:lbu\t$1, 3(%2)\n\t"              \
307                         "dsll\t%0, 0x8\n\t"                 \
308                         "or\t%0, $1\n\t"                    \
309                         "5:lbu\t$1, 4(%2)\n\t"              \
310                         "dsll\t%0, 0x8\n\t"                 \
311                         "or\t%0, $1\n\t"                    \
312                         "6:lbu\t$1, 5(%2)\n\t"              \
313                         "dsll\t%0, 0x8\n\t"                 \
314                         "or\t%0, $1\n\t"                    \
315                         "7:lbu\t$1, 6(%2)\n\t"              \
316                         "dsll\t%0, 0x8\n\t"                 \
317                         "or\t%0, $1\n\t"                    \
318                         "8:lbu\t$1, 7(%2)\n\t"              \
319                         "dsll\t%0, 0x8\n\t"                 \
320                         "or\t%0, $1\n\t"                    \
321                         "li\t%1, 0\n"                       \
322                         ".set\tpop\n\t"                     \
323                         "10:\n\t"                           \
324                         ".insn\n\t"                         \
325                         ".section\t.fixup,\"ax\"\n\t"       \
326                         "11:\tli\t%1, %3\n\t"               \
327                         "j\t10b\n\t"                        \
328                         ".previous\n\t"                     \
329                         ".section\t__ex_table,\"a\"\n\t"    \
330                         STR(PTR)"\t1b, 11b\n\t"             \
331                         STR(PTR)"\t2b, 11b\n\t"             \
332                         STR(PTR)"\t3b, 11b\n\t"             \
333                         STR(PTR)"\t4b, 11b\n\t"             \
334                         STR(PTR)"\t5b, 11b\n\t"             \
335                         STR(PTR)"\t6b, 11b\n\t"             \
336                         STR(PTR)"\t7b, 11b\n\t"             \
337                         STR(PTR)"\t8b, 11b\n\t"             \
338                         ".previous"                         \
339                         : "=&r" (value), "=r" (res)         \
340                         : "r" (addr), "i" (-EFAULT));       \
341 } while(0)
342 
343 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
344 
345 
346 #define     _StoreHW(addr, value, res, type) \
347 do {                                                        \
348                 __asm__ __volatile__ (                      \
349                         ".set\tnoat\n"                      \
350                         "1:\t"type##_sb("%1", "1(%2)")"\n"  \
351                         "srl\t$1, %1, 0x8\n"                \
352                         "2:\t"type##_sb("$1", "0(%2)")"\n"  \
353                         ".set\tat\n\t"                      \
354                         "li\t%0, 0\n"                       \
355                         "3:\n\t"                            \
356                         ".insn\n\t"                         \
357                         ".section\t.fixup,\"ax\"\n\t"       \
358                         "4:\tli\t%0, %3\n\t"                \
359                         "j\t3b\n\t"                         \
360                         ".previous\n\t"                     \
361                         ".section\t__ex_table,\"a\"\n\t"    \
362                         STR(PTR)"\t1b, 4b\n\t"              \
363                         STR(PTR)"\t2b, 4b\n\t"              \
364                         ".previous"                         \
365                         : "=r" (res)                        \
366                         : "r" (value), "r" (addr), "i" (-EFAULT));\
367 } while(0)
368 
369 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
370 #define     _StoreW(addr, value, res, type)  \
371 do {                                                        \
372                 __asm__ __volatile__ (                      \
373                         "1:\t"type##_swl("%1", "(%2)")"\n"  \
374                         "2:\t"type##_swr("%1", "3(%2)")"\n\t"\
375                         "li\t%0, 0\n"                       \
376                         "3:\n\t"                            \
377                         ".insn\n\t"                         \
378                         ".section\t.fixup,\"ax\"\n\t"       \
379                         "4:\tli\t%0, %3\n\t"                \
380                         "j\t3b\n\t"                         \
381                         ".previous\n\t"                     \
382                         ".section\t__ex_table,\"a\"\n\t"    \
383                         STR(PTR)"\t1b, 4b\n\t"              \
384                         STR(PTR)"\t2b, 4b\n\t"              \
385                         ".previous"                         \
386                 : "=r" (res)                                \
387                 : "r" (value), "r" (addr), "i" (-EFAULT));  \
388 } while(0)
389 
390 #define     _StoreDW(addr, value, res) \
391 do {                                                        \
392                 __asm__ __volatile__ (                      \
393                         "1:\tsdl\t%1,(%2)\n"                \
394                         "2:\tsdr\t%1, 7(%2)\n\t"            \
395                         "li\t%0, 0\n"                       \
396                         "3:\n\t"                            \
397                         ".insn\n\t"                         \
398                         ".section\t.fixup,\"ax\"\n\t"       \
399                         "4:\tli\t%0, %3\n\t"                \
400                         "j\t3b\n\t"                         \
401                         ".previous\n\t"                     \
402                         ".section\t__ex_table,\"a\"\n\t"    \
403                         STR(PTR)"\t1b, 4b\n\t"              \
404                         STR(PTR)"\t2b, 4b\n\t"              \
405                         ".previous"                         \
406                 : "=r" (res)                                \
407                 : "r" (value), "r" (addr), "i" (-EFAULT));  \
408 } while(0)
409 
410 #else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
411 #define     _StoreW(addr, value, res, type)  \
412 do {                                                        \
413                 __asm__ __volatile__ (                      \
414                         ".set\tpush\n\t"                    \
415                         ".set\tnoat\n\t"                    \
416                         "1:"type##_sb("%1", "3(%2)")"\n\t"  \
417                         "srl\t$1, %1, 0x8\n\t"              \
418                         "2:"type##_sb("$1", "2(%2)")"\n\t"  \
419                         "srl\t$1, $1,  0x8\n\t"             \
420                         "3:"type##_sb("$1", "1(%2)")"\n\t"  \
421                         "srl\t$1, $1, 0x8\n\t"              \
422                         "4:"type##_sb("$1", "0(%2)")"\n\t"  \
423                         ".set\tpop\n\t"                     \
424                         "li\t%0, 0\n"                       \
425                         "10:\n\t"                           \
426                         ".insn\n\t"                         \
427                         ".section\t.fixup,\"ax\"\n\t"       \
428                         "11:\tli\t%0, %3\n\t"               \
429                         "j\t10b\n\t"                        \
430                         ".previous\n\t"                     \
431                         ".section\t__ex_table,\"a\"\n\t"    \
432                         STR(PTR)"\t1b, 11b\n\t"             \
433                         STR(PTR)"\t2b, 11b\n\t"             \
434                         STR(PTR)"\t3b, 11b\n\t"             \
435                         STR(PTR)"\t4b, 11b\n\t"             \
436                         ".previous"                         \
437                 : "=&r" (res)                               \
438                 : "r" (value), "r" (addr), "i" (-EFAULT)    \
439                 : "memory");                                \
440 } while(0)
441 
442 #define     _StoreDW(addr, value, res) \
443 do {                                                        \
444                 __asm__ __volatile__ (                      \
445                         ".set\tpush\n\t"                    \
446                         ".set\tnoat\n\t"                    \
447                         "1:sb\t%1, 7(%2)\n\t"               \
448                         "dsrl\t$1, %1, 0x8\n\t"             \
449                         "2:sb\t$1, 6(%2)\n\t"               \
450                         "dsrl\t$1, $1, 0x8\n\t"             \
451                         "3:sb\t$1, 5(%2)\n\t"               \
452                         "dsrl\t$1, $1, 0x8\n\t"             \
453                         "4:sb\t$1, 4(%2)\n\t"               \
454                         "dsrl\t$1, $1, 0x8\n\t"             \
455                         "5:sb\t$1, 3(%2)\n\t"               \
456                         "dsrl\t$1, $1, 0x8\n\t"             \
457                         "6:sb\t$1, 2(%2)\n\t"               \
458                         "dsrl\t$1, $1, 0x8\n\t"             \
459                         "7:sb\t$1, 1(%2)\n\t"               \
460                         "dsrl\t$1, $1, 0x8\n\t"             \
461                         "8:sb\t$1, 0(%2)\n\t"               \
462                         "dsrl\t$1, $1, 0x8\n\t"             \
463                         ".set\tpop\n\t"                     \
464                         "li\t%0, 0\n"                       \
465                         "10:\n\t"                           \
466                         ".insn\n\t"                         \
467                         ".section\t.fixup,\"ax\"\n\t"       \
468                         "11:\tli\t%0, %3\n\t"               \
469                         "j\t10b\n\t"                        \
470                         ".previous\n\t"                     \
471                         ".section\t__ex_table,\"a\"\n\t"    \
472                         STR(PTR)"\t1b, 11b\n\t"             \
473                         STR(PTR)"\t2b, 11b\n\t"             \
474                         STR(PTR)"\t3b, 11b\n\t"             \
475                         STR(PTR)"\t4b, 11b\n\t"             \
476                         STR(PTR)"\t5b, 11b\n\t"             \
477                         STR(PTR)"\t6b, 11b\n\t"             \
478                         STR(PTR)"\t7b, 11b\n\t"             \
479                         STR(PTR)"\t8b, 11b\n\t"             \
480                         ".previous"                         \
481                 : "=&r" (res)                               \
482                 : "r" (value), "r" (addr), "i" (-EFAULT)    \
483                 : "memory");                                \
484 } while(0)
485 
486 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
487 
488 #else /* __BIG_ENDIAN */
489 
490 #define     _LoadHW(addr, value, res, type)  \
491 do {                                                        \
492                 __asm__ __volatile__ (".set\tnoat\n"        \
493                         "1:\t"type##_lb("%0", "1(%2)")"\n"  \
494                         "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\
495                         "sll\t%0, 0x8\n\t"                  \
496                         "or\t%0, $1\n\t"                    \
497                         "li\t%1, 0\n"                       \
498                         "3:\t.set\tat\n\t"                  \
499                         ".insn\n\t"                         \
500                         ".section\t.fixup,\"ax\"\n\t"       \
501                         "4:\tli\t%1, %3\n\t"                \
502                         "j\t3b\n\t"                         \
503                         ".previous\n\t"                     \
504                         ".section\t__ex_table,\"a\"\n\t"    \
505                         STR(PTR)"\t1b, 4b\n\t"              \
506                         STR(PTR)"\t2b, 4b\n\t"              \
507                         ".previous"                         \
508                         : "=&r" (value), "=r" (res)         \
509                         : "r" (addr), "i" (-EFAULT));       \
510 } while(0)
511 
512 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
513 #define     _LoadW(addr, value, res, type)   \
514 do {                                                        \
515                 __asm__ __volatile__ (                      \
516                         "1:\t"type##_lwl("%0", "3(%2)")"\n" \
517                         "2:\t"type##_lwr("%0", "(%2)")"\n\t"\
518                         "li\t%1, 0\n"                       \
519                         "3:\n\t"                            \
520                         ".insn\n\t"                         \
521                         ".section\t.fixup,\"ax\"\n\t"       \
522                         "4:\tli\t%1, %3\n\t"                \
523                         "j\t3b\n\t"                         \
524                         ".previous\n\t"                     \
525                         ".section\t__ex_table,\"a\"\n\t"    \
526                         STR(PTR)"\t1b, 4b\n\t"              \
527                         STR(PTR)"\t2b, 4b\n\t"              \
528                         ".previous"                         \
529                         : "=&r" (value), "=r" (res)         \
530                         : "r" (addr), "i" (-EFAULT));       \
531 } while(0)
532 
533 #else  /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
534 /* For CPUs without lwl instruction */
535 #define     _LoadW(addr, value, res, type) \
536 do {                                                        \
537                 __asm__ __volatile__ (                      \
538                         ".set\tpush\n"                      \
539                         ".set\tnoat\n\t"                    \
540                         "1:"type##_lb("%0", "3(%2)")"\n\t"  \
541                         "2:"type##_lbu("$1", "2(%2)")"\n\t" \
542                         "sll\t%0, 0x8\n\t"                  \
543                         "or\t%0, $1\n\t"                    \
544                         "3:"type##_lbu("$1", "1(%2)")"\n\t" \
545                         "sll\t%0, 0x8\n\t"                  \
546                         "or\t%0, $1\n\t"                    \
547                         "4:"type##_lbu("$1", "0(%2)")"\n\t" \
548                         "sll\t%0, 0x8\n\t"                  \
549                         "or\t%0, $1\n\t"                    \
550                         "li\t%1, 0\n"                       \
551                         ".set\tpop\n"                       \
552                         "10:\n\t"                           \
553                         ".insn\n\t"                         \
554                         ".section\t.fixup,\"ax\"\n\t"       \
555                         "11:\tli\t%1, %3\n\t"               \
556                         "j\t10b\n\t"                        \
557                         ".previous\n\t"                     \
558                         ".section\t__ex_table,\"a\"\n\t"    \
559                         STR(PTR)"\t1b, 11b\n\t"             \
560                         STR(PTR)"\t2b, 11b\n\t"             \
561                         STR(PTR)"\t3b, 11b\n\t"             \
562                         STR(PTR)"\t4b, 11b\n\t"             \
563                         ".previous"                         \
564                         : "=&r" (value), "=r" (res)         \
565                         : "r" (addr), "i" (-EFAULT));       \
566 } while(0)
567 
568 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
569 
570 
571 #define     _LoadHWU(addr, value, res, type) \
572 do {                                                        \
573                 __asm__ __volatile__ (                      \
574                         ".set\tnoat\n"                      \
575                         "1:\t"type##_lbu("%0", "1(%2)")"\n" \
576                         "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\
577                         "sll\t%0, 0x8\n\t"                  \
578                         "or\t%0, $1\n\t"                    \
579                         "li\t%1, 0\n"                       \
580                         "3:\n\t"                            \
581                         ".insn\n\t"                         \
582                         ".set\tat\n\t"                      \
583                         ".section\t.fixup,\"ax\"\n\t"       \
584                         "4:\tli\t%1, %3\n\t"                \
585                         "j\t3b\n\t"                         \
586                         ".previous\n\t"                     \
587                         ".section\t__ex_table,\"a\"\n\t"    \
588                         STR(PTR)"\t1b, 4b\n\t"              \
589                         STR(PTR)"\t2b, 4b\n\t"              \
590                         ".previous"                         \
591                         : "=&r" (value), "=r" (res)         \
592                         : "r" (addr), "i" (-EFAULT));       \
593 } while(0)
594 
595 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
596 #define     _LoadWU(addr, value, res, type)  \
597 do {                                                        \
598                 __asm__ __volatile__ (                      \
599                         "1:\t"type##_lwl("%0", "3(%2)")"\n" \
600                         "2:\t"type##_lwr("%0", "(%2)")"\n\t"\
601                         "dsll\t%0, %0, 32\n\t"              \
602                         "dsrl\t%0, %0, 32\n\t"              \
603                         "li\t%1, 0\n"                       \
604                         "3:\n\t"                            \
605                         ".insn\n\t"                         \
606                         "\t.section\t.fixup,\"ax\"\n\t"     \
607                         "4:\tli\t%1, %3\n\t"                \
608                         "j\t3b\n\t"                         \
609                         ".previous\n\t"                     \
610                         ".section\t__ex_table,\"a\"\n\t"    \
611                         STR(PTR)"\t1b, 4b\n\t"              \
612                         STR(PTR)"\t2b, 4b\n\t"              \
613                         ".previous"                         \
614                         : "=&r" (value), "=r" (res)         \
615                         : "r" (addr), "i" (-EFAULT));       \
616 } while(0)
617 
618 #define     _LoadDW(addr, value, res)  \
619 do {                                                        \
620                 __asm__ __volatile__ (                      \
621                         "1:\tldl\t%0, 7(%2)\n"              \
622                         "2:\tldr\t%0, (%2)\n\t"             \
623                         "li\t%1, 0\n"                       \
624                         "3:\n\t"                            \
625                         ".insn\n\t"                         \
626                         "\t.section\t.fixup,\"ax\"\n\t"     \
627                         "4:\tli\t%1, %3\n\t"                \
628                         "j\t3b\n\t"                         \
629                         ".previous\n\t"                     \
630                         ".section\t__ex_table,\"a\"\n\t"    \
631                         STR(PTR)"\t1b, 4b\n\t"              \
632                         STR(PTR)"\t2b, 4b\n\t"              \
633                         ".previous"                         \
634                         : "=&r" (value), "=r" (res)         \
635                         : "r" (addr), "i" (-EFAULT));       \
636 } while(0)
637 
638 #else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
639 /* For CPUs without lwl and ldl instructions */
640 #define     _LoadWU(addr, value, res, type) \
641 do {                                                        \
642                 __asm__ __volatile__ (                      \
643                         ".set\tpush\n\t"                    \
644                         ".set\tnoat\n\t"                    \
645                         "1:"type##_lbu("%0", "3(%2)")"\n\t" \
646                         "2:"type##_lbu("$1", "2(%2)")"\n\t" \
647                         "sll\t%0, 0x8\n\t"                  \
648                         "or\t%0, $1\n\t"                    \
649                         "3:"type##_lbu("$1", "1(%2)")"\n\t" \
650                         "sll\t%0, 0x8\n\t"                  \
651                         "or\t%0, $1\n\t"                    \
652                         "4:"type##_lbu("$1", "0(%2)")"\n\t" \
653                         "sll\t%0, 0x8\n\t"                  \
654                         "or\t%0, $1\n\t"                    \
655                         "li\t%1, 0\n"                       \
656                         ".set\tpop\n"                       \
657                         "10:\n\t"                           \
658                         ".insn\n\t"                         \
659                         ".section\t.fixup,\"ax\"\n\t"       \
660                         "11:\tli\t%1, %3\n\t"               \
661                         "j\t10b\n\t"                        \
662                         ".previous\n\t"                     \
663                         ".section\t__ex_table,\"a\"\n\t"    \
664                         STR(PTR)"\t1b, 11b\n\t"             \
665                         STR(PTR)"\t2b, 11b\n\t"             \
666                         STR(PTR)"\t3b, 11b\n\t"             \
667                         STR(PTR)"\t4b, 11b\n\t"             \
668                         ".previous"                         \
669                         : "=&r" (value), "=r" (res)         \
670                         : "r" (addr), "i" (-EFAULT));       \
671 } while(0)
672 
673 #define     _LoadDW(addr, value, res)  \
674 do {                                                        \
675                 __asm__ __volatile__ (                      \
676                         ".set\tpush\n\t"                    \
677                         ".set\tnoat\n\t"                    \
678                         "1:lb\t%0, 7(%2)\n\t"               \
679                         "2:lbu\t$1, 6(%2)\n\t"              \
680                         "dsll\t%0, 0x8\n\t"                 \
681                         "or\t%0, $1\n\t"                    \
682                         "3:lbu\t$1, 5(%2)\n\t"              \
683                         "dsll\t%0, 0x8\n\t"                 \
684                         "or\t%0, $1\n\t"                    \
685                         "4:lbu\t$1, 4(%2)\n\t"              \
686                         "dsll\t%0, 0x8\n\t"                 \
687                         "or\t%0, $1\n\t"                    \
688                         "5:lbu\t$1, 3(%2)\n\t"              \
689                         "dsll\t%0, 0x8\n\t"                 \
690                         "or\t%0, $1\n\t"                    \
691                         "6:lbu\t$1, 2(%2)\n\t"              \
692                         "dsll\t%0, 0x8\n\t"                 \
693                         "or\t%0, $1\n\t"                    \
694                         "7:lbu\t$1, 1(%2)\n\t"              \
695                         "dsll\t%0, 0x8\n\t"                 \
696                         "or\t%0, $1\n\t"                    \
697                         "8:lbu\t$1, 0(%2)\n\t"              \
698                         "dsll\t%0, 0x8\n\t"                 \
699                         "or\t%0, $1\n\t"                    \
700                         "li\t%1, 0\n"                       \
701                         ".set\tpop\n\t"                     \
702                         "10:\n\t"                           \
703                         ".insn\n\t"                         \
704                         ".section\t.fixup,\"ax\"\n\t"       \
705                         "11:\tli\t%1, %3\n\t"               \
706                         "j\t10b\n\t"                        \
707                         ".previous\n\t"                     \
708                         ".section\t__ex_table,\"a\"\n\t"    \
709                         STR(PTR)"\t1b, 11b\n\t"             \
710                         STR(PTR)"\t2b, 11b\n\t"             \
711                         STR(PTR)"\t3b, 11b\n\t"             \
712                         STR(PTR)"\t4b, 11b\n\t"             \
713                         STR(PTR)"\t5b, 11b\n\t"             \
714                         STR(PTR)"\t6b, 11b\n\t"             \
715                         STR(PTR)"\t7b, 11b\n\t"             \
716                         STR(PTR)"\t8b, 11b\n\t"             \
717                         ".previous"                         \
718                         : "=&r" (value), "=r" (res)         \
719                         : "r" (addr), "i" (-EFAULT));       \
720 } while(0)
721 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
722 
723 #define     _StoreHW(addr, value, res, type) \
724 do {                                                        \
725                 __asm__ __volatile__ (                      \
726                         ".set\tnoat\n"                      \
727                         "1:\t"type##_sb("%1", "0(%2)")"\n"  \
728                         "srl\t$1,%1, 0x8\n"                 \
729                         "2:\t"type##_sb("$1", "1(%2)")"\n"  \
730                         ".set\tat\n\t"                      \
731                         "li\t%0, 0\n"                       \
732                         "3:\n\t"                            \
733                         ".insn\n\t"                         \
734                         ".section\t.fixup,\"ax\"\n\t"       \
735                         "4:\tli\t%0, %3\n\t"                \
736                         "j\t3b\n\t"                         \
737                         ".previous\n\t"                     \
738                         ".section\t__ex_table,\"a\"\n\t"    \
739                         STR(PTR)"\t1b, 4b\n\t"              \
740                         STR(PTR)"\t2b, 4b\n\t"              \
741                         ".previous"                         \
742                         : "=r" (res)                        \
743                         : "r" (value), "r" (addr), "i" (-EFAULT));\
744 } while(0)
745 
746 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
747 #define     _StoreW(addr, value, res, type)  \
748 do {                                                        \
749                 __asm__ __volatile__ (                      \
750                         "1:\t"type##_swl("%1", "3(%2)")"\n" \
751                         "2:\t"type##_swr("%1", "(%2)")"\n\t"\
752                         "li\t%0, 0\n"                       \
753                         "3:\n\t"                            \
754                         ".insn\n\t"                         \
755                         ".section\t.fixup,\"ax\"\n\t"       \
756                         "4:\tli\t%0, %3\n\t"                \
757                         "j\t3b\n\t"                         \
758                         ".previous\n\t"                     \
759                         ".section\t__ex_table,\"a\"\n\t"    \
760                         STR(PTR)"\t1b, 4b\n\t"              \
761                         STR(PTR)"\t2b, 4b\n\t"              \
762                         ".previous"                         \
763                 : "=r" (res)                                \
764                 : "r" (value), "r" (addr), "i" (-EFAULT));  \
765 } while(0)
766 
767 #define     _StoreDW(addr, value, res) \
768 do {                                                        \
769                 __asm__ __volatile__ (                      \
770                         "1:\tsdl\t%1, 7(%2)\n"              \
771                         "2:\tsdr\t%1, (%2)\n\t"             \
772                         "li\t%0, 0\n"                       \
773                         "3:\n\t"                            \
774                         ".insn\n\t"                         \
775                         ".section\t.fixup,\"ax\"\n\t"       \
776                         "4:\tli\t%0, %3\n\t"                \
777                         "j\t3b\n\t"                         \
778                         ".previous\n\t"                     \
779                         ".section\t__ex_table,\"a\"\n\t"    \
780                         STR(PTR)"\t1b, 4b\n\t"              \
781                         STR(PTR)"\t2b, 4b\n\t"              \
782                         ".previous"                         \
783                 : "=r" (res)                                \
784                 : "r" (value), "r" (addr), "i" (-EFAULT));  \
785 } while(0)
786 
787 #else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
788 /* For CPUs without swl and sdl instructions */
789 #define     _StoreW(addr, value, res, type)  \
790 do {                                                        \
791                 __asm__ __volatile__ (                      \
792                         ".set\tpush\n\t"                    \
793                         ".set\tnoat\n\t"                    \
794                         "1:"type##_sb("%1", "0(%2)")"\n\t"  \
795                         "srl\t$1, %1, 0x8\n\t"              \
796                         "2:"type##_sb("$1", "1(%2)")"\n\t"  \
797                         "srl\t$1, $1,  0x8\n\t"             \
798                         "3:"type##_sb("$1", "2(%2)")"\n\t"  \
799                         "srl\t$1, $1, 0x8\n\t"              \
800                         "4:"type##_sb("$1", "3(%2)")"\n\t"  \
801                         ".set\tpop\n\t"                     \
802                         "li\t%0, 0\n"                       \
803                         "10:\n\t"                           \
804                         ".insn\n\t"                         \
805                         ".section\t.fixup,\"ax\"\n\t"       \
806                         "11:\tli\t%0, %3\n\t"               \
807                         "j\t10b\n\t"                        \
808                         ".previous\n\t"                     \
809                         ".section\t__ex_table,\"a\"\n\t"    \
810                         STR(PTR)"\t1b, 11b\n\t"             \
811                         STR(PTR)"\t2b, 11b\n\t"             \
812                         STR(PTR)"\t3b, 11b\n\t"             \
813                         STR(PTR)"\t4b, 11b\n\t"             \
814                         ".previous"                         \
815                 : "=&r" (res)                               \
816                 : "r" (value), "r" (addr), "i" (-EFAULT)    \
817                 : "memory");                                \
818 } while(0)
819 
820 #define     _StoreDW(addr, value, res) \
821 do {                                                        \
822                 __asm__ __volatile__ (                      \
823                         ".set\tpush\n\t"                    \
824                         ".set\tnoat\n\t"                    \
825                         "1:sb\t%1, 0(%2)\n\t"               \
826                         "dsrl\t$1, %1, 0x8\n\t"             \
827                         "2:sb\t$1, 1(%2)\n\t"               \
828                         "dsrl\t$1, $1, 0x8\n\t"             \
829                         "3:sb\t$1, 2(%2)\n\t"               \
830                         "dsrl\t$1, $1, 0x8\n\t"             \
831                         "4:sb\t$1, 3(%2)\n\t"               \
832                         "dsrl\t$1, $1, 0x8\n\t"             \
833                         "5:sb\t$1, 4(%2)\n\t"               \
834                         "dsrl\t$1, $1, 0x8\n\t"             \
835                         "6:sb\t$1, 5(%2)\n\t"               \
836                         "dsrl\t$1, $1, 0x8\n\t"             \
837                         "7:sb\t$1, 6(%2)\n\t"               \
838                         "dsrl\t$1, $1, 0x8\n\t"             \
839                         "8:sb\t$1, 7(%2)\n\t"               \
840                         "dsrl\t$1, $1, 0x8\n\t"             \
841                         ".set\tpop\n\t"                     \
842                         "li\t%0, 0\n"                       \
843                         "10:\n\t"                           \
844                         ".insn\n\t"                         \
845                         ".section\t.fixup,\"ax\"\n\t"       \
846                         "11:\tli\t%0, %3\n\t"               \
847                         "j\t10b\n\t"                        \
848                         ".previous\n\t"                     \
849                         ".section\t__ex_table,\"a\"\n\t"    \
850                         STR(PTR)"\t1b, 11b\n\t"             \
851                         STR(PTR)"\t2b, 11b\n\t"             \
852                         STR(PTR)"\t3b, 11b\n\t"             \
853                         STR(PTR)"\t4b, 11b\n\t"             \
854                         STR(PTR)"\t5b, 11b\n\t"             \
855                         STR(PTR)"\t6b, 11b\n\t"             \
856                         STR(PTR)"\t7b, 11b\n\t"             \
857                         STR(PTR)"\t8b, 11b\n\t"             \
858                         ".previous"                         \
859                 : "=&r" (res)                               \
860                 : "r" (value), "r" (addr), "i" (-EFAULT)    \
861                 : "memory");                                \
862 } while(0)
863 
864 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
865 #endif
866 
867 #define LoadHWU(addr, value, res)       _LoadHWU(addr, value, res, kernel)
868 #define LoadHWUE(addr, value, res)      _LoadHWU(addr, value, res, user)
869 #define LoadWU(addr, value, res)        _LoadWU(addr, value, res, kernel)
870 #define LoadWUE(addr, value, res)       _LoadWU(addr, value, res, user)
871 #define LoadHW(addr, value, res)        _LoadHW(addr, value, res, kernel)
872 #define LoadHWE(addr, value, res)       _LoadHW(addr, value, res, user)
873 #define LoadW(addr, value, res)         _LoadW(addr, value, res, kernel)
874 #define LoadWE(addr, value, res)        _LoadW(addr, value, res, user)
875 #define LoadDW(addr, value, res)        _LoadDW(addr, value, res)
876 
877 #define StoreHW(addr, value, res)       _StoreHW(addr, value, res, kernel)
878 #define StoreHWE(addr, value, res)      _StoreHW(addr, value, res, user)
879 #define StoreW(addr, value, res)        _StoreW(addr, value, res, kernel)
880 #define StoreWE(addr, value, res)       _StoreW(addr, value, res, user)
881 #define StoreDW(addr, value, res)       _StoreDW(addr, value, res)
882 
883 static void emulate_load_store_insn(struct pt_regs *regs,
884         void __user *addr, unsigned int __user *pc)
885 {
886         unsigned long origpc, orig31, value;
887         union mips_instruction insn;
888         unsigned int res;
889 #ifdef  CONFIG_EVA
890         mm_segment_t seg;
891 #endif
892         origpc = (unsigned long)pc;
893         orig31 = regs->regs[31];
894 
895         perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0);
896 
897         /*
898          * This load never faults.
899          */
900         __get_user(insn.word, pc);
901 
902         switch (insn.i_format.opcode) {
903                 /*
904                  * These are instructions that a compiler doesn't generate.  We
905                  * can assume therefore that the code is MIPS-aware and
906                  * really buggy.  Emulating these instructions would break the
907                  * semantics anyway.
908                  */
909         case ll_op:
910         case lld_op:
911         case sc_op:
912         case scd_op:
913 
914                 /*
915                  * For these instructions the only way to create an address
916                  * error is an attempted access to kernel/supervisor address
917                  * space.
918                  */
919         case ldl_op:
920         case ldr_op:
921         case lwl_op:
922         case lwr_op:
923         case sdl_op:
924         case sdr_op:
925         case swl_op:
926         case swr_op:
927         case lb_op:
928         case lbu_op:
929         case sb_op:
930                 goto sigbus;
931 
932                 /*
933                  * The remaining opcodes are the ones that are really of
934                  * interest.
935                  */
936         case spec3_op:
937                 if (insn.dsp_format.func == lx_op) {
938                         switch (insn.dsp_format.op) {
939                         case lwx_op:
940                                 if (!access_ok(addr, 4))
941                                         goto sigbus;
942                                 LoadW(addr, value, res);
943                                 if (res)
944                                         goto fault;
945                                 compute_return_epc(regs);
946                                 regs->regs[insn.dsp_format.rd] = value;
947                                 break;
948                         case lhx_op:
949                                 if (!access_ok(addr, 2))
950                                         goto sigbus;
951                                 LoadHW(addr, value, res);
952                                 if (res)
953                                         goto fault;
954                                 compute_return_epc(regs);
955                                 regs->regs[insn.dsp_format.rd] = value;
956                                 break;
957                         default:
958                                 goto sigill;
959                         }
960                 }
961 #ifdef CONFIG_EVA
962                 else {
963                         /*
964                          * we can land here only from kernel accessing user
965                          * memory, so we need to "switch" the address limit to
966                          * user space, so that address check can work properly.
967                          */
968                         seg = get_fs();
969                         set_fs(USER_DS);
970                         switch (insn.spec3_format.func) {
971                         case lhe_op:
972                                 if (!access_ok(addr, 2)) {
973                                         set_fs(seg);
974                                         goto sigbus;
975                                 }
976                                 LoadHWE(addr, value, res);
977                                 if (res) {
978                                         set_fs(seg);
979                                         goto fault;
980                                 }
981                                 compute_return_epc(regs);
982                                 regs->regs[insn.spec3_format.rt] = value;
983                                 break;
984                         case lwe_op:
985                                 if (!access_ok(addr, 4)) {
986                                         set_fs(seg);
987                                         goto sigbus;
988                                 }
989                                 LoadWE(addr, value, res);
990                                 if (res) {
991                                         set_fs(seg);
992                                         goto fault;
993                                 }
994                                 compute_return_epc(regs);
995                                 regs->regs[insn.spec3_format.rt] = value;
996                                 break;
997                         case lhue_op:
998                                 if (!access_ok(addr, 2)) {
999                                         set_fs(seg);
1000                                         goto sigbus;
1001                                 }
1002                                 LoadHWUE(addr, value, res);
1003                                 if (res) {
1004                                         set_fs(seg);
1005                                         goto fault;
1006                                 }
1007                                 compute_return_epc(regs);
1008                                 regs->regs[insn.spec3_format.rt] = value;
1009                                 break;
1010                         case she_op:
1011                                 if (!access_ok(addr, 2)) {
1012                                         set_fs(seg);
1013                                         goto sigbus;
1014                                 }
1015                                 compute_return_epc(regs);
1016                                 value = regs->regs[insn.spec3_format.rt];
1017                                 StoreHWE(addr, value, res);
1018                                 if (res) {
1019                                         set_fs(seg);
1020                                         goto fault;
1021                                 }
1022                                 break;
1023                         case swe_op:
1024                                 if (!access_ok(addr, 4)) {
1025                                         set_fs(seg);
1026                                         goto sigbus;
1027                                 }
1028                                 compute_return_epc(regs);
1029                                 value = regs->regs[insn.spec3_format.rt];
1030                                 StoreWE(addr, value, res);
1031                                 if (res) {
1032                                         set_fs(seg);
1033                                         goto fault;
1034                                 }
1035                                 break;
1036                         default:
1037                                 set_fs(seg);
1038                                 goto sigill;
1039                         }
1040                         set_fs(seg);
1041                 }
1042 #endif
1043                 break;
1044         case lh_op:
1045                 if (!access_ok(addr, 2))
1046                         goto sigbus;
1047 
1048                 if (IS_ENABLED(CONFIG_EVA)) {
1049                         if (uaccess_kernel())
1050                                 LoadHW(addr, value, res);
1051                         else
1052                                 LoadHWE(addr, value, res);
1053                 } else {
1054                         LoadHW(addr, value, res);
1055                 }
1056 
1057                 if (res)
1058                         goto fault;
1059                 compute_return_epc(regs);
1060                 regs->regs[insn.i_format.rt] = value;
1061                 break;
1062 
1063         case lw_op:
1064                 if (!access_ok(addr, 4))
1065                         goto sigbus;
1066 
1067                 if (IS_ENABLED(CONFIG_EVA)) {
1068                         if (uaccess_kernel())
1069                                 LoadW(addr, value, res);
1070                         else
1071                                 LoadWE(addr, value, res);
1072                 } else {
1073                         LoadW(addr, value, res);
1074                 }
1075 
1076                 if (res)
1077                         goto fault;
1078                 compute_return_epc(regs);
1079                 regs->regs[insn.i_format.rt] = value;
1080                 break;
1081 
1082         case lhu_op:
1083                 if (!access_ok(addr, 2))
1084                         goto sigbus;
1085 
1086                 if (IS_ENABLED(CONFIG_EVA)) {
1087                         if (uaccess_kernel())
1088                                 LoadHWU(addr, value, res);
1089                         else
1090                                 LoadHWUE(addr, value, res);
1091                 } else {
1092                         LoadHWU(addr, value, res);
1093                 }
1094 
1095                 if (res)
1096                         goto fault;
1097                 compute_return_epc(regs);
1098                 regs->regs[insn.i_format.rt] = value;
1099                 break;
1100 
1101         case lwu_op:
1102 #ifdef CONFIG_64BIT
1103                 /*
1104                  * A 32-bit kernel might be running on a 64-bit processor.  But
1105                  * if we're on a 32-bit processor and an i-cache incoherency
1106                  * or race makes us see a 64-bit instruction here the sdl/sdr
1107                  * would blow up, so for now we don't handle unaligned 64-bit
1108                  * instructions on 32-bit kernels.
1109                  */
1110                 if (!access_ok(addr, 4))
1111                         goto sigbus;
1112 
1113                 LoadWU(addr, value, res);
1114                 if (res)
1115                         goto fault;
1116                 compute_return_epc(regs);
1117                 regs->regs[insn.i_format.rt] = value;
1118                 break;
1119 #endif /* CONFIG_64BIT */
1120 
1121                 /* Cannot handle 64-bit instructions in 32-bit kernel */
1122                 goto sigill;
1123 
1124         case ld_op:
1125 #ifdef CONFIG_64BIT
1126                 /*
1127                  * A 32-bit kernel might be running on a 64-bit processor.  But
1128                  * if we're on a 32-bit processor and an i-cache incoherency
1129                  * or race makes us see a 64-bit instruction here the sdl/sdr
1130                  * would blow up, so for now we don't handle unaligned 64-bit
1131                  * instructions on 32-bit kernels.
1132                  */
1133                 if (!access_ok(addr, 8))
1134                         goto sigbus;
1135 
1136                 LoadDW(addr, value, res);
1137                 if (res)
1138                         goto fault;
1139                 compute_return_epc(regs);
1140                 regs->regs[insn.i_format.rt] = value;
1141                 break;
1142 #endif /* CONFIG_64BIT */
1143 
1144                 /* Cannot handle 64-bit instructions in 32-bit kernel */
1145                 goto sigill;
1146 
1147         case sh_op:
1148                 if (!access_ok(addr, 2))
1149                         goto sigbus;
1150 
1151                 compute_return_epc(regs);
1152                 value = regs->regs[insn.i_format.rt];
1153 
1154                 if (IS_ENABLED(CONFIG_EVA)) {
1155                         if (uaccess_kernel())
1156                                 StoreHW(addr, value, res);
1157                         else
1158                                 StoreHWE(addr, value, res);
1159                 } else {
1160                         StoreHW(addr, value, res);
1161                 }
1162 
1163                 if (res)
1164                         goto fault;
1165                 break;
1166 
1167         case sw_op:
1168                 if (!access_ok(addr, 4))
1169                         goto sigbus;
1170 
1171                 compute_return_epc(regs);
1172                 value = regs->regs[insn.i_format.rt];
1173 
1174                 if (IS_ENABLED(CONFIG_EVA)) {
1175                         if (uaccess_kernel())
1176                                 StoreW(addr, value, res);
1177                         else
1178                                 StoreWE(addr, value, res);
1179                 } else {
1180                         StoreW(addr, value, res);
1181                 }
1182 
1183                 if (res)
1184                         goto fault;
1185                 break;
1186 
1187         case sd_op:
1188 #ifdef CONFIG_64BIT
1189                 /*
1190                  * A 32-bit kernel might be running on a 64-bit processor.  But
1191                  * if we're on a 32-bit processor and an i-cache incoherency
1192                  * or race makes us see a 64-bit instruction here the sdl/sdr
1193                  * would blow up, so for now we don't handle unaligned 64-bit
1194                  * instructions on 32-bit kernels.
1195                  */
1196                 if (!access_ok(addr, 8))
1197                         goto sigbus;
1198 
1199                 compute_return_epc(regs);
1200                 value = regs->regs[insn.i_format.rt];
1201                 StoreDW(addr, value, res);
1202                 if (res)
1203                         goto fault;
1204                 break;
1205 #endif /* CONFIG_64BIT */
1206 
1207                 /* Cannot handle 64-bit instructions in 32-bit kernel */
1208                 goto sigill;
1209 
1210 #ifdef CONFIG_MIPS_FP_SUPPORT
1211 
1212         case lwc1_op:
1213         case ldc1_op:
1214         case swc1_op:
1215         case sdc1_op:
1216         case cop1x_op: {
1217                 void __user *fault_addr = NULL;
1218 
1219                 die_if_kernel("Unaligned FP access in kernel code", regs);
1220                 BUG_ON(!used_math());
1221 
1222                 res = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
1223                                                &fault_addr);
1224                 own_fpu(1);     /* Restore FPU state. */
1225 
1226                 /* Signal if something went wrong. */
1227                 process_fpemu_return(res, fault_addr, 0);
1228 
1229                 if (res == 0)
1230                         break;
1231                 return;
1232         }
1233 #endif /* CONFIG_MIPS_FP_SUPPORT */
1234 
1235 #ifdef CONFIG_CPU_HAS_MSA
1236 
1237         case msa_op: {
1238                 unsigned int wd, preempted;
1239                 enum msa_2b_fmt df;
1240                 union fpureg *fpr;
1241 
1242                 if (!cpu_has_msa)
1243                         goto sigill;
1244 
1245                 /*
1246                  * If we've reached this point then userland should have taken
1247                  * the MSA disabled exception & initialised vector context at
1248                  * some point in the past.
1249                  */
1250                 BUG_ON(!thread_msa_context_live());
1251 
1252                 df = insn.msa_mi10_format.df;
1253                 wd = insn.msa_mi10_format.wd;
1254                 fpr = &current->thread.fpu.fpr[wd];
1255 
1256                 switch (insn.msa_mi10_format.func) {
1257                 case msa_ld_op:
1258                         if (!access_ok(addr, sizeof(*fpr)))
1259                                 goto sigbus;
1260 
1261                         do {
1262                                 /*
1263                                  * If we have live MSA context keep track of
1264                                  * whether we get preempted in order to avoid
1265                                  * the register context we load being clobbered
1266                                  * by the live context as it's saved during
1267                                  * preemption. If we don't have live context
1268                                  * then it can't be saved to clobber the value
1269                                  * we load.
1270                                  */
1271                                 preempted = test_thread_flag(TIF_USEDMSA);
1272 
1273                                 res = __copy_from_user_inatomic(fpr, addr,
1274                                                                 sizeof(*fpr));
1275                                 if (res)
1276                                         goto fault;
1277 
1278                                 /*
1279                                  * Update the hardware register if it is in use
1280                                  * by the task in this quantum, in order to
1281                                  * avoid having to save & restore the whole
1282                                  * vector context.
1283                                  */
1284                                 preempt_disable();
1285                                 if (test_thread_flag(TIF_USEDMSA)) {
1286                                         write_msa_wr(wd, fpr, df);
1287                                         preempted = 0;
1288                                 }
1289                                 preempt_enable();
1290                         } while (preempted);
1291                         break;
1292 
1293                 case msa_st_op:
1294                         if (!access_ok(addr, sizeof(*fpr)))
1295                                 goto sigbus;
1296 
1297                         /*
1298                          * Update from the hardware register if it is in use by
1299                          * the task in this quantum, in order to avoid having to
1300                          * save & restore the whole vector context.
1301                          */
1302                         preempt_disable();
1303                         if (test_thread_flag(TIF_USEDMSA))
1304                                 read_msa_wr(wd, fpr, df);
1305                         preempt_enable();
1306 
1307                         res = __copy_to_user_inatomic(addr, fpr, sizeof(*fpr));
1308                         if (res)
1309                                 goto fault;
1310                         break;
1311 
1312                 default:
1313                         goto sigbus;
1314                 }
1315 
1316                 compute_return_epc(regs);
1317                 break;
1318         }
1319 #endif /* CONFIG_CPU_HAS_MSA */
1320 
1321 #ifndef CONFIG_CPU_MIPSR6
1322         /*
1323          * COP2 is available to implementor for application specific use.
1324          * It's up to applications to register a notifier chain and do
1325          * whatever they have to do, including possible sending of signals.
1326          *
1327          * This instruction has been reallocated in Release 6
1328          */
1329         case lwc2_op:
1330                 cu2_notifier_call_chain(CU2_LWC2_OP, regs);
1331                 break;
1332 
1333         case ldc2_op:
1334                 cu2_notifier_call_chain(CU2_LDC2_OP, regs);
1335                 break;
1336 
1337         case swc2_op:
1338                 cu2_notifier_call_chain(CU2_SWC2_OP, regs);
1339                 break;
1340 
1341         case sdc2_op:
1342                 cu2_notifier_call_chain(CU2_SDC2_OP, regs);
1343                 break;
1344 #endif
1345         default:
1346                 /*
1347                  * Pheeee...  We encountered an yet unknown instruction or
1348                  * cache coherence problem.  Die sucker, die ...
1349                  */
1350                 goto sigill;
1351         }
1352 
1353 #ifdef CONFIG_DEBUG_FS
1354         unaligned_instructions++;
1355 #endif
1356 
1357         return;
1358 
1359 fault:
1360         /* roll back jump/branch */
1361         regs->cp0_epc = origpc;
1362         regs->regs[31] = orig31;
1363         /* Did we have an exception handler installed? */
1364         if (fixup_exception(regs))
1365                 return;
1366 
1367         die_if_kernel("Unhandled kernel unaligned access", regs);
1368         force_sig(SIGSEGV);
1369 
1370         return;
1371 
1372 sigbus:
1373         die_if_kernel("Unhandled kernel unaligned access", regs);
1374         force_sig(SIGBUS);
1375 
1376         return;
1377 
1378 sigill:
1379         die_if_kernel
1380             ("Unhandled kernel unaligned access or invalid instruction", regs);
1381         force_sig(SIGILL);
1382 }
1383 
1384 /* Recode table from 16-bit register notation to 32-bit GPR. */
1385 const int reg16to32[] = { 16, 17, 2, 3, 4, 5, 6, 7 };
1386 
1387 /* Recode table from 16-bit STORE register notation to 32-bit GPR. */
1388 static const int reg16to32st[] = { 0, 17, 2, 3, 4, 5, 6, 7 };
1389 
1390 static void emulate_load_store_microMIPS(struct pt_regs *regs,
1391                                          void __user *addr)
1392 {
1393         unsigned long value;
1394         unsigned int res;
1395         int i;
1396         unsigned int reg = 0, rvar;
1397         unsigned long orig31;
1398         u16 __user *pc16;
1399         u16 halfword;
1400         unsigned int word;
1401         unsigned long origpc, contpc;
1402         union mips_instruction insn;
1403         struct mm_decoded_insn mminsn;
1404 
1405         origpc = regs->cp0_epc;
1406         orig31 = regs->regs[31];
1407 
1408         mminsn.micro_mips_mode = 1;
1409 
1410         /*
1411          * This load never faults.
1412          */
1413         pc16 = (unsigned short __user *)msk_isa16_mode(regs->cp0_epc);
1414         __get_user(halfword, pc16);
1415         pc16++;
1416         contpc = regs->cp0_epc + 2;
1417         word = ((unsigned int)halfword << 16);
1418         mminsn.pc_inc = 2;
1419 
1420         if (!mm_insn_16bit(halfword)) {
1421                 __get_user(halfword, pc16);
1422                 pc16++;
1423                 contpc = regs->cp0_epc + 4;
1424                 mminsn.pc_inc = 4;
1425                 word |= halfword;
1426         }
1427         mminsn.insn = word;
1428 
1429         if (get_user(halfword, pc16))
1430                 goto fault;
1431         mminsn.next_pc_inc = 2;
1432         word = ((unsigned int)halfword << 16);
1433 
1434         if (!mm_insn_16bit(halfword)) {
1435                 pc16++;
1436                 if (get_user(halfword, pc16))
1437                         goto fault;
1438                 mminsn.next_pc_inc = 4;
1439                 word |= halfword;
1440         }
1441         mminsn.next_insn = word;
1442 
1443         insn = (union mips_instruction)(mminsn.insn);
1444         if (mm_isBranchInstr(regs, mminsn, &contpc))
1445                 insn = (union mips_instruction)(mminsn.next_insn);
1446 
1447         /*  Parse instruction to find what to do */
1448 
1449         switch (insn.mm_i_format.opcode) {
1450 
1451         case mm_pool32a_op:
1452                 switch (insn.mm_x_format.func) {
1453                 case mm_lwxs_op:
1454                         reg = insn.mm_x_format.rd;
1455                         goto loadW;
1456                 }
1457 
1458                 goto sigbus;
1459 
1460         case mm_pool32b_op:
1461                 switch (insn.mm_m_format.func) {
1462                 case mm_lwp_func:
1463                         reg = insn.mm_m_format.rd;
1464                         if (reg == 31)
1465                                 goto sigbus;
1466 
1467                         if (!access_ok(addr, 8))
1468                                 goto sigbus;
1469 
1470                         LoadW(addr, value, res);
1471                         if (res)
1472                                 goto fault;
1473                         regs->regs[reg] = value;
1474                         addr += 4;
1475                         LoadW(addr, value, res);
1476                         if (res)
1477                                 goto fault;
1478                         regs->regs[reg + 1] = value;
1479                         goto success;
1480 
1481                 case mm_swp_func:
1482                         reg = insn.mm_m_format.rd;
1483                         if (reg == 31)
1484                                 goto sigbus;
1485 
1486                         if (!access_ok(addr, 8))
1487                                 goto sigbus;
1488 
1489                         value = regs->regs[reg];
1490                         StoreW(addr, value, res);
1491                         if (res)
1492                                 goto fault;
1493                         addr += 4;
1494                         value = regs->regs[reg + 1];
1495                         StoreW(addr, value, res);
1496                         if (res)
1497                                 goto fault;
1498                         goto success;
1499 
1500                 case mm_ldp_func:
1501 #ifdef CONFIG_64BIT
1502                         reg = insn.mm_m_format.rd;
1503                         if (reg == 31)
1504                                 goto sigbus;
1505 
1506                         if (!access_ok(addr, 16))
1507                                 goto sigbus;
1508 
1509                         LoadDW(addr, value, res);
1510                         if (res)
1511                                 goto fault;
1512                         regs->regs[reg] = value;
1513                         addr += 8;
1514                         LoadDW(addr, value, res);
1515                         if (res)
1516                                 goto fault;
1517                         regs->regs[reg + 1] = value;
1518                         goto success;
1519 #endif /* CONFIG_64BIT */
1520 
1521                         goto sigill;
1522 
1523                 case mm_sdp_func:
1524 #ifdef CONFIG_64BIT
1525                         reg = insn.mm_m_format.rd;
1526                         if (reg == 31)
1527                                 goto sigbus;
1528 
1529                         if (!access_ok(addr, 16))
1530                                 goto sigbus;
1531 
1532                         value = regs->regs[reg];
1533                         StoreDW(addr, value, res);
1534                         if (res)
1535                                 goto fault;
1536                         addr += 8;
1537                         value = regs->regs[reg + 1];
1538                         StoreDW(addr, value, res);
1539                         if (res)
1540                                 goto fault;
1541                         goto success;
1542 #endif /* CONFIG_64BIT */
1543 
1544                         goto sigill;
1545 
1546                 case mm_lwm32_func:
1547                         reg = insn.mm_m_format.rd;
1548                         rvar = reg & 0xf;
1549                         if ((rvar > 9) || !reg)
1550                                 goto sigill;
1551                         if (reg & 0x10) {
1552                                 if (!access_ok(addr, 4 * (rvar + 1)))
1553                                         goto sigbus;
1554                         } else {
1555                                 if (!access_ok(addr, 4 * rvar))
1556                                         goto sigbus;
1557                         }
1558                         if (rvar == 9)
1559                                 rvar = 8;
1560                         for (i = 16; rvar; rvar--, i++) {
1561                                 LoadW(addr, value, res);
1562                                 if (res)
1563                                         goto fault;
1564                                 addr += 4;
1565                                 regs->regs[i] = value;
1566                         }
1567                         if ((reg & 0xf) == 9) {
1568                                 LoadW(addr, value, res);
1569                                 if (res)
1570                                         goto fault;
1571                                 addr += 4;
1572                                 regs->regs[30] = value;
1573                         }
1574                         if (reg & 0x10) {
1575                                 LoadW(addr, value, res);
1576                                 if (res)
1577                                         goto fault;
1578                                 regs->regs[31] = value;
1579                         }
1580                         goto success;
1581 
1582                 case mm_swm32_func:
1583                         reg = insn.mm_m_format.rd;
1584                         rvar = reg & 0xf;
1585                         if ((rvar > 9) || !reg)
1586                                 goto sigill;
1587                         if (reg & 0x10) {
1588                                 if (!access_ok(addr, 4 * (rvar + 1)))
1589                                         goto sigbus;
1590                         } else {
1591                                 if (!access_ok(addr, 4 * rvar))
1592                                         goto sigbus;
1593                         }
1594                         if (rvar == 9)
1595                                 rvar = 8;
1596                         for (i = 16; rvar; rvar--, i++) {
1597                                 value = regs->regs[i];
1598                                 StoreW(addr, value, res);
1599                                 if (res)
1600                                         goto fault;
1601                                 addr += 4;
1602                         }
1603                         if ((reg & 0xf) == 9) {
1604                                 value = regs->regs[30];
1605                                 StoreW(addr, value, res);
1606                                 if (res)
1607                                         goto fault;
1608                                 addr += 4;
1609                         }
1610                         if (reg & 0x10) {
1611                                 value = regs->regs[31];
1612                                 StoreW(addr, value, res);
1613                                 if (res)
1614                                         goto fault;
1615                         }
1616                         goto success;
1617 
1618                 case mm_ldm_func:
1619 #ifdef CONFIG_64BIT
1620                         reg = insn.mm_m_format.rd;
1621                         rvar = reg & 0xf;
1622                         if ((rvar > 9) || !reg)
1623                                 goto sigill;
1624                         if (reg & 0x10) {
1625                                 if (!access_ok(addr, 8 * (rvar + 1)))
1626                                         goto sigbus;
1627                         } else {
1628                                 if (!access_ok(addr, 8 * rvar))
1629                                         goto sigbus;
1630                         }
1631                         if (rvar == 9)
1632                                 rvar = 8;
1633 
1634                         for (i = 16; rvar; rvar--, i++) {
1635                                 LoadDW(addr, value, res);
1636                                 if (res)
1637                                         goto fault;
1638                                 addr += 4;
1639                                 regs->regs[i] = value;
1640                         }
1641                         if ((reg & 0xf) == 9) {
1642                                 LoadDW(addr, value, res);
1643                                 if (res)
1644                                         goto fault;
1645                                 addr += 8;
1646                                 regs->regs[30] = value;
1647                         }
1648                         if (reg & 0x10) {
1649                                 LoadDW(addr, value, res);
1650                                 if (res)
1651                                         goto fault;
1652                                 regs->regs[31] = value;
1653                         }
1654                         goto success;
1655 #endif /* CONFIG_64BIT */
1656 
1657                         goto sigill;
1658 
1659                 case mm_sdm_func:
1660 #ifdef CONFIG_64BIT
1661                         reg = insn.mm_m_format.rd;
1662                         rvar = reg & 0xf;
1663                         if ((rvar > 9) || !reg)
1664                                 goto sigill;
1665                         if (reg & 0x10) {
1666                                 if (!access_ok(addr, 8 * (rvar + 1)))
1667                                         goto sigbus;
1668                         } else {
1669                                 if (!access_ok(addr, 8 * rvar))
1670                                         goto sigbus;
1671                         }
1672                         if (rvar == 9)
1673                                 rvar = 8;
1674 
1675                         for (i = 16; rvar; rvar--, i++) {
1676                                 value = regs->regs[i];
1677                                 StoreDW(addr, value, res);
1678                                 if (res)
1679                                         goto fault;
1680                                 addr += 8;
1681                         }
1682                         if ((reg & 0xf) == 9) {
1683                                 value = regs->regs[30];
1684                                 StoreDW(addr, value, res);
1685                                 if (res)
1686                                         goto fault;
1687                                 addr += 8;
1688                         }
1689                         if (reg & 0x10) {
1690                                 value = regs->regs[31];
1691                                 StoreDW(addr, value, res);
1692                                 if (res)
1693                                         goto fault;
1694                         }
1695                         goto success;
1696 #endif /* CONFIG_64BIT */
1697 
1698                         goto sigill;
1699 
1700                         /*  LWC2, SWC2, LDC2, SDC2 are not serviced */
1701                 }
1702 
1703                 goto sigbus;
1704 
1705         case mm_pool32c_op:
1706                 switch (insn.mm_m_format.func) {
1707                 case mm_lwu_func:
1708                         reg = insn.mm_m_format.rd;
1709                         goto loadWU;
1710                 }
1711 
1712                 /*  LL,SC,LLD,SCD are not serviced */
1713                 goto sigbus;
1714 
1715 #ifdef CONFIG_MIPS_FP_SUPPORT
1716         case mm_pool32f_op:
1717                 switch (insn.mm_x_format.func) {
1718                 case mm_lwxc1_func:
1719                 case mm_swxc1_func:
1720                 case mm_ldxc1_func:
1721                 case mm_sdxc1_func:
1722                         goto fpu_emul;
1723                 }
1724 
1725                 goto sigbus;
1726 
1727         case mm_ldc132_op:
1728         case mm_sdc132_op:
1729         case mm_lwc132_op:
1730         case mm_swc132_op: {
1731                 void __user *fault_addr = NULL;
1732 
1733 fpu_emul:
1734                 /* roll back jump/branch */
1735                 regs->cp0_epc = origpc;
1736                 regs->regs[31] = orig31;
1737 
1738                 die_if_kernel("Unaligned FP access in kernel code", regs);
1739                 BUG_ON(!used_math());
1740                 BUG_ON(!is_fpu_owner());
1741 
1742                 res = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
1743                                                &fault_addr);
1744                 own_fpu(1);     /* restore FPU state */
1745 
1746                 /* If something went wrong, signal */
1747                 process_fpemu_return(res, fault_addr, 0);
1748 
1749                 if (res == 0)
1750                         goto success;
1751                 return;
1752         }
1753 #endif /* CONFIG_MIPS_FP_SUPPORT */
1754 
1755         case mm_lh32_op:
1756                 reg = insn.mm_i_format.rt;
1757                 goto loadHW;
1758 
1759         case mm_lhu32_op:
1760                 reg = insn.mm_i_format.rt;
1761                 goto loadHWU;
1762 
1763         case mm_lw32_op:
1764                 reg = insn.mm_i_format.rt;
1765                 goto loadW;
1766 
1767         case mm_sh32_op:
1768                 reg = insn.mm_i_format.rt;
1769                 goto storeHW;
1770 
1771         case mm_sw32_op:
1772                 reg = insn.mm_i_format.rt;
1773                 goto storeW;
1774 
1775         case mm_ld32_op:
1776                 reg = insn.mm_i_format.rt;
1777                 goto loadDW;
1778 
1779         case mm_sd32_op:
1780                 reg = insn.mm_i_format.rt;
1781                 goto storeDW;
1782 
1783         case mm_pool16c_op:
1784                 switch (insn.mm16_m_format.func) {
1785                 case mm_lwm16_op:
1786                         reg = insn.mm16_m_format.rlist;
1787                         rvar = reg + 1;
1788                         if (!access_ok(addr, 4 * rvar))
1789                                 goto sigbus;
1790 
1791                         for (i = 16; rvar; rvar--, i++) {
1792                                 LoadW(addr, value, res);
1793                                 if (res)
1794                                         goto fault;
1795                                 addr += 4;
1796                                 regs->regs[i] = value;
1797                         }
1798                         LoadW(addr, value, res);
1799                         if (res)
1800                                 goto fault;
1801                         regs->regs[31] = value;
1802 
1803                         goto success;
1804 
1805                 case mm_swm16_op:
1806                         reg = insn.mm16_m_format.rlist;
1807                         rvar = reg + 1;
1808                         if (!access_ok(addr, 4 * rvar))
1809                                 goto sigbus;
1810 
1811                         for (i = 16; rvar; rvar--, i++) {
1812                                 value = regs->regs[i];
1813                                 StoreW(addr, value, res);
1814                                 if (res)
1815                                         goto fault;
1816                                 addr += 4;
1817                         }
1818                         value = regs->regs[31];
1819                         StoreW(addr, value, res);
1820                         if (res)
1821                                 goto fault;
1822 
1823                         goto success;
1824 
1825                 }
1826 
1827                 goto sigbus;
1828 
1829         case mm_lhu16_op:
1830                 reg = reg16to32[insn.mm16_rb_format.rt];
1831                 goto loadHWU;
1832 
1833         case mm_lw16_op:
1834                 reg = reg16to32[insn.mm16_rb_format.rt];
1835                 goto loadW;
1836 
1837         case mm_sh16_op:
1838                 reg = reg16to32st[insn.mm16_rb_format.rt];
1839                 goto storeHW;
1840 
1841         case mm_sw16_op:
1842                 reg = reg16to32st[insn.mm16_rb_format.rt];
1843                 goto storeW;
1844 
1845         case mm_lwsp16_op:
1846                 reg = insn.mm16_r5_format.rt;
1847                 goto loadW;
1848 
1849         case mm_swsp16_op:
1850                 reg = insn.mm16_r5_format.rt;
1851                 goto storeW;
1852 
1853         case mm_lwgp16_op:
1854                 reg = reg16to32[insn.mm16_r3_format.rt];
1855                 goto loadW;
1856 
1857         default:
1858                 goto sigill;
1859         }
1860 
1861 loadHW:
1862         if (!access_ok(addr, 2))
1863                 goto sigbus;
1864 
1865         LoadHW(addr, value, res);
1866         if (res)
1867                 goto fault;
1868         regs->regs[reg] = value;
1869         goto success;
1870 
1871 loadHWU:
1872         if (!access_ok(addr, 2))
1873                 goto sigbus;
1874 
1875         LoadHWU(addr, value, res);
1876         if (res)
1877                 goto fault;
1878         regs->regs[reg] = value;
1879         goto success;
1880 
1881 loadW:
1882         if (!access_ok(addr, 4))
1883                 goto sigbus;
1884 
1885         LoadW(addr, value, res);
1886         if (res)
1887                 goto fault;
1888         regs->regs[reg] = value;
1889         goto success;
1890 
1891 loadWU:
1892 #ifdef CONFIG_64BIT
1893         /*
1894          * A 32-bit kernel might be running on a 64-bit processor.  But
1895          * if we're on a 32-bit processor and an i-cache incoherency
1896          * or race makes us see a 64-bit instruction here the sdl/sdr
1897          * would blow up, so for now we don't handle unaligned 64-bit
1898          * instructions on 32-bit kernels.
1899          */
1900         if (!access_ok(addr, 4))
1901                 goto sigbus;
1902 
1903         LoadWU(addr, value, res);
1904         if (res)
1905                 goto fault;
1906         regs->regs[reg] = value;
1907         goto success;
1908 #endif /* CONFIG_64BIT */
1909 
1910         /* Cannot handle 64-bit instructions in 32-bit kernel */
1911         goto sigill;
1912 
1913 loadDW:
1914 #ifdef CONFIG_64BIT
1915         /*
1916          * A 32-bit kernel might be running on a 64-bit processor.  But
1917          * if we're on a 32-bit processor and an i-cache incoherency
1918          * or race makes us see a 64-bit instruction here the sdl/sdr
1919          * would blow up, so for now we don't handle unaligned 64-bit
1920          * instructions on 32-bit kernels.
1921          */
1922         if (!access_ok(addr, 8))
1923                 goto sigbus;
1924 
1925         LoadDW(addr, value, res);
1926         if (res)
1927                 goto fault;
1928         regs->regs[reg] = value;
1929         goto success;
1930 #endif /* CONFIG_64BIT */
1931 
1932         /* Cannot handle 64-bit instructions in 32-bit kernel */
1933         goto sigill;
1934 
1935 storeHW:
1936         if (!access_ok(addr, 2))
1937                 goto sigbus;
1938 
1939         value = regs->regs[reg];
1940         StoreHW(addr, value, res);
1941         if (res)
1942                 goto fault;
1943         goto success;
1944 
1945 storeW:
1946         if (!access_ok(addr, 4))
1947                 goto sigbus;
1948 
1949         value = regs->regs[reg];
1950         StoreW(addr, value, res);
1951         if (res)
1952                 goto fault;
1953         goto success;
1954 
1955 storeDW:
1956 #ifdef CONFIG_64BIT
1957         /*
1958          * A 32-bit kernel might be running on a 64-bit processor.  But
1959          * if we're on a 32-bit processor and an i-cache incoherency
1960          * or race makes us see a 64-bit instruction here the sdl/sdr
1961          * would blow up, so for now we don't handle unaligned 64-bit
1962          * instructions on 32-bit kernels.
1963          */
1964         if (!access_ok(addr, 8))
1965                 goto sigbus;
1966 
1967         value = regs->regs[reg];
1968         StoreDW(addr, value, res);
1969         if (res)
1970                 goto fault;
1971         goto success;
1972 #endif /* CONFIG_64BIT */
1973 
1974         /* Cannot handle 64-bit instructions in 32-bit kernel */
1975         goto sigill;
1976 
1977 success:
1978         regs->cp0_epc = contpc; /* advance or branch */
1979 
1980 #ifdef CONFIG_DEBUG_FS
1981         unaligned_instructions++;
1982 #endif
1983         return;
1984 
1985 fault:
1986         /* roll back jump/branch */
1987         regs->cp0_epc = origpc;
1988         regs->regs[31] = orig31;
1989         /* Did we have an exception handler installed? */
1990         if (fixup_exception(regs))
1991                 return;
1992 
1993         die_if_kernel("Unhandled kernel unaligned access", regs);
1994         force_sig(SIGSEGV);
1995 
1996         return;
1997 
1998 sigbus:
1999         die_if_kernel("Unhandled kernel unaligned access", regs);
2000         force_sig(SIGBUS);
2001 
2002         return;
2003 
2004 sigill:
2005         die_if_kernel
2006             ("Unhandled kernel unaligned access or invalid instruction", regs);
2007         force_sig(SIGILL);
2008 }
2009 
2010 static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr)
2011 {
2012         unsigned long value;
2013         unsigned int res;
2014         int reg;
2015         unsigned long orig31;
2016         u16 __user *pc16;
2017         unsigned long origpc;
2018         union mips16e_instruction mips16inst, oldinst;
2019         unsigned int opcode;
2020         int extended = 0;
2021 
2022         origpc = regs->cp0_epc;
2023         orig31 = regs->regs[31];
2024         pc16 = (unsigned short __user *)msk_isa16_mode(origpc);
2025         /*
2026          * This load never faults.
2027          */
2028         __get_user(mips16inst.full, pc16);
2029         oldinst = mips16inst;
2030 
2031         /* skip EXTEND instruction */
2032         if (mips16inst.ri.opcode == MIPS16e_extend_op) {
2033                 extended = 1;
2034                 pc16++;
2035                 __get_user(mips16inst.full, pc16);
2036         } else if (delay_slot(regs)) {
2037                 /*  skip jump instructions */
2038                 /*  JAL/JALX are 32 bits but have OPCODE in first short int */
2039                 if (mips16inst.ri.opcode == MIPS16e_jal_op)
2040                         pc16++;
2041                 pc16++;
2042                 if (get_user(mips16inst.full, pc16))
2043                         goto sigbus;
2044         }
2045 
2046         opcode = mips16inst.ri.opcode;
2047         switch (opcode) {
2048         case MIPS16e_i64_op:    /* I64 or RI64 instruction */
2049                 switch (mips16inst.i64.func) {  /* I64/RI64 func field check */
2050                 case MIPS16e_ldpc_func:
2051                 case MIPS16e_ldsp_func:
2052                         reg = reg16to32[mips16inst.ri64.ry];
2053                         goto loadDW;
2054 
2055                 case MIPS16e_sdsp_func:
2056                         reg = reg16to32[mips16inst.ri64.ry];
2057                         goto writeDW;
2058 
2059                 case MIPS16e_sdrasp_func:
2060                         reg = 29;       /* GPRSP */
2061                         goto writeDW;
2062                 }
2063 
2064                 goto sigbus;
2065 
2066         case MIPS16e_swsp_op:
2067                 reg = reg16to32[mips16inst.ri.rx];
2068                 if (extended && cpu_has_mips16e2)
2069                         switch (mips16inst.ri.imm >> 5) {
2070                         case 0:         /* SWSP */
2071                         case 1:         /* SWGP */
2072                                 break;
2073                         case 2:         /* SHGP */
2074                                 opcode = MIPS16e_sh_op;
2075                                 break;
2076                         default:
2077                                 goto sigbus;
2078                         }
2079                 break;
2080 
2081         case MIPS16e_lwpc_op:
2082                 reg = reg16to32[mips16inst.ri.rx];
2083                 break;
2084 
2085         case MIPS16e_lwsp_op:
2086                 reg = reg16to32[mips16inst.ri.rx];
2087                 if (extended && cpu_has_mips16e2)
2088                         switch (mips16inst.ri.imm >> 5) {
2089                         case 0:         /* LWSP */
2090                         case 1:         /* LWGP */
2091                                 break;
2092                         case 2:         /* LHGP */
2093                                 opcode = MIPS16e_lh_op;
2094                                 break;
2095                         case 4:         /* LHUGP */
2096                                 opcode = MIPS16e_lhu_op;
2097                                 break;
2098                         default:
2099                                 goto sigbus;
2100                         }
2101                 break;
2102 
2103         case MIPS16e_i8_op:
2104                 if (mips16inst.i8.func != MIPS16e_swrasp_func)
2105                         goto sigbus;
2106                 reg = 29;       /* GPRSP */
2107                 break;
2108 
2109         default:
2110                 reg = reg16to32[mips16inst.rri.ry];
2111                 break;
2112         }
2113 
2114         switch (opcode) {
2115 
2116         case MIPS16e_lb_op:
2117         case MIPS16e_lbu_op:
2118         case MIPS16e_sb_op:
2119                 goto sigbus;
2120 
2121         case MIPS16e_lh_op:
2122                 if (!access_ok(addr, 2))
2123                         goto sigbus;
2124 
2125                 LoadHW(addr, value, res);
2126                 if (res)
2127                         goto fault;
2128                 MIPS16e_compute_return_epc(regs, &oldinst);
2129                 regs->regs[reg] = value;
2130                 break;
2131 
2132         case MIPS16e_lhu_op:
2133                 if (!access_ok(addr, 2))
2134                         goto sigbus;
2135 
2136                 LoadHWU(addr, value, res);
2137                 if (res)
2138                         goto fault;
2139                 MIPS16e_compute_return_epc(regs, &oldinst);
2140                 regs->regs[reg] = value;
2141                 break;
2142 
2143         case MIPS16e_lw_op:
2144         case MIPS16e_lwpc_op:
2145         case MIPS16e_lwsp_op:
2146                 if (!access_ok(addr, 4))
2147                         goto sigbus;
2148 
2149                 LoadW(addr, value, res);
2150                 if (res)
2151                         goto fault;
2152                 MIPS16e_compute_return_epc(regs, &oldinst);
2153                 regs->regs[reg] = value;
2154                 break;
2155 
2156         case MIPS16e_lwu_op:
2157 #ifdef CONFIG_64BIT
2158                 /*
2159                  * A 32-bit kernel might be running on a 64-bit processor.  But
2160                  * if we're on a 32-bit processor and an i-cache incoherency
2161                  * or race makes us see a 64-bit instruction here the sdl/sdr
2162                  * would blow up, so for now we don't handle unaligned 64-bit
2163                  * instructions on 32-bit kernels.
2164                  */
2165                 if (!access_ok(addr, 4))
2166                         goto sigbus;
2167 
2168                 LoadWU(addr, value, res);
2169                 if (res)
2170                         goto fault;
2171                 MIPS16e_compute_return_epc(regs, &oldinst);
2172                 regs->regs[reg] = value;
2173                 break;
2174 #endif /* CONFIG_64BIT */
2175 
2176                 /* Cannot handle 64-bit instructions in 32-bit kernel */
2177                 goto sigill;
2178 
2179         case MIPS16e_ld_op:
2180 loadDW:
2181 #ifdef CONFIG_64BIT
2182                 /*
2183                  * A 32-bit kernel might be running on a 64-bit processor.  But
2184                  * if we're on a 32-bit processor and an i-cache incoherency
2185                  * or race makes us see a 64-bit instruction here the sdl/sdr
2186                  * would blow up, so for now we don't handle unaligned 64-bit
2187                  * instructions on 32-bit kernels.
2188                  */
2189                 if (!access_ok(addr, 8))
2190                         goto sigbus;
2191 
2192                 LoadDW(addr, value, res);
2193                 if (res)
2194                         goto fault;
2195                 MIPS16e_compute_return_epc(regs, &oldinst);
2196                 regs->regs[reg] = value;
2197                 break;
2198 #endif /* CONFIG_64BIT */
2199 
2200                 /* Cannot handle 64-bit instructions in 32-bit kernel */
2201                 goto sigill;
2202 
2203         case MIPS16e_sh_op:
2204                 if (!access_ok(addr, 2))
2205                         goto sigbus;
2206 
2207                 MIPS16e_compute_return_epc(regs, &oldinst);
2208                 value = regs->regs[reg];
2209                 StoreHW(addr, value, res);
2210                 if (res)
2211                         goto fault;
2212                 break;
2213 
2214         case MIPS16e_sw_op:
2215         case MIPS16e_swsp_op:
2216         case MIPS16e_i8_op:     /* actually - MIPS16e_swrasp_func */
2217                 if (!access_ok(addr, 4))
2218                         goto sigbus;
2219 
2220                 MIPS16e_compute_return_epc(regs, &oldinst);
2221                 value = regs->regs[reg];
2222                 StoreW(addr, value, res);
2223                 if (res)
2224                         goto fault;
2225                 break;
2226 
2227         case MIPS16e_sd_op:
2228 writeDW:
2229 #ifdef CONFIG_64BIT
2230                 /*
2231                  * A 32-bit kernel might be running on a 64-bit processor.  But
2232                  * if we're on a 32-bit processor and an i-cache incoherency
2233                  * or race makes us see a 64-bit instruction here the sdl/sdr
2234                  * would blow up, so for now we don't handle unaligned 64-bit
2235                  * instructions on 32-bit kernels.
2236                  */
2237                 if (!access_ok(addr, 8))
2238                         goto sigbus;
2239 
2240                 MIPS16e_compute_return_epc(regs, &oldinst);
2241                 value = regs->regs[reg];
2242                 StoreDW(addr, value, res);
2243                 if (res)
2244                         goto fault;
2245                 break;
2246 #endif /* CONFIG_64BIT */
2247 
2248                 /* Cannot handle 64-bit instructions in 32-bit kernel */
2249                 goto sigill;
2250 
2251         default:
2252                 /*
2253                  * Pheeee...  We encountered an yet unknown instruction or
2254                  * cache coherence problem.  Die sucker, die ...
2255                  */
2256                 goto sigill;
2257         }
2258 
2259 #ifdef CONFIG_DEBUG_FS
2260         unaligned_instructions++;
2261 #endif
2262 
2263         return;
2264 
2265 fault:
2266         /* roll back jump/branch */
2267         regs->cp0_epc = origpc;
2268         regs->regs[31] = orig31;
2269         /* Did we have an exception handler installed? */
2270         if (fixup_exception(regs))
2271                 return;
2272 
2273         die_if_kernel("Unhandled kernel unaligned access", regs);
2274         force_sig(SIGSEGV);
2275 
2276         return;
2277 
2278 sigbus:
2279         die_if_kernel("Unhandled kernel unaligned access", regs);
2280         force_sig(SIGBUS);
2281 
2282         return;
2283 
2284 sigill:
2285         die_if_kernel
2286             ("Unhandled kernel unaligned access or invalid instruction", regs);
2287         force_sig(SIGILL);
2288 }
2289 
2290 asmlinkage void do_ade(struct pt_regs *regs)
2291 {
2292         enum ctx_state prev_state;
2293         unsigned int __user *pc;
2294         mm_segment_t seg;
2295 
2296         prev_state = exception_enter();
2297         perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS,
2298                         1, regs, regs->cp0_badvaddr);
2299         /*
2300          * Did we catch a fault trying to load an instruction?
2301          */
2302         if (regs->cp0_badvaddr == regs->cp0_epc)
2303                 goto sigbus;
2304 
2305         if (user_mode(regs) && !test_thread_flag(TIF_FIXADE))
2306                 goto sigbus;
2307         if (unaligned_action == UNALIGNED_ACTION_SIGNAL)
2308                 goto sigbus;
2309 
2310         /*
2311          * Do branch emulation only if we didn't forward the exception.
2312          * This is all so but ugly ...
2313          */
2314 
2315         /*
2316          * Are we running in microMIPS mode?
2317          */
2318         if (get_isa16_mode(regs->cp0_epc)) {
2319                 /*
2320                  * Did we catch a fault trying to load an instruction in
2321                  * 16-bit mode?
2322                  */
2323                 if (regs->cp0_badvaddr == msk_isa16_mode(regs->cp0_epc))
2324                         goto sigbus;
2325                 if (unaligned_action == UNALIGNED_ACTION_SHOW)
2326                         show_registers(regs);
2327 
2328                 if (cpu_has_mmips) {
2329                         seg = get_fs();
2330                         if (!user_mode(regs))
2331                                 set_fs(KERNEL_DS);
2332                         emulate_load_store_microMIPS(regs,
2333                                 (void __user *)regs->cp0_badvaddr);
2334                         set_fs(seg);
2335 
2336                         return;
2337                 }
2338 
2339                 if (cpu_has_mips16) {
2340                         seg = get_fs();
2341                         if (!user_mode(regs))
2342                                 set_fs(KERNEL_DS);
2343                         emulate_load_store_MIPS16e(regs,
2344                                 (void __user *)regs->cp0_badvaddr);
2345                         set_fs(seg);
2346 
2347                         return;
2348                 }
2349 
2350                 goto sigbus;
2351         }
2352 
2353         if (unaligned_action == UNALIGNED_ACTION_SHOW)
2354                 show_registers(regs);
2355         pc = (unsigned int __user *)exception_epc(regs);
2356 
2357         seg = get_fs();
2358         if (!user_mode(regs))
2359                 set_fs(KERNEL_DS);
2360         emulate_load_store_insn(regs, (void __user *)regs->cp0_badvaddr, pc);
2361         set_fs(seg);
2362 
2363         return;
2364 
2365 sigbus:
2366         die_if_kernel("Kernel unaligned instruction access", regs);
2367         force_sig(SIGBUS);
2368 
2369         /*
2370          * XXX On return from the signal handler we should advance the epc
2371          */
2372         exception_exit(prev_state);
2373 }
2374 
2375 #ifdef CONFIG_DEBUG_FS
2376 static int __init debugfs_unaligned(void)
2377 {
2378         debugfs_create_u32("unaligned_instructions", S_IRUGO, mips_debugfs_dir,
2379                            &unaligned_instructions);
2380         debugfs_create_u32("unaligned_action", S_IRUGO | S_IWUSR,
2381                            mips_debugfs_dir, &unaligned_action);
2382         return 0;
2383 }
2384 arch_initcall(debugfs_unaligned);
2385 #endif
2386 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp