~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/mips/kernel/unaligned.c

Version: ~ [ linux-4.20-rc6 ] ~ [ linux-4.19.8 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.87 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.144 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.166 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.128 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.61 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.31.14 ] ~ [ linux-2.6.30.10 ] ~ [ linux-2.6.29.6 ] ~ [ linux-2.6.28.10 ] ~ [ linux-2.6.27.62 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Handle unaligned accesses by emulation.
  3  *
  4  * This file is subject to the terms and conditions of the GNU General Public
  5  * License.  See the file "COPYING" in the main directory of this archive
  6  * for more details.
  7  *
  8  * Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle
  9  * Copyright (C) 1999 Silicon Graphics, Inc.
 10  * Copyright (C) 2014 Imagination Technologies Ltd.
 11  *
 12  * This file contains exception handler for address error exception with the
 13  * special capability to execute faulting instructions in software.  The
 14  * handler does not try to handle the case when the program counter points
 15  * to an address not aligned to a word boundary.
 16  *
 17  * Putting data to unaligned addresses is a bad practice even on Intel where
 18  * only the performance is affected.  Much worse is that such code is non-
 19  * portable.  Due to several programs that die on MIPS due to alignment
 20  * problems I decided to implement this handler anyway though I originally
 21  * didn't intend to do this at all for user code.
 22  *
 23  * For now I enable fixing of address errors by default to make life easier.
 24  * I however intend to disable this somewhen in the future when the alignment
 25  * problems with user programs have been fixed.  For programmers this is the
 26  * right way to go.
 27  *
 28  * Fixing address errors is a per process option.  The option is inherited
 29  * across fork(2) and execve(2) calls.  If you really want to use the
 30  * option in your user programs - I discourage the use of the software
 31  * emulation strongly - use the following code in your userland stuff:
 32  *
 33  * #include <sys/sysmips.h>
 34  *
 35  * ...
 36  * sysmips(MIPS_FIXADE, x);
 37  * ...
 38  *
 39  * The argument x is 0 for disabling software emulation, enabled otherwise.
 40  *
 41  * Below a little program to play around with this feature.
 42  *
 43  * #include <stdio.h>
 44  * #include <sys/sysmips.h>
 45  *
 46  * struct foo {
 47  *         unsigned char bar[8];
 48  * };
 49  *
 50  * main(int argc, char *argv[])
 51  * {
 52  *         struct foo x = {0, 1, 2, 3, 4, 5, 6, 7};
 53  *         unsigned int *p = (unsigned int *) (x.bar + 3);
 54  *         int i;
 55  *
 56  *         if (argc > 1)
 57  *                 sysmips(MIPS_FIXADE, atoi(argv[1]));
 58  *
 59  *         printf("*p = %08lx\n", *p);
 60  *
 61  *         *p = 0xdeadface;
 62  *
 63  *         for(i = 0; i <= 7; i++)
 64  *         printf("%02x ", x.bar[i]);
 65  *         printf("\n");
 66  * }
 67  *
 68  * Coprocessor loads are not supported; I think this case is unimportant
 69  * in the practice.
 70  *
 71  * TODO: Handle ndc (attempted store to doubleword in uncached memory)
 72  *       exception for the R6000.
 73  *       A store crossing a page boundary might be executed only partially.
 74  *       Undo the partial store in this case.
 75  */
 76 #include <linux/context_tracking.h>
 77 #include <linux/mm.h>
 78 #include <linux/signal.h>
 79 #include <linux/smp.h>
 80 #include <linux/sched.h>
 81 #include <linux/debugfs.h>
 82 #include <linux/perf_event.h>
 83 
 84 #include <asm/asm.h>
 85 #include <asm/branch.h>
 86 #include <asm/byteorder.h>
 87 #include <asm/cop2.h>
 88 #include <asm/debug.h>
 89 #include <asm/fpu.h>
 90 #include <asm/fpu_emulator.h>
 91 #include <asm/inst.h>
 92 #include <linux/uaccess.h>
 93 
 94 #define STR(x)  __STR(x)
 95 #define __STR(x)  #x
 96 
 97 enum {
 98         UNALIGNED_ACTION_QUIET,
 99         UNALIGNED_ACTION_SIGNAL,
100         UNALIGNED_ACTION_SHOW,
101 };
102 #ifdef CONFIG_DEBUG_FS
103 static u32 unaligned_instructions;
104 static u32 unaligned_action;
105 #else
106 #define unaligned_action UNALIGNED_ACTION_QUIET
107 #endif
108 extern void show_registers(struct pt_regs *regs);
109 
110 #ifdef __BIG_ENDIAN
111 #define     _LoadHW(addr, value, res, type)  \
112 do {                                                        \
113                 __asm__ __volatile__ (".set\tnoat\n"        \
114                         "1:\t"type##_lb("%0", "0(%2)")"\n"  \
115                         "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\
116                         "sll\t%0, 0x8\n\t"                  \
117                         "or\t%0, $1\n\t"                    \
118                         "li\t%1, 0\n"                       \
119                         "3:\t.set\tat\n\t"                  \
120                         ".insn\n\t"                         \
121                         ".section\t.fixup,\"ax\"\n\t"       \
122                         "4:\tli\t%1, %3\n\t"                \
123                         "j\t3b\n\t"                         \
124                         ".previous\n\t"                     \
125                         ".section\t__ex_table,\"a\"\n\t"    \
126                         STR(PTR)"\t1b, 4b\n\t"              \
127                         STR(PTR)"\t2b, 4b\n\t"              \
128                         ".previous"                         \
129                         : "=&r" (value), "=r" (res)         \
130                         : "r" (addr), "i" (-EFAULT));       \
131 } while(0)
132 
133 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
134 #define     _LoadW(addr, value, res, type)   \
135 do {                                                        \
136                 __asm__ __volatile__ (                      \
137                         "1:\t"type##_lwl("%0", "(%2)")"\n"   \
138                         "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\
139                         "li\t%1, 0\n"                       \
140                         "3:\n\t"                            \
141                         ".insn\n\t"                         \
142                         ".section\t.fixup,\"ax\"\n\t"       \
143                         "4:\tli\t%1, %3\n\t"                \
144                         "j\t3b\n\t"                         \
145                         ".previous\n\t"                     \
146                         ".section\t__ex_table,\"a\"\n\t"    \
147                         STR(PTR)"\t1b, 4b\n\t"              \
148                         STR(PTR)"\t2b, 4b\n\t"              \
149                         ".previous"                         \
150                         : "=&r" (value), "=r" (res)         \
151                         : "r" (addr), "i" (-EFAULT));       \
152 } while(0)
153 
154 #else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
155 /* For CPUs without lwl instruction */
156 #define     _LoadW(addr, value, res, type) \
157 do {                                                        \
158                 __asm__ __volatile__ (                      \
159                         ".set\tpush\n"                      \
160                         ".set\tnoat\n\t"                    \
161                         "1:"type##_lb("%0", "0(%2)")"\n\t"  \
162                         "2:"type##_lbu("$1", "1(%2)")"\n\t" \
163                         "sll\t%0, 0x8\n\t"                  \
164                         "or\t%0, $1\n\t"                    \
165                         "3:"type##_lbu("$1", "2(%2)")"\n\t" \
166                         "sll\t%0, 0x8\n\t"                  \
167                         "or\t%0, $1\n\t"                    \
168                         "4:"type##_lbu("$1", "3(%2)")"\n\t" \
169                         "sll\t%0, 0x8\n\t"                  \
170                         "or\t%0, $1\n\t"                    \
171                         "li\t%1, 0\n"                       \
172                         ".set\tpop\n"                       \
173                         "10:\n\t"                           \
174                         ".insn\n\t"                         \
175                         ".section\t.fixup,\"ax\"\n\t"       \
176                         "11:\tli\t%1, %3\n\t"               \
177                         "j\t10b\n\t"                        \
178                         ".previous\n\t"                     \
179                         ".section\t__ex_table,\"a\"\n\t"    \
180                         STR(PTR)"\t1b, 11b\n\t"             \
181                         STR(PTR)"\t2b, 11b\n\t"             \
182                         STR(PTR)"\t3b, 11b\n\t"             \
183                         STR(PTR)"\t4b, 11b\n\t"             \
184                         ".previous"                         \
185                         : "=&r" (value), "=r" (res)         \
186                         : "r" (addr), "i" (-EFAULT));       \
187 } while(0)
188 
189 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
190 
191 #define     _LoadHWU(addr, value, res, type) \
192 do {                                                        \
193                 __asm__ __volatile__ (                      \
194                         ".set\tnoat\n"                      \
195                         "1:\t"type##_lbu("%0", "0(%2)")"\n" \
196                         "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\
197                         "sll\t%0, 0x8\n\t"                  \
198                         "or\t%0, $1\n\t"                    \
199                         "li\t%1, 0\n"                       \
200                         "3:\n\t"                            \
201                         ".insn\n\t"                         \
202                         ".set\tat\n\t"                      \
203                         ".section\t.fixup,\"ax\"\n\t"       \
204                         "4:\tli\t%1, %3\n\t"                \
205                         "j\t3b\n\t"                         \
206                         ".previous\n\t"                     \
207                         ".section\t__ex_table,\"a\"\n\t"    \
208                         STR(PTR)"\t1b, 4b\n\t"              \
209                         STR(PTR)"\t2b, 4b\n\t"              \
210                         ".previous"                         \
211                         : "=&r" (value), "=r" (res)         \
212                         : "r" (addr), "i" (-EFAULT));       \
213 } while(0)
214 
215 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
216 #define     _LoadWU(addr, value, res, type)  \
217 do {                                                        \
218                 __asm__ __volatile__ (                      \
219                         "1:\t"type##_lwl("%0", "(%2)")"\n"  \
220                         "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\
221                         "dsll\t%0, %0, 32\n\t"              \
222                         "dsrl\t%0, %0, 32\n\t"              \
223                         "li\t%1, 0\n"                       \
224                         "3:\n\t"                            \
225                         ".insn\n\t"                         \
226                         "\t.section\t.fixup,\"ax\"\n\t"     \
227                         "4:\tli\t%1, %3\n\t"                \
228                         "j\t3b\n\t"                         \
229                         ".previous\n\t"                     \
230                         ".section\t__ex_table,\"a\"\n\t"    \
231                         STR(PTR)"\t1b, 4b\n\t"              \
232                         STR(PTR)"\t2b, 4b\n\t"              \
233                         ".previous"                         \
234                         : "=&r" (value), "=r" (res)         \
235                         : "r" (addr), "i" (-EFAULT));       \
236 } while(0)
237 
238 #define     _LoadDW(addr, value, res)  \
239 do {                                                        \
240                 __asm__ __volatile__ (                      \
241                         "1:\tldl\t%0, (%2)\n"               \
242                         "2:\tldr\t%0, 7(%2)\n\t"            \
243                         "li\t%1, 0\n"                       \
244                         "3:\n\t"                            \
245                         ".insn\n\t"                         \
246                         "\t.section\t.fixup,\"ax\"\n\t"     \
247                         "4:\tli\t%1, %3\n\t"                \
248                         "j\t3b\n\t"                         \
249                         ".previous\n\t"                     \
250                         ".section\t__ex_table,\"a\"\n\t"    \
251                         STR(PTR)"\t1b, 4b\n\t"              \
252                         STR(PTR)"\t2b, 4b\n\t"              \
253                         ".previous"                         \
254                         : "=&r" (value), "=r" (res)         \
255                         : "r" (addr), "i" (-EFAULT));       \
256 } while(0)
257 
258 #else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
259 /* For CPUs without lwl and ldl instructions */
260 #define     _LoadWU(addr, value, res, type) \
261 do {                                                        \
262                 __asm__ __volatile__ (                      \
263                         ".set\tpush\n\t"                    \
264                         ".set\tnoat\n\t"                    \
265                         "1:"type##_lbu("%0", "0(%2)")"\n\t" \
266                         "2:"type##_lbu("$1", "1(%2)")"\n\t" \
267                         "sll\t%0, 0x8\n\t"                  \
268                         "or\t%0, $1\n\t"                    \
269                         "3:"type##_lbu("$1", "2(%2)")"\n\t" \
270                         "sll\t%0, 0x8\n\t"                  \
271                         "or\t%0, $1\n\t"                    \
272                         "4:"type##_lbu("$1", "3(%2)")"\n\t" \
273                         "sll\t%0, 0x8\n\t"                  \
274                         "or\t%0, $1\n\t"                    \
275                         "li\t%1, 0\n"                       \
276                         ".set\tpop\n"                       \
277                         "10:\n\t"                           \
278                         ".insn\n\t"                         \
279                         ".section\t.fixup,\"ax\"\n\t"       \
280                         "11:\tli\t%1, %3\n\t"               \
281                         "j\t10b\n\t"                        \
282                         ".previous\n\t"                     \
283                         ".section\t__ex_table,\"a\"\n\t"    \
284                         STR(PTR)"\t1b, 11b\n\t"             \
285                         STR(PTR)"\t2b, 11b\n\t"             \
286                         STR(PTR)"\t3b, 11b\n\t"             \
287                         STR(PTR)"\t4b, 11b\n\t"             \
288                         ".previous"                         \
289                         : "=&r" (value), "=r" (res)         \
290                         : "r" (addr), "i" (-EFAULT));       \
291 } while(0)
292 
293 #define     _LoadDW(addr, value, res)  \
294 do {                                                        \
295                 __asm__ __volatile__ (                      \
296                         ".set\tpush\n\t"                    \
297                         ".set\tnoat\n\t"                    \
298                         "1:lb\t%0, 0(%2)\n\t"               \
299                         "2:lbu\t $1, 1(%2)\n\t"             \
300                         "dsll\t%0, 0x8\n\t"                 \
301                         "or\t%0, $1\n\t"                    \
302                         "3:lbu\t$1, 2(%2)\n\t"              \
303                         "dsll\t%0, 0x8\n\t"                 \
304                         "or\t%0, $1\n\t"                    \
305                         "4:lbu\t$1, 3(%2)\n\t"              \
306                         "dsll\t%0, 0x8\n\t"                 \
307                         "or\t%0, $1\n\t"                    \
308                         "5:lbu\t$1, 4(%2)\n\t"              \
309                         "dsll\t%0, 0x8\n\t"                 \
310                         "or\t%0, $1\n\t"                    \
311                         "6:lbu\t$1, 5(%2)\n\t"              \
312                         "dsll\t%0, 0x8\n\t"                 \
313                         "or\t%0, $1\n\t"                    \
314                         "7:lbu\t$1, 6(%2)\n\t"              \
315                         "dsll\t%0, 0x8\n\t"                 \
316                         "or\t%0, $1\n\t"                    \
317                         "8:lbu\t$1, 7(%2)\n\t"              \
318                         "dsll\t%0, 0x8\n\t"                 \
319                         "or\t%0, $1\n\t"                    \
320                         "li\t%1, 0\n"                       \
321                         ".set\tpop\n\t"                     \
322                         "10:\n\t"                           \
323                         ".insn\n\t"                         \
324                         ".section\t.fixup,\"ax\"\n\t"       \
325                         "11:\tli\t%1, %3\n\t"               \
326                         "j\t10b\n\t"                        \
327                         ".previous\n\t"                     \
328                         ".section\t__ex_table,\"a\"\n\t"    \
329                         STR(PTR)"\t1b, 11b\n\t"             \
330                         STR(PTR)"\t2b, 11b\n\t"             \
331                         STR(PTR)"\t3b, 11b\n\t"             \
332                         STR(PTR)"\t4b, 11b\n\t"             \
333                         STR(PTR)"\t5b, 11b\n\t"             \
334                         STR(PTR)"\t6b, 11b\n\t"             \
335                         STR(PTR)"\t7b, 11b\n\t"             \
336                         STR(PTR)"\t8b, 11b\n\t"             \
337                         ".previous"                         \
338                         : "=&r" (value), "=r" (res)         \
339                         : "r" (addr), "i" (-EFAULT));       \
340 } while(0)
341 
342 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
343 
344 
345 #define     _StoreHW(addr, value, res, type) \
346 do {                                                        \
347                 __asm__ __volatile__ (                      \
348                         ".set\tnoat\n"                      \
349                         "1:\t"type##_sb("%1", "1(%2)")"\n"  \
350                         "srl\t$1, %1, 0x8\n"                \
351                         "2:\t"type##_sb("$1", "0(%2)")"\n"  \
352                         ".set\tat\n\t"                      \
353                         "li\t%0, 0\n"                       \
354                         "3:\n\t"                            \
355                         ".insn\n\t"                         \
356                         ".section\t.fixup,\"ax\"\n\t"       \
357                         "4:\tli\t%0, %3\n\t"                \
358                         "j\t3b\n\t"                         \
359                         ".previous\n\t"                     \
360                         ".section\t__ex_table,\"a\"\n\t"    \
361                         STR(PTR)"\t1b, 4b\n\t"              \
362                         STR(PTR)"\t2b, 4b\n\t"              \
363                         ".previous"                         \
364                         : "=r" (res)                        \
365                         : "r" (value), "r" (addr), "i" (-EFAULT));\
366 } while(0)
367 
368 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
369 #define     _StoreW(addr, value, res, type)  \
370 do {                                                        \
371                 __asm__ __volatile__ (                      \
372                         "1:\t"type##_swl("%1", "(%2)")"\n"  \
373                         "2:\t"type##_swr("%1", "3(%2)")"\n\t"\
374                         "li\t%0, 0\n"                       \
375                         "3:\n\t"                            \
376                         ".insn\n\t"                         \
377                         ".section\t.fixup,\"ax\"\n\t"       \
378                         "4:\tli\t%0, %3\n\t"                \
379                         "j\t3b\n\t"                         \
380                         ".previous\n\t"                     \
381                         ".section\t__ex_table,\"a\"\n\t"    \
382                         STR(PTR)"\t1b, 4b\n\t"              \
383                         STR(PTR)"\t2b, 4b\n\t"              \
384                         ".previous"                         \
385                 : "=r" (res)                                \
386                 : "r" (value), "r" (addr), "i" (-EFAULT));  \
387 } while(0)
388 
389 #define     _StoreDW(addr, value, res) \
390 do {                                                        \
391                 __asm__ __volatile__ (                      \
392                         "1:\tsdl\t%1,(%2)\n"                \
393                         "2:\tsdr\t%1, 7(%2)\n\t"            \
394                         "li\t%0, 0\n"                       \
395                         "3:\n\t"                            \
396                         ".insn\n\t"                         \
397                         ".section\t.fixup,\"ax\"\n\t"       \
398                         "4:\tli\t%0, %3\n\t"                \
399                         "j\t3b\n\t"                         \
400                         ".previous\n\t"                     \
401                         ".section\t__ex_table,\"a\"\n\t"    \
402                         STR(PTR)"\t1b, 4b\n\t"              \
403                         STR(PTR)"\t2b, 4b\n\t"              \
404                         ".previous"                         \
405                 : "=r" (res)                                \
406                 : "r" (value), "r" (addr), "i" (-EFAULT));  \
407 } while(0)
408 
409 #else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
410 #define     _StoreW(addr, value, res, type)  \
411 do {                                                        \
412                 __asm__ __volatile__ (                      \
413                         ".set\tpush\n\t"                    \
414                         ".set\tnoat\n\t"                    \
415                         "1:"type##_sb("%1", "3(%2)")"\n\t"  \
416                         "srl\t$1, %1, 0x8\n\t"              \
417                         "2:"type##_sb("$1", "2(%2)")"\n\t"  \
418                         "srl\t$1, $1,  0x8\n\t"             \
419                         "3:"type##_sb("$1", "1(%2)")"\n\t"  \
420                         "srl\t$1, $1, 0x8\n\t"              \
421                         "4:"type##_sb("$1", "0(%2)")"\n\t"  \
422                         ".set\tpop\n\t"                     \
423                         "li\t%0, 0\n"                       \
424                         "10:\n\t"                           \
425                         ".insn\n\t"                         \
426                         ".section\t.fixup,\"ax\"\n\t"       \
427                         "11:\tli\t%0, %3\n\t"               \
428                         "j\t10b\n\t"                        \
429                         ".previous\n\t"                     \
430                         ".section\t__ex_table,\"a\"\n\t"    \
431                         STR(PTR)"\t1b, 11b\n\t"             \
432                         STR(PTR)"\t2b, 11b\n\t"             \
433                         STR(PTR)"\t3b, 11b\n\t"             \
434                         STR(PTR)"\t4b, 11b\n\t"             \
435                         ".previous"                         \
436                 : "=&r" (res)                               \
437                 : "r" (value), "r" (addr), "i" (-EFAULT)    \
438                 : "memory");                                \
439 } while(0)
440 
441 #define     _StoreDW(addr, value, res) \
442 do {                                                        \
443                 __asm__ __volatile__ (                      \
444                         ".set\tpush\n\t"                    \
445                         ".set\tnoat\n\t"                    \
446                         "1:sb\t%1, 7(%2)\n\t"               \
447                         "dsrl\t$1, %1, 0x8\n\t"             \
448                         "2:sb\t$1, 6(%2)\n\t"               \
449                         "dsrl\t$1, $1, 0x8\n\t"             \
450                         "3:sb\t$1, 5(%2)\n\t"               \
451                         "dsrl\t$1, $1, 0x8\n\t"             \
452                         "4:sb\t$1, 4(%2)\n\t"               \
453                         "dsrl\t$1, $1, 0x8\n\t"             \
454                         "5:sb\t$1, 3(%2)\n\t"               \
455                         "dsrl\t$1, $1, 0x8\n\t"             \
456                         "6:sb\t$1, 2(%2)\n\t"               \
457                         "dsrl\t$1, $1, 0x8\n\t"             \
458                         "7:sb\t$1, 1(%2)\n\t"               \
459                         "dsrl\t$1, $1, 0x8\n\t"             \
460                         "8:sb\t$1, 0(%2)\n\t"               \
461                         "dsrl\t$1, $1, 0x8\n\t"             \
462                         ".set\tpop\n\t"                     \
463                         "li\t%0, 0\n"                       \
464                         "10:\n\t"                           \
465                         ".insn\n\t"                         \
466                         ".section\t.fixup,\"ax\"\n\t"       \
467                         "11:\tli\t%0, %3\n\t"               \
468                         "j\t10b\n\t"                        \
469                         ".previous\n\t"                     \
470                         ".section\t__ex_table,\"a\"\n\t"    \
471                         STR(PTR)"\t1b, 11b\n\t"             \
472                         STR(PTR)"\t2b, 11b\n\t"             \
473                         STR(PTR)"\t3b, 11b\n\t"             \
474                         STR(PTR)"\t4b, 11b\n\t"             \
475                         STR(PTR)"\t5b, 11b\n\t"             \
476                         STR(PTR)"\t6b, 11b\n\t"             \
477                         STR(PTR)"\t7b, 11b\n\t"             \
478                         STR(PTR)"\t8b, 11b\n\t"             \
479                         ".previous"                         \
480                 : "=&r" (res)                               \
481                 : "r" (value), "r" (addr), "i" (-EFAULT)    \
482                 : "memory");                                \
483 } while(0)
484 
485 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
486 
487 #else /* __BIG_ENDIAN */
488 
489 #define     _LoadHW(addr, value, res, type)  \
490 do {                                                        \
491                 __asm__ __volatile__ (".set\tnoat\n"        \
492                         "1:\t"type##_lb("%0", "1(%2)")"\n"  \
493                         "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\
494                         "sll\t%0, 0x8\n\t"                  \
495                         "or\t%0, $1\n\t"                    \
496                         "li\t%1, 0\n"                       \
497                         "3:\t.set\tat\n\t"                  \
498                         ".insn\n\t"                         \
499                         ".section\t.fixup,\"ax\"\n\t"       \
500                         "4:\tli\t%1, %3\n\t"                \
501                         "j\t3b\n\t"                         \
502                         ".previous\n\t"                     \
503                         ".section\t__ex_table,\"a\"\n\t"    \
504                         STR(PTR)"\t1b, 4b\n\t"              \
505                         STR(PTR)"\t2b, 4b\n\t"              \
506                         ".previous"                         \
507                         : "=&r" (value), "=r" (res)         \
508                         : "r" (addr), "i" (-EFAULT));       \
509 } while(0)
510 
511 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
512 #define     _LoadW(addr, value, res, type)   \
513 do {                                                        \
514                 __asm__ __volatile__ (                      \
515                         "1:\t"type##_lwl("%0", "3(%2)")"\n" \
516                         "2:\t"type##_lwr("%0", "(%2)")"\n\t"\
517                         "li\t%1, 0\n"                       \
518                         "3:\n\t"                            \
519                         ".insn\n\t"                         \
520                         ".section\t.fixup,\"ax\"\n\t"       \
521                         "4:\tli\t%1, %3\n\t"                \
522                         "j\t3b\n\t"                         \
523                         ".previous\n\t"                     \
524                         ".section\t__ex_table,\"a\"\n\t"    \
525                         STR(PTR)"\t1b, 4b\n\t"              \
526                         STR(PTR)"\t2b, 4b\n\t"              \
527                         ".previous"                         \
528                         : "=&r" (value), "=r" (res)         \
529                         : "r" (addr), "i" (-EFAULT));       \
530 } while(0)
531 
532 #else  /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
533 /* For CPUs without lwl instruction */
534 #define     _LoadW(addr, value, res, type) \
535 do {                                                        \
536                 __asm__ __volatile__ (                      \
537                         ".set\tpush\n"                      \
538                         ".set\tnoat\n\t"                    \
539                         "1:"type##_lb("%0", "3(%2)")"\n\t"  \
540                         "2:"type##_lbu("$1", "2(%2)")"\n\t" \
541                         "sll\t%0, 0x8\n\t"                  \
542                         "or\t%0, $1\n\t"                    \
543                         "3:"type##_lbu("$1", "1(%2)")"\n\t" \
544                         "sll\t%0, 0x8\n\t"                  \
545                         "or\t%0, $1\n\t"                    \
546                         "4:"type##_lbu("$1", "0(%2)")"\n\t" \
547                         "sll\t%0, 0x8\n\t"                  \
548                         "or\t%0, $1\n\t"                    \
549                         "li\t%1, 0\n"                       \
550                         ".set\tpop\n"                       \
551                         "10:\n\t"                           \
552                         ".insn\n\t"                         \
553                         ".section\t.fixup,\"ax\"\n\t"       \
554                         "11:\tli\t%1, %3\n\t"               \
555                         "j\t10b\n\t"                        \
556                         ".previous\n\t"                     \
557                         ".section\t__ex_table,\"a\"\n\t"    \
558                         STR(PTR)"\t1b, 11b\n\t"             \
559                         STR(PTR)"\t2b, 11b\n\t"             \
560                         STR(PTR)"\t3b, 11b\n\t"             \
561                         STR(PTR)"\t4b, 11b\n\t"             \
562                         ".previous"                         \
563                         : "=&r" (value), "=r" (res)         \
564                         : "r" (addr), "i" (-EFAULT));       \
565 } while(0)
566 
567 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
568 
569 
570 #define     _LoadHWU(addr, value, res, type) \
571 do {                                                        \
572                 __asm__ __volatile__ (                      \
573                         ".set\tnoat\n"                      \
574                         "1:\t"type##_lbu("%0", "1(%2)")"\n" \
575                         "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\
576                         "sll\t%0, 0x8\n\t"                  \
577                         "or\t%0, $1\n\t"                    \
578                         "li\t%1, 0\n"                       \
579                         "3:\n\t"                            \
580                         ".insn\n\t"                         \
581                         ".set\tat\n\t"                      \
582                         ".section\t.fixup,\"ax\"\n\t"       \
583                         "4:\tli\t%1, %3\n\t"                \
584                         "j\t3b\n\t"                         \
585                         ".previous\n\t"                     \
586                         ".section\t__ex_table,\"a\"\n\t"    \
587                         STR(PTR)"\t1b, 4b\n\t"              \
588                         STR(PTR)"\t2b, 4b\n\t"              \
589                         ".previous"                         \
590                         : "=&r" (value), "=r" (res)         \
591                         : "r" (addr), "i" (-EFAULT));       \
592 } while(0)
593 
594 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
595 #define     _LoadWU(addr, value, res, type)  \
596 do {                                                        \
597                 __asm__ __volatile__ (                      \
598                         "1:\t"type##_lwl("%0", "3(%2)")"\n" \
599                         "2:\t"type##_lwr("%0", "(%2)")"\n\t"\
600                         "dsll\t%0, %0, 32\n\t"              \
601                         "dsrl\t%0, %0, 32\n\t"              \
602                         "li\t%1, 0\n"                       \
603                         "3:\n\t"                            \
604                         ".insn\n\t"                         \
605                         "\t.section\t.fixup,\"ax\"\n\t"     \
606                         "4:\tli\t%1, %3\n\t"                \
607                         "j\t3b\n\t"                         \
608                         ".previous\n\t"                     \
609                         ".section\t__ex_table,\"a\"\n\t"    \
610                         STR(PTR)"\t1b, 4b\n\t"              \
611                         STR(PTR)"\t2b, 4b\n\t"              \
612                         ".previous"                         \
613                         : "=&r" (value), "=r" (res)         \
614                         : "r" (addr), "i" (-EFAULT));       \
615 } while(0)
616 
617 #define     _LoadDW(addr, value, res)  \
618 do {                                                        \
619                 __asm__ __volatile__ (                      \
620                         "1:\tldl\t%0, 7(%2)\n"              \
621                         "2:\tldr\t%0, (%2)\n\t"             \
622                         "li\t%1, 0\n"                       \
623                         "3:\n\t"                            \
624                         ".insn\n\t"                         \
625                         "\t.section\t.fixup,\"ax\"\n\t"     \
626                         "4:\tli\t%1, %3\n\t"                \
627                         "j\t3b\n\t"                         \
628                         ".previous\n\t"                     \
629                         ".section\t__ex_table,\"a\"\n\t"    \
630                         STR(PTR)"\t1b, 4b\n\t"              \
631                         STR(PTR)"\t2b, 4b\n\t"              \
632                         ".previous"                         \
633                         : "=&r" (value), "=r" (res)         \
634                         : "r" (addr), "i" (-EFAULT));       \
635 } while(0)
636 
637 #else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
638 /* For CPUs without lwl and ldl instructions */
639 #define     _LoadWU(addr, value, res, type) \
640 do {                                                        \
641                 __asm__ __volatile__ (                      \
642                         ".set\tpush\n\t"                    \
643                         ".set\tnoat\n\t"                    \
644                         "1:"type##_lbu("%0", "3(%2)")"\n\t" \
645                         "2:"type##_lbu("$1", "2(%2)")"\n\t" \
646                         "sll\t%0, 0x8\n\t"                  \
647                         "or\t%0, $1\n\t"                    \
648                         "3:"type##_lbu("$1", "1(%2)")"\n\t" \
649                         "sll\t%0, 0x8\n\t"                  \
650                         "or\t%0, $1\n\t"                    \
651                         "4:"type##_lbu("$1", "0(%2)")"\n\t" \
652                         "sll\t%0, 0x8\n\t"                  \
653                         "or\t%0, $1\n\t"                    \
654                         "li\t%1, 0\n"                       \
655                         ".set\tpop\n"                       \
656                         "10:\n\t"                           \
657                         ".insn\n\t"                         \
658                         ".section\t.fixup,\"ax\"\n\t"       \
659                         "11:\tli\t%1, %3\n\t"               \
660                         "j\t10b\n\t"                        \
661                         ".previous\n\t"                     \
662                         ".section\t__ex_table,\"a\"\n\t"    \
663                         STR(PTR)"\t1b, 11b\n\t"             \
664                         STR(PTR)"\t2b, 11b\n\t"             \
665                         STR(PTR)"\t3b, 11b\n\t"             \
666                         STR(PTR)"\t4b, 11b\n\t"             \
667                         ".previous"                         \
668                         : "=&r" (value), "=r" (res)         \
669                         : "r" (addr), "i" (-EFAULT));       \
670 } while(0)
671 
672 #define     _LoadDW(addr, value, res)  \
673 do {                                                        \
674                 __asm__ __volatile__ (                      \
675                         ".set\tpush\n\t"                    \
676                         ".set\tnoat\n\t"                    \
677                         "1:lb\t%0, 7(%2)\n\t"               \
678                         "2:lbu\t$1, 6(%2)\n\t"              \
679                         "dsll\t%0, 0x8\n\t"                 \
680                         "or\t%0, $1\n\t"                    \
681                         "3:lbu\t$1, 5(%2)\n\t"              \
682                         "dsll\t%0, 0x8\n\t"                 \
683                         "or\t%0, $1\n\t"                    \
684                         "4:lbu\t$1, 4(%2)\n\t"              \
685                         "dsll\t%0, 0x8\n\t"                 \
686                         "or\t%0, $1\n\t"                    \
687                         "5:lbu\t$1, 3(%2)\n\t"              \
688                         "dsll\t%0, 0x8\n\t"                 \
689                         "or\t%0, $1\n\t"                    \
690                         "6:lbu\t$1, 2(%2)\n\t"              \
691                         "dsll\t%0, 0x8\n\t"                 \
692                         "or\t%0, $1\n\t"                    \
693                         "7:lbu\t$1, 1(%2)\n\t"              \
694                         "dsll\t%0, 0x8\n\t"                 \
695                         "or\t%0, $1\n\t"                    \
696                         "8:lbu\t$1, 0(%2)\n\t"              \
697                         "dsll\t%0, 0x8\n\t"                 \
698                         "or\t%0, $1\n\t"                    \
699                         "li\t%1, 0\n"                       \
700                         ".set\tpop\n\t"                     \
701                         "10:\n\t"                           \
702                         ".insn\n\t"                         \
703                         ".section\t.fixup,\"ax\"\n\t"       \
704                         "11:\tli\t%1, %3\n\t"               \
705                         "j\t10b\n\t"                        \
706                         ".previous\n\t"                     \
707                         ".section\t__ex_table,\"a\"\n\t"    \
708                         STR(PTR)"\t1b, 11b\n\t"             \
709                         STR(PTR)"\t2b, 11b\n\t"             \
710                         STR(PTR)"\t3b, 11b\n\t"             \
711                         STR(PTR)"\t4b, 11b\n\t"             \
712                         STR(PTR)"\t5b, 11b\n\t"             \
713                         STR(PTR)"\t6b, 11b\n\t"             \
714                         STR(PTR)"\t7b, 11b\n\t"             \
715                         STR(PTR)"\t8b, 11b\n\t"             \
716                         ".previous"                         \
717                         : "=&r" (value), "=r" (res)         \
718                         : "r" (addr), "i" (-EFAULT));       \
719 } while(0)
720 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
721 
722 #define     _StoreHW(addr, value, res, type) \
723 do {                                                        \
724                 __asm__ __volatile__ (                      \
725                         ".set\tnoat\n"                      \
726                         "1:\t"type##_sb("%1", "0(%2)")"\n"  \
727                         "srl\t$1,%1, 0x8\n"                 \
728                         "2:\t"type##_sb("$1", "1(%2)")"\n"  \
729                         ".set\tat\n\t"                      \
730                         "li\t%0, 0\n"                       \
731                         "3:\n\t"                            \
732                         ".insn\n\t"                         \
733                         ".section\t.fixup,\"ax\"\n\t"       \
734                         "4:\tli\t%0, %3\n\t"                \
735                         "j\t3b\n\t"                         \
736                         ".previous\n\t"                     \
737                         ".section\t__ex_table,\"a\"\n\t"    \
738                         STR(PTR)"\t1b, 4b\n\t"              \
739                         STR(PTR)"\t2b, 4b\n\t"              \
740                         ".previous"                         \
741                         : "=r" (res)                        \
742                         : "r" (value), "r" (addr), "i" (-EFAULT));\
743 } while(0)
744 
745 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
746 #define     _StoreW(addr, value, res, type)  \
747 do {                                                        \
748                 __asm__ __volatile__ (                      \
749                         "1:\t"type##_swl("%1", "3(%2)")"\n" \
750                         "2:\t"type##_swr("%1", "(%2)")"\n\t"\
751                         "li\t%0, 0\n"                       \
752                         "3:\n\t"                            \
753                         ".insn\n\t"                         \
754                         ".section\t.fixup,\"ax\"\n\t"       \
755                         "4:\tli\t%0, %3\n\t"                \
756                         "j\t3b\n\t"                         \
757                         ".previous\n\t"                     \
758                         ".section\t__ex_table,\"a\"\n\t"    \
759                         STR(PTR)"\t1b, 4b\n\t"              \
760                         STR(PTR)"\t2b, 4b\n\t"              \
761                         ".previous"                         \
762                 : "=r" (res)                                \
763                 : "r" (value), "r" (addr), "i" (-EFAULT));  \
764 } while(0)
765 
766 #define     _StoreDW(addr, value, res) \
767 do {                                                        \
768                 __asm__ __volatile__ (                      \
769                         "1:\tsdl\t%1, 7(%2)\n"              \
770                         "2:\tsdr\t%1, (%2)\n\t"             \
771                         "li\t%0, 0\n"                       \
772                         "3:\n\t"                            \
773                         ".insn\n\t"                         \
774                         ".section\t.fixup,\"ax\"\n\t"       \
775                         "4:\tli\t%0, %3\n\t"                \
776                         "j\t3b\n\t"                         \
777                         ".previous\n\t"                     \
778                         ".section\t__ex_table,\"a\"\n\t"    \
779                         STR(PTR)"\t1b, 4b\n\t"              \
780                         STR(PTR)"\t2b, 4b\n\t"              \
781                         ".previous"                         \
782                 : "=r" (res)                                \
783                 : "r" (value), "r" (addr), "i" (-EFAULT));  \
784 } while(0)
785 
786 #else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
787 /* For CPUs without swl and sdl instructions */
788 #define     _StoreW(addr, value, res, type)  \
789 do {                                                        \
790                 __asm__ __volatile__ (                      \
791                         ".set\tpush\n\t"                    \
792                         ".set\tnoat\n\t"                    \
793                         "1:"type##_sb("%1", "0(%2)")"\n\t"  \
794                         "srl\t$1, %1, 0x8\n\t"              \
795                         "2:"type##_sb("$1", "1(%2)")"\n\t"  \
796                         "srl\t$1, $1,  0x8\n\t"             \
797                         "3:"type##_sb("$1", "2(%2)")"\n\t"  \
798                         "srl\t$1, $1, 0x8\n\t"              \
799                         "4:"type##_sb("$1", "3(%2)")"\n\t"  \
800                         ".set\tpop\n\t"                     \
801                         "li\t%0, 0\n"                       \
802                         "10:\n\t"                           \
803                         ".insn\n\t"                         \
804                         ".section\t.fixup,\"ax\"\n\t"       \
805                         "11:\tli\t%0, %3\n\t"               \
806                         "j\t10b\n\t"                        \
807                         ".previous\n\t"                     \
808                         ".section\t__ex_table,\"a\"\n\t"    \
809                         STR(PTR)"\t1b, 11b\n\t"             \
810                         STR(PTR)"\t2b, 11b\n\t"             \
811                         STR(PTR)"\t3b, 11b\n\t"             \
812                         STR(PTR)"\t4b, 11b\n\t"             \
813                         ".previous"                         \
814                 : "=&r" (res)                               \
815                 : "r" (value), "r" (addr), "i" (-EFAULT)    \
816                 : "memory");                                \
817 } while(0)
818 
819 #define     _StoreDW(addr, value, res) \
820 do {                                                        \
821                 __asm__ __volatile__ (                      \
822                         ".set\tpush\n\t"                    \
823                         ".set\tnoat\n\t"                    \
824                         "1:sb\t%1, 0(%2)\n\t"               \
825                         "dsrl\t$1, %1, 0x8\n\t"             \
826                         "2:sb\t$1, 1(%2)\n\t"               \
827                         "dsrl\t$1, $1, 0x8\n\t"             \
828                         "3:sb\t$1, 2(%2)\n\t"               \
829                         "dsrl\t$1, $1, 0x8\n\t"             \
830                         "4:sb\t$1, 3(%2)\n\t"               \
831                         "dsrl\t$1, $1, 0x8\n\t"             \
832                         "5:sb\t$1, 4(%2)\n\t"               \
833                         "dsrl\t$1, $1, 0x8\n\t"             \
834                         "6:sb\t$1, 5(%2)\n\t"               \
835                         "dsrl\t$1, $1, 0x8\n\t"             \
836                         "7:sb\t$1, 6(%2)\n\t"               \
837                         "dsrl\t$1, $1, 0x8\n\t"             \
838                         "8:sb\t$1, 7(%2)\n\t"               \
839                         "dsrl\t$1, $1, 0x8\n\t"             \
840                         ".set\tpop\n\t"                     \
841                         "li\t%0, 0\n"                       \
842                         "10:\n\t"                           \
843                         ".insn\n\t"                         \
844                         ".section\t.fixup,\"ax\"\n\t"       \
845                         "11:\tli\t%0, %3\n\t"               \
846                         "j\t10b\n\t"                        \
847                         ".previous\n\t"                     \
848                         ".section\t__ex_table,\"a\"\n\t"    \
849                         STR(PTR)"\t1b, 11b\n\t"             \
850                         STR(PTR)"\t2b, 11b\n\t"             \
851                         STR(PTR)"\t3b, 11b\n\t"             \
852                         STR(PTR)"\t4b, 11b\n\t"             \
853                         STR(PTR)"\t5b, 11b\n\t"             \
854                         STR(PTR)"\t6b, 11b\n\t"             \
855                         STR(PTR)"\t7b, 11b\n\t"             \
856                         STR(PTR)"\t8b, 11b\n\t"             \
857                         ".previous"                         \
858                 : "=&r" (res)                               \
859                 : "r" (value), "r" (addr), "i" (-EFAULT)    \
860                 : "memory");                                \
861 } while(0)
862 
863 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
864 #endif
865 
866 #define LoadHWU(addr, value, res)       _LoadHWU(addr, value, res, kernel)
867 #define LoadHWUE(addr, value, res)      _LoadHWU(addr, value, res, user)
868 #define LoadWU(addr, value, res)        _LoadWU(addr, value, res, kernel)
869 #define LoadWUE(addr, value, res)       _LoadWU(addr, value, res, user)
870 #define LoadHW(addr, value, res)        _LoadHW(addr, value, res, kernel)
871 #define LoadHWE(addr, value, res)       _LoadHW(addr, value, res, user)
872 #define LoadW(addr, value, res)         _LoadW(addr, value, res, kernel)
873 #define LoadWE(addr, value, res)        _LoadW(addr, value, res, user)
874 #define LoadDW(addr, value, res)        _LoadDW(addr, value, res)
875 
876 #define StoreHW(addr, value, res)       _StoreHW(addr, value, res, kernel)
877 #define StoreHWE(addr, value, res)      _StoreHW(addr, value, res, user)
878 #define StoreW(addr, value, res)        _StoreW(addr, value, res, kernel)
879 #define StoreWE(addr, value, res)       _StoreW(addr, value, res, user)
880 #define StoreDW(addr, value, res)       _StoreDW(addr, value, res)
881 
882 static void emulate_load_store_insn(struct pt_regs *regs,
883         void __user *addr, unsigned int __user *pc)
884 {
885         union mips_instruction insn;
886         unsigned long value;
887         unsigned int res, preempted;
888         unsigned long origpc;
889         unsigned long orig31;
890         void __user *fault_addr = NULL;
891 #ifdef  CONFIG_EVA
892         mm_segment_t seg;
893 #endif
894         union fpureg *fpr;
895         enum msa_2b_fmt df;
896         unsigned int wd;
897         origpc = (unsigned long)pc;
898         orig31 = regs->regs[31];
899 
900         perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0);
901 
902         /*
903          * This load never faults.
904          */
905         __get_user(insn.word, pc);
906 
907         switch (insn.i_format.opcode) {
908                 /*
909                  * These are instructions that a compiler doesn't generate.  We
910                  * can assume therefore that the code is MIPS-aware and
911                  * really buggy.  Emulating these instructions would break the
912                  * semantics anyway.
913                  */
914         case ll_op:
915         case lld_op:
916         case sc_op:
917         case scd_op:
918 
919                 /*
920                  * For these instructions the only way to create an address
921                  * error is an attempted access to kernel/supervisor address
922                  * space.
923                  */
924         case ldl_op:
925         case ldr_op:
926         case lwl_op:
927         case lwr_op:
928         case sdl_op:
929         case sdr_op:
930         case swl_op:
931         case swr_op:
932         case lb_op:
933         case lbu_op:
934         case sb_op:
935                 goto sigbus;
936 
937                 /*
938                  * The remaining opcodes are the ones that are really of
939                  * interest.
940                  */
941         case spec3_op:
942                 if (insn.dsp_format.func == lx_op) {
943                         switch (insn.dsp_format.op) {
944                         case lwx_op:
945                                 if (!access_ok(VERIFY_READ, addr, 4))
946                                         goto sigbus;
947                                 LoadW(addr, value, res);
948                                 if (res)
949                                         goto fault;
950                                 compute_return_epc(regs);
951                                 regs->regs[insn.dsp_format.rd] = value;
952                                 break;
953                         case lhx_op:
954                                 if (!access_ok(VERIFY_READ, addr, 2))
955                                         goto sigbus;
956                                 LoadHW(addr, value, res);
957                                 if (res)
958                                         goto fault;
959                                 compute_return_epc(regs);
960                                 regs->regs[insn.dsp_format.rd] = value;
961                                 break;
962                         default:
963                                 goto sigill;
964                         }
965                 }
966 #ifdef CONFIG_EVA
967                 else {
968                         /*
969                          * we can land here only from kernel accessing user
970                          * memory, so we need to "switch" the address limit to
971                          * user space, so that address check can work properly.
972                          */
973                         seg = get_fs();
974                         set_fs(USER_DS);
975                         switch (insn.spec3_format.func) {
976                         case lhe_op:
977                                 if (!access_ok(VERIFY_READ, addr, 2)) {
978                                         set_fs(seg);
979                                         goto sigbus;
980                                 }
981                                 LoadHWE(addr, value, res);
982                                 if (res) {
983                                         set_fs(seg);
984                                         goto fault;
985                                 }
986                                 compute_return_epc(regs);
987                                 regs->regs[insn.spec3_format.rt] = value;
988                                 break;
989                         case lwe_op:
990                                 if (!access_ok(VERIFY_READ, addr, 4)) {
991                                         set_fs(seg);
992                                         goto sigbus;
993                                 }
994                                 LoadWE(addr, value, res);
995                                 if (res) {
996                                         set_fs(seg);
997                                         goto fault;
998                                 }
999                                 compute_return_epc(regs);
1000                                 regs->regs[insn.spec3_format.rt] = value;
1001                                 break;
1002                         case lhue_op:
1003                                 if (!access_ok(VERIFY_READ, addr, 2)) {
1004                                         set_fs(seg);
1005                                         goto sigbus;
1006                                 }
1007                                 LoadHWUE(addr, value, res);
1008                                 if (res) {
1009                                         set_fs(seg);
1010                                         goto fault;
1011                                 }
1012                                 compute_return_epc(regs);
1013                                 regs->regs[insn.spec3_format.rt] = value;
1014                                 break;
1015                         case she_op:
1016                                 if (!access_ok(VERIFY_WRITE, addr, 2)) {
1017                                         set_fs(seg);
1018                                         goto sigbus;
1019                                 }
1020                                 compute_return_epc(regs);
1021                                 value = regs->regs[insn.spec3_format.rt];
1022                                 StoreHWE(addr, value, res);
1023                                 if (res) {
1024                                         set_fs(seg);
1025                                         goto fault;
1026                                 }
1027                                 break;
1028                         case swe_op:
1029                                 if (!access_ok(VERIFY_WRITE, addr, 4)) {
1030                                         set_fs(seg);
1031                                         goto sigbus;
1032                                 }
1033                                 compute_return_epc(regs);
1034                                 value = regs->regs[insn.spec3_format.rt];
1035                                 StoreWE(addr, value, res);
1036                                 if (res) {
1037                                         set_fs(seg);
1038                                         goto fault;
1039                                 }
1040                                 break;
1041                         default:
1042                                 set_fs(seg);
1043                                 goto sigill;
1044                         }
1045                         set_fs(seg);
1046                 }
1047 #endif
1048                 break;
1049         case lh_op:
1050                 if (!access_ok(VERIFY_READ, addr, 2))
1051                         goto sigbus;
1052 
1053                 if (IS_ENABLED(CONFIG_EVA)) {
1054                         if (uaccess_kernel())
1055                                 LoadHW(addr, value, res);
1056                         else
1057                                 LoadHWE(addr, value, res);
1058                 } else {
1059                         LoadHW(addr, value, res);
1060                 }
1061 
1062                 if (res)
1063                         goto fault;
1064                 compute_return_epc(regs);
1065                 regs->regs[insn.i_format.rt] = value;
1066                 break;
1067 
1068         case lw_op:
1069                 if (!access_ok(VERIFY_READ, addr, 4))
1070                         goto sigbus;
1071 
1072                 if (IS_ENABLED(CONFIG_EVA)) {
1073                         if (uaccess_kernel())
1074                                 LoadW(addr, value, res);
1075                         else
1076                                 LoadWE(addr, value, res);
1077                 } else {
1078                         LoadW(addr, value, res);
1079                 }
1080 
1081                 if (res)
1082                         goto fault;
1083                 compute_return_epc(regs);
1084                 regs->regs[insn.i_format.rt] = value;
1085                 break;
1086 
1087         case lhu_op:
1088                 if (!access_ok(VERIFY_READ, addr, 2))
1089                         goto sigbus;
1090 
1091                 if (IS_ENABLED(CONFIG_EVA)) {
1092                         if (uaccess_kernel())
1093                                 LoadHWU(addr, value, res);
1094                         else
1095                                 LoadHWUE(addr, value, res);
1096                 } else {
1097                         LoadHWU(addr, value, res);
1098                 }
1099 
1100                 if (res)
1101                         goto fault;
1102                 compute_return_epc(regs);
1103                 regs->regs[insn.i_format.rt] = value;
1104                 break;
1105 
1106         case lwu_op:
1107 #ifdef CONFIG_64BIT
1108                 /*
1109                  * A 32-bit kernel might be running on a 64-bit processor.  But
1110                  * if we're on a 32-bit processor and an i-cache incoherency
1111                  * or race makes us see a 64-bit instruction here the sdl/sdr
1112                  * would blow up, so for now we don't handle unaligned 64-bit
1113                  * instructions on 32-bit kernels.
1114                  */
1115                 if (!access_ok(VERIFY_READ, addr, 4))
1116                         goto sigbus;
1117 
1118                 LoadWU(addr, value, res);
1119                 if (res)
1120                         goto fault;
1121                 compute_return_epc(regs);
1122                 regs->regs[insn.i_format.rt] = value;
1123                 break;
1124 #endif /* CONFIG_64BIT */
1125 
1126                 /* Cannot handle 64-bit instructions in 32-bit kernel */
1127                 goto sigill;
1128 
1129         case ld_op:
1130 #ifdef CONFIG_64BIT
1131                 /*
1132                  * A 32-bit kernel might be running on a 64-bit processor.  But
1133                  * if we're on a 32-bit processor and an i-cache incoherency
1134                  * or race makes us see a 64-bit instruction here the sdl/sdr
1135                  * would blow up, so for now we don't handle unaligned 64-bit
1136                  * instructions on 32-bit kernels.
1137                  */
1138                 if (!access_ok(VERIFY_READ, addr, 8))
1139                         goto sigbus;
1140 
1141                 LoadDW(addr, value, res);
1142                 if (res)
1143                         goto fault;
1144                 compute_return_epc(regs);
1145                 regs->regs[insn.i_format.rt] = value;
1146                 break;
1147 #endif /* CONFIG_64BIT */
1148 
1149                 /* Cannot handle 64-bit instructions in 32-bit kernel */
1150                 goto sigill;
1151 
1152         case sh_op:
1153                 if (!access_ok(VERIFY_WRITE, addr, 2))
1154                         goto sigbus;
1155 
1156                 compute_return_epc(regs);
1157                 value = regs->regs[insn.i_format.rt];
1158 
1159                 if (IS_ENABLED(CONFIG_EVA)) {
1160                         if (uaccess_kernel())
1161                                 StoreHW(addr, value, res);
1162                         else
1163                                 StoreHWE(addr, value, res);
1164                 } else {
1165                         StoreHW(addr, value, res);
1166                 }
1167 
1168                 if (res)
1169                         goto fault;
1170                 break;
1171 
1172         case sw_op:
1173                 if (!access_ok(VERIFY_WRITE, addr, 4))
1174                         goto sigbus;
1175 
1176                 compute_return_epc(regs);
1177                 value = regs->regs[insn.i_format.rt];
1178 
1179                 if (IS_ENABLED(CONFIG_EVA)) {
1180                         if (uaccess_kernel())
1181                                 StoreW(addr, value, res);
1182                         else
1183                                 StoreWE(addr, value, res);
1184                 } else {
1185                         StoreW(addr, value, res);
1186                 }
1187 
1188                 if (res)
1189                         goto fault;
1190                 break;
1191 
1192         case sd_op:
1193 #ifdef CONFIG_64BIT
1194                 /*
1195                  * A 32-bit kernel might be running on a 64-bit processor.  But
1196                  * if we're on a 32-bit processor and an i-cache incoherency
1197                  * or race makes us see a 64-bit instruction here the sdl/sdr
1198                  * would blow up, so for now we don't handle unaligned 64-bit
1199                  * instructions on 32-bit kernels.
1200                  */
1201                 if (!access_ok(VERIFY_WRITE, addr, 8))
1202                         goto sigbus;
1203 
1204                 compute_return_epc(regs);
1205                 value = regs->regs[insn.i_format.rt];
1206                 StoreDW(addr, value, res);
1207                 if (res)
1208                         goto fault;
1209                 break;
1210 #endif /* CONFIG_64BIT */
1211 
1212                 /* Cannot handle 64-bit instructions in 32-bit kernel */
1213                 goto sigill;
1214 
1215         case lwc1_op:
1216         case ldc1_op:
1217         case swc1_op:
1218         case sdc1_op:
1219         case cop1x_op:
1220                 die_if_kernel("Unaligned FP access in kernel code", regs);
1221                 BUG_ON(!used_math());
1222 
1223                 lose_fpu(1);    /* Save FPU state for the emulator. */
1224                 res = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
1225                                                &fault_addr);
1226                 own_fpu(1);     /* Restore FPU state. */
1227 
1228                 /* Signal if something went wrong. */
1229                 process_fpemu_return(res, fault_addr, 0);
1230 
1231                 if (res == 0)
1232                         break;
1233                 return;
1234 
1235         case msa_op:
1236                 if (!cpu_has_msa)
1237                         goto sigill;
1238 
1239                 /*
1240                  * If we've reached this point then userland should have taken
1241                  * the MSA disabled exception & initialised vector context at
1242                  * some point in the past.
1243                  */
1244                 BUG_ON(!thread_msa_context_live());
1245 
1246                 df = insn.msa_mi10_format.df;
1247                 wd = insn.msa_mi10_format.wd;
1248                 fpr = &current->thread.fpu.fpr[wd];
1249 
1250                 switch (insn.msa_mi10_format.func) {
1251                 case msa_ld_op:
1252                         if (!access_ok(VERIFY_READ, addr, sizeof(*fpr)))
1253                                 goto sigbus;
1254 
1255                         do {
1256                                 /*
1257                                  * If we have live MSA context keep track of
1258                                  * whether we get preempted in order to avoid
1259                                  * the register context we load being clobbered
1260                                  * by the live context as it's saved during
1261                                  * preemption. If we don't have live context
1262                                  * then it can't be saved to clobber the value
1263                                  * we load.
1264                                  */
1265                                 preempted = test_thread_flag(TIF_USEDMSA);
1266 
1267                                 res = __copy_from_user_inatomic(fpr, addr,
1268                                                                 sizeof(*fpr));
1269                                 if (res)
1270                                         goto fault;
1271 
1272                                 /*
1273                                  * Update the hardware register if it is in use
1274                                  * by the task in this quantum, in order to
1275                                  * avoid having to save & restore the whole
1276                                  * vector context.
1277                                  */
1278                                 preempt_disable();
1279                                 if (test_thread_flag(TIF_USEDMSA)) {
1280                                         write_msa_wr(wd, fpr, df);
1281                                         preempted = 0;
1282                                 }
1283                                 preempt_enable();
1284                         } while (preempted);
1285                         break;
1286 
1287                 case msa_st_op:
1288                         if (!access_ok(VERIFY_WRITE, addr, sizeof(*fpr)))
1289                                 goto sigbus;
1290 
1291                         /*
1292                          * Update from the hardware register if it is in use by
1293                          * the task in this quantum, in order to avoid having to
1294                          * save & restore the whole vector context.
1295                          */
1296                         preempt_disable();
1297                         if (test_thread_flag(TIF_USEDMSA))
1298                                 read_msa_wr(wd, fpr, df);
1299                         preempt_enable();
1300 
1301                         res = __copy_to_user_inatomic(addr, fpr, sizeof(*fpr));
1302                         if (res)
1303                                 goto fault;
1304                         break;
1305 
1306                 default:
1307                         goto sigbus;
1308                 }
1309 
1310                 compute_return_epc(regs);
1311                 break;
1312 
1313 #ifndef CONFIG_CPU_MIPSR6
1314         /*
1315          * COP2 is available to implementor for application specific use.
1316          * It's up to applications to register a notifier chain and do
1317          * whatever they have to do, including possible sending of signals.
1318          *
1319          * This instruction has been reallocated in Release 6
1320          */
1321         case lwc2_op:
1322                 cu2_notifier_call_chain(CU2_LWC2_OP, regs);
1323                 break;
1324 
1325         case ldc2_op:
1326                 cu2_notifier_call_chain(CU2_LDC2_OP, regs);
1327                 break;
1328 
1329         case swc2_op:
1330                 cu2_notifier_call_chain(CU2_SWC2_OP, regs);
1331                 break;
1332 
1333         case sdc2_op:
1334                 cu2_notifier_call_chain(CU2_SDC2_OP, regs);
1335                 break;
1336 #endif
1337         default:
1338                 /*
1339                  * Pheeee...  We encountered an yet unknown instruction or
1340                  * cache coherence problem.  Die sucker, die ...
1341                  */
1342                 goto sigill;
1343         }
1344 
1345 #ifdef CONFIG_DEBUG_FS
1346         unaligned_instructions++;
1347 #endif
1348 
1349         return;
1350 
1351 fault:
1352         /* roll back jump/branch */
1353         regs->cp0_epc = origpc;
1354         regs->regs[31] = orig31;
1355         /* Did we have an exception handler installed? */
1356         if (fixup_exception(regs))
1357                 return;
1358 
1359         die_if_kernel("Unhandled kernel unaligned access", regs);
1360         force_sig(SIGSEGV, current);
1361 
1362         return;
1363 
1364 sigbus:
1365         die_if_kernel("Unhandled kernel unaligned access", regs);
1366         force_sig(SIGBUS, current);
1367 
1368         return;
1369 
1370 sigill:
1371         die_if_kernel
1372             ("Unhandled kernel unaligned access or invalid instruction", regs);
1373         force_sig(SIGILL, current);
1374 }
1375 
1376 /* Recode table from 16-bit register notation to 32-bit GPR. */
1377 const int reg16to32[] = { 16, 17, 2, 3, 4, 5, 6, 7 };
1378 
1379 /* Recode table from 16-bit STORE register notation to 32-bit GPR. */
1380 static const int reg16to32st[] = { 0, 17, 2, 3, 4, 5, 6, 7 };
1381 
1382 static void emulate_load_store_microMIPS(struct pt_regs *regs,
1383                                          void __user *addr)
1384 {
1385         unsigned long value;
1386         unsigned int res;
1387         int i;
1388         unsigned int reg = 0, rvar;
1389         unsigned long orig31;
1390         u16 __user *pc16;
1391         u16 halfword;
1392         unsigned int word;
1393         unsigned long origpc, contpc;
1394         union mips_instruction insn;
1395         struct mm_decoded_insn mminsn;
1396         void __user *fault_addr = NULL;
1397 
1398         origpc = regs->cp0_epc;
1399         orig31 = regs->regs[31];
1400 
1401         mminsn.micro_mips_mode = 1;
1402 
1403         /*
1404          * This load never faults.
1405          */
1406         pc16 = (unsigned short __user *)msk_isa16_mode(regs->cp0_epc);
1407         __get_user(halfword, pc16);
1408         pc16++;
1409         contpc = regs->cp0_epc + 2;
1410         word = ((unsigned int)halfword << 16);
1411         mminsn.pc_inc = 2;
1412 
1413         if (!mm_insn_16bit(halfword)) {
1414                 __get_user(halfword, pc16);
1415                 pc16++;
1416                 contpc = regs->cp0_epc + 4;
1417                 mminsn.pc_inc = 4;
1418                 word |= halfword;
1419         }
1420         mminsn.insn = word;
1421 
1422         if (get_user(halfword, pc16))
1423                 goto fault;
1424         mminsn.next_pc_inc = 2;
1425         word = ((unsigned int)halfword << 16);
1426 
1427         if (!mm_insn_16bit(halfword)) {
1428                 pc16++;
1429                 if (get_user(halfword, pc16))
1430                         goto fault;
1431                 mminsn.next_pc_inc = 4;
1432                 word |= halfword;
1433         }
1434         mminsn.next_insn = word;
1435 
1436         insn = (union mips_instruction)(mminsn.insn);
1437         if (mm_isBranchInstr(regs, mminsn, &contpc))
1438                 insn = (union mips_instruction)(mminsn.next_insn);
1439 
1440         /*  Parse instruction to find what to do */
1441 
1442         switch (insn.mm_i_format.opcode) {
1443 
1444         case mm_pool32a_op:
1445                 switch (insn.mm_x_format.func) {
1446                 case mm_lwxs_op:
1447                         reg = insn.mm_x_format.rd;
1448                         goto loadW;
1449                 }
1450 
1451                 goto sigbus;
1452 
1453         case mm_pool32b_op:
1454                 switch (insn.mm_m_format.func) {
1455                 case mm_lwp_func:
1456                         reg = insn.mm_m_format.rd;
1457                         if (reg == 31)
1458                                 goto sigbus;
1459 
1460                         if (!access_ok(VERIFY_READ, addr, 8))
1461                                 goto sigbus;
1462 
1463                         LoadW(addr, value, res);
1464                         if (res)
1465                                 goto fault;
1466                         regs->regs[reg] = value;
1467                         addr += 4;
1468                         LoadW(addr, value, res);
1469                         if (res)
1470                                 goto fault;
1471                         regs->regs[reg + 1] = value;
1472                         goto success;
1473 
1474                 case mm_swp_func:
1475                         reg = insn.mm_m_format.rd;
1476                         if (reg == 31)
1477                                 goto sigbus;
1478 
1479                         if (!access_ok(VERIFY_WRITE, addr, 8))
1480                                 goto sigbus;
1481 
1482                         value = regs->regs[reg];
1483                         StoreW(addr, value, res);
1484                         if (res)
1485                                 goto fault;
1486                         addr += 4;
1487                         value = regs->regs[reg + 1];
1488                         StoreW(addr, value, res);
1489                         if (res)
1490                                 goto fault;
1491                         goto success;
1492 
1493                 case mm_ldp_func:
1494 #ifdef CONFIG_64BIT
1495                         reg = insn.mm_m_format.rd;
1496                         if (reg == 31)
1497                                 goto sigbus;
1498 
1499                         if (!access_ok(VERIFY_READ, addr, 16))
1500                                 goto sigbus;
1501 
1502                         LoadDW(addr, value, res);
1503                         if (res)
1504                                 goto fault;
1505                         regs->regs[reg] = value;
1506                         addr += 8;
1507                         LoadDW(addr, value, res);
1508                         if (res)
1509                                 goto fault;
1510                         regs->regs[reg + 1] = value;
1511                         goto success;
1512 #endif /* CONFIG_64BIT */
1513 
1514                         goto sigill;
1515 
1516                 case mm_sdp_func:
1517 #ifdef CONFIG_64BIT
1518                         reg = insn.mm_m_format.rd;
1519                         if (reg == 31)
1520                                 goto sigbus;
1521 
1522                         if (!access_ok(VERIFY_WRITE, addr, 16))
1523                                 goto sigbus;
1524 
1525                         value = regs->regs[reg];
1526                         StoreDW(addr, value, res);
1527                         if (res)
1528                                 goto fault;
1529                         addr += 8;
1530                         value = regs->regs[reg + 1];
1531                         StoreDW(addr, value, res);
1532                         if (res)
1533                                 goto fault;
1534                         goto success;
1535 #endif /* CONFIG_64BIT */
1536 
1537                         goto sigill;
1538 
1539                 case mm_lwm32_func:
1540                         reg = insn.mm_m_format.rd;
1541                         rvar = reg & 0xf;
1542                         if ((rvar > 9) || !reg)
1543                                 goto sigill;
1544                         if (reg & 0x10) {
1545                                 if (!access_ok
1546                                     (VERIFY_READ, addr, 4 * (rvar + 1)))
1547                                         goto sigbus;
1548                         } else {
1549                                 if (!access_ok(VERIFY_READ, addr, 4 * rvar))
1550                                         goto sigbus;
1551                         }
1552                         if (rvar == 9)
1553                                 rvar = 8;
1554                         for (i = 16; rvar; rvar--, i++) {
1555                                 LoadW(addr, value, res);
1556                                 if (res)
1557                                         goto fault;
1558                                 addr += 4;
1559                                 regs->regs[i] = value;
1560                         }
1561                         if ((reg & 0xf) == 9) {
1562                                 LoadW(addr, value, res);
1563                                 if (res)
1564                                         goto fault;
1565                                 addr += 4;
1566                                 regs->regs[30] = value;
1567                         }
1568                         if (reg & 0x10) {
1569                                 LoadW(addr, value, res);
1570                                 if (res)
1571                                         goto fault;
1572                                 regs->regs[31] = value;
1573                         }
1574                         goto success;
1575 
1576                 case mm_swm32_func:
1577                         reg = insn.mm_m_format.rd;
1578                         rvar = reg & 0xf;
1579                         if ((rvar > 9) || !reg)
1580                                 goto sigill;
1581                         if (reg & 0x10) {
1582                                 if (!access_ok
1583                                     (VERIFY_WRITE, addr, 4 * (rvar + 1)))
1584                                         goto sigbus;
1585                         } else {
1586                                 if (!access_ok(VERIFY_WRITE, addr, 4 * rvar))
1587                                         goto sigbus;
1588                         }
1589                         if (rvar == 9)
1590                                 rvar = 8;
1591                         for (i = 16; rvar; rvar--, i++) {
1592                                 value = regs->regs[i];
1593                                 StoreW(addr, value, res);
1594                                 if (res)
1595                                         goto fault;
1596                                 addr += 4;
1597                         }
1598                         if ((reg & 0xf) == 9) {
1599                                 value = regs->regs[30];
1600                                 StoreW(addr, value, res);
1601                                 if (res)
1602                                         goto fault;
1603                                 addr += 4;
1604                         }
1605                         if (reg & 0x10) {
1606                                 value = regs->regs[31];
1607                                 StoreW(addr, value, res);
1608                                 if (res)
1609                                         goto fault;
1610                         }
1611                         goto success;
1612 
1613                 case mm_ldm_func:
1614 #ifdef CONFIG_64BIT
1615                         reg = insn.mm_m_format.rd;
1616                         rvar = reg & 0xf;
1617                         if ((rvar > 9) || !reg)
1618                                 goto sigill;
1619                         if (reg & 0x10) {
1620                                 if (!access_ok
1621                                     (VERIFY_READ, addr, 8 * (rvar + 1)))
1622                                         goto sigbus;
1623                         } else {
1624                                 if (!access_ok(VERIFY_READ, addr, 8 * rvar))
1625                                         goto sigbus;
1626                         }
1627                         if (rvar == 9)
1628                                 rvar = 8;
1629 
1630                         for (i = 16; rvar; rvar--, i++) {
1631                                 LoadDW(addr, value, res);
1632                                 if (res)
1633                                         goto fault;
1634                                 addr += 4;
1635                                 regs->regs[i] = value;
1636                         }
1637                         if ((reg & 0xf) == 9) {
1638                                 LoadDW(addr, value, res);
1639                                 if (res)
1640                                         goto fault;
1641                                 addr += 8;
1642                                 regs->regs[30] = value;
1643                         }
1644                         if (reg & 0x10) {
1645                                 LoadDW(addr, value, res);
1646                                 if (res)
1647                                         goto fault;
1648                                 regs->regs[31] = value;
1649                         }
1650                         goto success;
1651 #endif /* CONFIG_64BIT */
1652 
1653                         goto sigill;
1654 
1655                 case mm_sdm_func:
1656 #ifdef CONFIG_64BIT
1657                         reg = insn.mm_m_format.rd;
1658                         rvar = reg & 0xf;
1659                         if ((rvar > 9) || !reg)
1660                                 goto sigill;
1661                         if (reg & 0x10) {
1662                                 if (!access_ok
1663                                     (VERIFY_WRITE, addr, 8 * (rvar + 1)))
1664                                         goto sigbus;
1665                         } else {
1666                                 if (!access_ok(VERIFY_WRITE, addr, 8 * rvar))
1667                                         goto sigbus;
1668                         }
1669                         if (rvar == 9)
1670                                 rvar = 8;
1671 
1672                         for (i = 16; rvar; rvar--, i++) {
1673                                 value = regs->regs[i];
1674                                 StoreDW(addr, value, res);
1675                                 if (res)
1676                                         goto fault;
1677                                 addr += 8;
1678                         }
1679                         if ((reg & 0xf) == 9) {
1680                                 value = regs->regs[30];
1681                                 StoreDW(addr, value, res);
1682                                 if (res)
1683                                         goto fault;
1684                                 addr += 8;
1685                         }
1686                         if (reg & 0x10) {
1687                                 value = regs->regs[31];
1688                                 StoreDW(addr, value, res);
1689                                 if (res)
1690                                         goto fault;
1691                         }
1692                         goto success;
1693 #endif /* CONFIG_64BIT */
1694 
1695                         goto sigill;
1696 
1697                         /*  LWC2, SWC2, LDC2, SDC2 are not serviced */
1698                 }
1699 
1700                 goto sigbus;
1701 
1702         case mm_pool32c_op:
1703                 switch (insn.mm_m_format.func) {
1704                 case mm_lwu_func:
1705                         reg = insn.mm_m_format.rd;
1706                         goto loadWU;
1707                 }
1708 
1709                 /*  LL,SC,LLD,SCD are not serviced */
1710                 goto sigbus;
1711 
1712         case mm_pool32f_op:
1713                 switch (insn.mm_x_format.func) {
1714                 case mm_lwxc1_func:
1715                 case mm_swxc1_func:
1716                 case mm_ldxc1_func:
1717                 case mm_sdxc1_func:
1718                         goto fpu_emul;
1719                 }
1720 
1721                 goto sigbus;
1722 
1723         case mm_ldc132_op:
1724         case mm_sdc132_op:
1725         case mm_lwc132_op:
1726         case mm_swc132_op:
1727 fpu_emul:
1728                 /* roll back jump/branch */
1729                 regs->cp0_epc = origpc;
1730                 regs->regs[31] = orig31;
1731 
1732                 die_if_kernel("Unaligned FP access in kernel code", regs);
1733                 BUG_ON(!used_math());
1734                 BUG_ON(!is_fpu_owner());
1735 
1736                 lose_fpu(1);    /* save the FPU state for the emulator */
1737                 res = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
1738                                                &fault_addr);
1739                 own_fpu(1);     /* restore FPU state */
1740 
1741                 /* If something went wrong, signal */
1742                 process_fpemu_return(res, fault_addr, 0);
1743 
1744                 if (res == 0)
1745                         goto success;
1746                 return;
1747 
1748         case mm_lh32_op:
1749                 reg = insn.mm_i_format.rt;
1750                 goto loadHW;
1751 
1752         case mm_lhu32_op:
1753                 reg = insn.mm_i_format.rt;
1754                 goto loadHWU;
1755 
1756         case mm_lw32_op:
1757                 reg = insn.mm_i_format.rt;
1758                 goto loadW;
1759 
1760         case mm_sh32_op:
1761                 reg = insn.mm_i_format.rt;
1762                 goto storeHW;
1763 
1764         case mm_sw32_op:
1765                 reg = insn.mm_i_format.rt;
1766                 goto storeW;
1767 
1768         case mm_ld32_op:
1769                 reg = insn.mm_i_format.rt;
1770                 goto loadDW;
1771 
1772         case mm_sd32_op:
1773                 reg = insn.mm_i_format.rt;
1774                 goto storeDW;
1775 
1776         case mm_pool16c_op:
1777                 switch (insn.mm16_m_format.func) {
1778                 case mm_lwm16_op:
1779                         reg = insn.mm16_m_format.rlist;
1780                         rvar = reg + 1;
1781                         if (!access_ok(VERIFY_READ, addr, 4 * rvar))
1782                                 goto sigbus;
1783 
1784                         for (i = 16; rvar; rvar--, i++) {
1785                                 LoadW(addr, value, res);
1786                                 if (res)
1787                                         goto fault;
1788                                 addr += 4;
1789                                 regs->regs[i] = value;
1790                         }
1791                         LoadW(addr, value, res);
1792                         if (res)
1793                                 goto fault;
1794                         regs->regs[31] = value;
1795 
1796                         goto success;
1797 
1798                 case mm_swm16_op:
1799                         reg = insn.mm16_m_format.rlist;
1800                         rvar = reg + 1;
1801                         if (!access_ok(VERIFY_WRITE, addr, 4 * rvar))
1802                                 goto sigbus;
1803 
1804                         for (i = 16; rvar; rvar--, i++) {
1805                                 value = regs->regs[i];
1806                                 StoreW(addr, value, res);
1807                                 if (res)
1808                                         goto fault;
1809                                 addr += 4;
1810                         }
1811                         value = regs->regs[31];
1812                         StoreW(addr, value, res);
1813                         if (res)
1814                                 goto fault;
1815 
1816                         goto success;
1817 
1818                 }
1819 
1820                 goto sigbus;
1821 
1822         case mm_lhu16_op:
1823                 reg = reg16to32[insn.mm16_rb_format.rt];
1824                 goto loadHWU;
1825 
1826         case mm_lw16_op:
1827                 reg = reg16to32[insn.mm16_rb_format.rt];
1828                 goto loadW;
1829 
1830         case mm_sh16_op:
1831                 reg = reg16to32st[insn.mm16_rb_format.rt];
1832                 goto storeHW;
1833 
1834         case mm_sw16_op:
1835                 reg = reg16to32st[insn.mm16_rb_format.rt];
1836                 goto storeW;
1837 
1838         case mm_lwsp16_op:
1839                 reg = insn.mm16_r5_format.rt;
1840                 goto loadW;
1841 
1842         case mm_swsp16_op:
1843                 reg = insn.mm16_r5_format.rt;
1844                 goto storeW;
1845 
1846         case mm_lwgp16_op:
1847                 reg = reg16to32[insn.mm16_r3_format.rt];
1848                 goto loadW;
1849 
1850         default:
1851                 goto sigill;
1852         }
1853 
1854 loadHW:
1855         if (!access_ok(VERIFY_READ, addr, 2))
1856                 goto sigbus;
1857 
1858         LoadHW(addr, value, res);
1859         if (res)
1860                 goto fault;
1861         regs->regs[reg] = value;
1862         goto success;
1863 
1864 loadHWU:
1865         if (!access_ok(VERIFY_READ, addr, 2))
1866                 goto sigbus;
1867 
1868         LoadHWU(addr, value, res);
1869         if (res)
1870                 goto fault;
1871         regs->regs[reg] = value;
1872         goto success;
1873 
1874 loadW:
1875         if (!access_ok(VERIFY_READ, addr, 4))
1876                 goto sigbus;
1877 
1878         LoadW(addr, value, res);
1879         if (res)
1880                 goto fault;
1881         regs->regs[reg] = value;
1882         goto success;
1883 
1884 loadWU:
1885 #ifdef CONFIG_64BIT
1886         /*
1887          * A 32-bit kernel might be running on a 64-bit processor.  But
1888          * if we're on a 32-bit processor and an i-cache incoherency
1889          * or race makes us see a 64-bit instruction here the sdl/sdr
1890          * would blow up, so for now we don't handle unaligned 64-bit
1891          * instructions on 32-bit kernels.
1892          */
1893         if (!access_ok(VERIFY_READ, addr, 4))
1894                 goto sigbus;
1895 
1896         LoadWU(addr, value, res);
1897         if (res)
1898                 goto fault;
1899         regs->regs[reg] = value;
1900         goto success;
1901 #endif /* CONFIG_64BIT */
1902 
1903         /* Cannot handle 64-bit instructions in 32-bit kernel */
1904         goto sigill;
1905 
1906 loadDW:
1907 #ifdef CONFIG_64BIT
1908         /*
1909          * A 32-bit kernel might be running on a 64-bit processor.  But
1910          * if we're on a 32-bit processor and an i-cache incoherency
1911          * or race makes us see a 64-bit instruction here the sdl/sdr
1912          * would blow up, so for now we don't handle unaligned 64-bit
1913          * instructions on 32-bit kernels.
1914          */
1915         if (!access_ok(VERIFY_READ, addr, 8))
1916                 goto sigbus;
1917 
1918         LoadDW(addr, value, res);
1919         if (res)
1920                 goto fault;
1921         regs->regs[reg] = value;
1922         goto success;
1923 #endif /* CONFIG_64BIT */
1924 
1925         /* Cannot handle 64-bit instructions in 32-bit kernel */
1926         goto sigill;
1927 
1928 storeHW:
1929         if (!access_ok(VERIFY_WRITE, addr, 2))
1930                 goto sigbus;
1931 
1932         value = regs->regs[reg];
1933         StoreHW(addr, value, res);
1934         if (res)
1935                 goto fault;
1936         goto success;
1937 
1938 storeW:
1939         if (!access_ok(VERIFY_WRITE, addr, 4))
1940                 goto sigbus;
1941 
1942         value = regs->regs[reg];
1943         StoreW(addr, value, res);
1944         if (res)
1945                 goto fault;
1946         goto success;
1947 
1948 storeDW:
1949 #ifdef CONFIG_64BIT
1950         /*
1951          * A 32-bit kernel might be running on a 64-bit processor.  But
1952          * if we're on a 32-bit processor and an i-cache incoherency
1953          * or race makes us see a 64-bit instruction here the sdl/sdr
1954          * would blow up, so for now we don't handle unaligned 64-bit
1955          * instructions on 32-bit kernels.
1956          */
1957         if (!access_ok(VERIFY_WRITE, addr, 8))
1958                 goto sigbus;
1959 
1960         value = regs->regs[reg];
1961         StoreDW(addr, value, res);
1962         if (res)
1963                 goto fault;
1964         goto success;
1965 #endif /* CONFIG_64BIT */
1966 
1967         /* Cannot handle 64-bit instructions in 32-bit kernel */
1968         goto sigill;
1969 
1970 success:
1971         regs->cp0_epc = contpc; /* advance or branch */
1972 
1973 #ifdef CONFIG_DEBUG_FS
1974         unaligned_instructions++;
1975 #endif
1976         return;
1977 
1978 fault:
1979         /* roll back jump/branch */
1980         regs->cp0_epc = origpc;
1981         regs->regs[31] = orig31;
1982         /* Did we have an exception handler installed? */
1983         if (fixup_exception(regs))
1984                 return;
1985 
1986         die_if_kernel("Unhandled kernel unaligned access", regs);
1987         force_sig(SIGSEGV, current);
1988 
1989         return;
1990 
1991 sigbus:
1992         die_if_kernel("Unhandled kernel unaligned access", regs);
1993         force_sig(SIGBUS, current);
1994 
1995         return;
1996 
1997 sigill:
1998         die_if_kernel
1999             ("Unhandled kernel unaligned access or invalid instruction", regs);
2000         force_sig(SIGILL, current);
2001 }
2002 
2003 static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr)
2004 {
2005         unsigned long value;
2006         unsigned int res;
2007         int reg;
2008         unsigned long orig31;
2009         u16 __user *pc16;
2010         unsigned long origpc;
2011         union mips16e_instruction mips16inst, oldinst;
2012         unsigned int opcode;
2013         int extended = 0;
2014 
2015         origpc = regs->cp0_epc;
2016         orig31 = regs->regs[31];
2017         pc16 = (unsigned short __user *)msk_isa16_mode(origpc);
2018         /*
2019          * This load never faults.
2020          */
2021         __get_user(mips16inst.full, pc16);
2022         oldinst = mips16inst;
2023 
2024         /* skip EXTEND instruction */
2025         if (mips16inst.ri.opcode == MIPS16e_extend_op) {
2026                 extended = 1;
2027                 pc16++;
2028                 __get_user(mips16inst.full, pc16);
2029         } else if (delay_slot(regs)) {
2030                 /*  skip jump instructions */
2031                 /*  JAL/JALX are 32 bits but have OPCODE in first short int */
2032                 if (mips16inst.ri.opcode == MIPS16e_jal_op)
2033                         pc16++;
2034                 pc16++;
2035                 if (get_user(mips16inst.full, pc16))
2036                         goto sigbus;
2037         }
2038 
2039         opcode = mips16inst.ri.opcode;
2040         switch (opcode) {
2041         case MIPS16e_i64_op:    /* I64 or RI64 instruction */
2042                 switch (mips16inst.i64.func) {  /* I64/RI64 func field check */
2043                 case MIPS16e_ldpc_func:
2044                 case MIPS16e_ldsp_func:
2045                         reg = reg16to32[mips16inst.ri64.ry];
2046                         goto loadDW;
2047 
2048                 case MIPS16e_sdsp_func:
2049                         reg = reg16to32[mips16inst.ri64.ry];
2050                         goto writeDW;
2051 
2052                 case MIPS16e_sdrasp_func:
2053                         reg = 29;       /* GPRSP */
2054                         goto writeDW;
2055                 }
2056 
2057                 goto sigbus;
2058 
2059         case MIPS16e_swsp_op:
2060                 reg = reg16to32[mips16inst.ri.rx];
2061                 if (extended && cpu_has_mips16e2)
2062                         switch (mips16inst.ri.imm >> 5) {
2063                         case 0:         /* SWSP */
2064                         case 1:         /* SWGP */
2065                                 break;
2066                         case 2:         /* SHGP */
2067                                 opcode = MIPS16e_sh_op;
2068                                 break;
2069                         default:
2070                                 goto sigbus;
2071                         }
2072                 break;
2073 
2074         case MIPS16e_lwpc_op:
2075                 reg = reg16to32[mips16inst.ri.rx];
2076                 break;
2077 
2078         case MIPS16e_lwsp_op:
2079                 reg = reg16to32[mips16inst.ri.rx];
2080                 if (extended && cpu_has_mips16e2)
2081                         switch (mips16inst.ri.imm >> 5) {
2082                         case 0:         /* LWSP */
2083                         case 1:         /* LWGP */
2084                                 break;
2085                         case 2:         /* LHGP */
2086                                 opcode = MIPS16e_lh_op;
2087                                 break;
2088                         case 4:         /* LHUGP */
2089                                 opcode = MIPS16e_lhu_op;
2090                                 break;
2091                         default:
2092                                 goto sigbus;
2093                         }
2094                 break;
2095 
2096         case MIPS16e_i8_op:
2097                 if (mips16inst.i8.func != MIPS16e_swrasp_func)
2098                         goto sigbus;
2099                 reg = 29;       /* GPRSP */
2100                 break;
2101 
2102         default:
2103                 reg = reg16to32[mips16inst.rri.ry];
2104                 break;
2105         }
2106 
2107         switch (opcode) {
2108 
2109         case MIPS16e_lb_op:
2110         case MIPS16e_lbu_op:
2111         case MIPS16e_sb_op:
2112                 goto sigbus;
2113 
2114         case MIPS16e_lh_op:
2115                 if (!access_ok(VERIFY_READ, addr, 2))
2116                         goto sigbus;
2117 
2118                 LoadHW(addr, value, res);
2119                 if (res)
2120                         goto fault;
2121                 MIPS16e_compute_return_epc(regs, &oldinst);
2122                 regs->regs[reg] = value;
2123                 break;
2124 
2125         case MIPS16e_lhu_op:
2126                 if (!access_ok(VERIFY_READ, addr, 2))
2127                         goto sigbus;
2128 
2129                 LoadHWU(addr, value, res);
2130                 if (res)
2131                         goto fault;
2132                 MIPS16e_compute_return_epc(regs, &oldinst);
2133                 regs->regs[reg] = value;
2134                 break;
2135 
2136         case MIPS16e_lw_op:
2137         case MIPS16e_lwpc_op:
2138         case MIPS16e_lwsp_op:
2139                 if (!access_ok(VERIFY_READ, addr, 4))
2140                         goto sigbus;
2141 
2142                 LoadW(addr, value, res);
2143                 if (res)
2144                         goto fault;
2145                 MIPS16e_compute_return_epc(regs, &oldinst);
2146                 regs->regs[reg] = value;
2147                 break;
2148 
2149         case MIPS16e_lwu_op:
2150 #ifdef CONFIG_64BIT
2151                 /*
2152                  * A 32-bit kernel might be running on a 64-bit processor.  But
2153                  * if we're on a 32-bit processor and an i-cache incoherency
2154                  * or race makes us see a 64-bit instruction here the sdl/sdr
2155                  * would blow up, so for now we don't handle unaligned 64-bit
2156                  * instructions on 32-bit kernels.
2157                  */
2158                 if (!access_ok(VERIFY_READ, addr, 4))
2159                         goto sigbus;
2160 
2161                 LoadWU(addr, value, res);
2162                 if (res)
2163                         goto fault;
2164                 MIPS16e_compute_return_epc(regs, &oldinst);
2165                 regs->regs[reg] = value;
2166                 break;
2167 #endif /* CONFIG_64BIT */
2168 
2169                 /* Cannot handle 64-bit instructions in 32-bit kernel */
2170                 goto sigill;
2171 
2172         case MIPS16e_ld_op:
2173 loadDW:
2174 #ifdef CONFIG_64BIT
2175                 /*
2176                  * A 32-bit kernel might be running on a 64-bit processor.  But
2177                  * if we're on a 32-bit processor and an i-cache incoherency
2178                  * or race makes us see a 64-bit instruction here the sdl/sdr
2179                  * would blow up, so for now we don't handle unaligned 64-bit
2180                  * instructions on 32-bit kernels.
2181                  */
2182                 if (!access_ok(VERIFY_READ, addr, 8))
2183                         goto sigbus;
2184 
2185                 LoadDW(addr, value, res);
2186                 if (res)
2187                         goto fault;
2188                 MIPS16e_compute_return_epc(regs, &oldinst);
2189                 regs->regs[reg] = value;
2190                 break;
2191 #endif /* CONFIG_64BIT */
2192 
2193                 /* Cannot handle 64-bit instructions in 32-bit kernel */
2194                 goto sigill;
2195 
2196         case MIPS16e_sh_op:
2197                 if (!access_ok(VERIFY_WRITE, addr, 2))
2198                         goto sigbus;
2199 
2200                 MIPS16e_compute_return_epc(regs, &oldinst);
2201                 value = regs->regs[reg];
2202                 StoreHW(addr, value, res);
2203                 if (res)
2204                         goto fault;
2205                 break;
2206 
2207         case MIPS16e_sw_op:
2208         case MIPS16e_swsp_op:
2209         case MIPS16e_i8_op:     /* actually - MIPS16e_swrasp_func */
2210                 if (!access_ok(VERIFY_WRITE, addr, 4))
2211                         goto sigbus;
2212 
2213                 MIPS16e_compute_return_epc(regs, &oldinst);
2214                 value = regs->regs[reg];
2215                 StoreW(addr, value, res);
2216                 if (res)
2217                         goto fault;
2218                 break;
2219 
2220         case MIPS16e_sd_op:
2221 writeDW:
2222 #ifdef CONFIG_64BIT
2223                 /*
2224                  * A 32-bit kernel might be running on a 64-bit processor.  But
2225                  * if we're on a 32-bit processor and an i-cache incoherency
2226                  * or race makes us see a 64-bit instruction here the sdl/sdr
2227                  * would blow up, so for now we don't handle unaligned 64-bit
2228                  * instructions on 32-bit kernels.
2229                  */
2230                 if (!access_ok(VERIFY_WRITE, addr, 8))
2231                         goto sigbus;
2232 
2233                 MIPS16e_compute_return_epc(regs, &oldinst);
2234                 value = regs->regs[reg];
2235                 StoreDW(addr, value, res);
2236                 if (res)
2237                         goto fault;
2238                 break;
2239 #endif /* CONFIG_64BIT */
2240 
2241                 /* Cannot handle 64-bit instructions in 32-bit kernel */
2242                 goto sigill;
2243 
2244         default:
2245                 /*
2246                  * Pheeee...  We encountered an yet unknown instruction or
2247                  * cache coherence problem.  Die sucker, die ...
2248                  */
2249                 goto sigill;
2250         }
2251 
2252 #ifdef CONFIG_DEBUG_FS
2253         unaligned_instructions++;
2254 #endif
2255 
2256         return;
2257 
2258 fault:
2259         /* roll back jump/branch */
2260         regs->cp0_epc = origpc;
2261         regs->regs[31] = orig31;
2262         /* Did we have an exception handler installed? */
2263         if (fixup_exception(regs))
2264                 return;
2265 
2266         die_if_kernel("Unhandled kernel unaligned access", regs);
2267         force_sig(SIGSEGV, current);
2268 
2269         return;
2270 
2271 sigbus:
2272         die_if_kernel("Unhandled kernel unaligned access", regs);
2273         force_sig(SIGBUS, current);
2274 
2275         return;
2276 
2277 sigill:
2278         die_if_kernel
2279             ("Unhandled kernel unaligned access or invalid instruction", regs);
2280         force_sig(SIGILL, current);
2281 }
2282 
2283 asmlinkage void do_ade(struct pt_regs *regs)
2284 {
2285         enum ctx_state prev_state;
2286         unsigned int __user *pc;
2287         mm_segment_t seg;
2288 
2289         prev_state = exception_enter();
2290         perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS,
2291                         1, regs, regs->cp0_badvaddr);
2292         /*
2293          * Did we catch a fault trying to load an instruction?
2294          */
2295         if (regs->cp0_badvaddr == regs->cp0_epc)
2296                 goto sigbus;
2297 
2298         if (user_mode(regs) && !test_thread_flag(TIF_FIXADE))
2299                 goto sigbus;
2300         if (unaligned_action == UNALIGNED_ACTION_SIGNAL)
2301                 goto sigbus;
2302 
2303         /*
2304          * Do branch emulation only if we didn't forward the exception.
2305          * This is all so but ugly ...
2306          */
2307 
2308         /*
2309          * Are we running in microMIPS mode?
2310          */
2311         if (get_isa16_mode(regs->cp0_epc)) {
2312                 /*
2313                  * Did we catch a fault trying to load an instruction in
2314                  * 16-bit mode?
2315                  */
2316                 if (regs->cp0_badvaddr == msk_isa16_mode(regs->cp0_epc))
2317                         goto sigbus;
2318                 if (unaligned_action == UNALIGNED_ACTION_SHOW)
2319                         show_registers(regs);
2320 
2321                 if (cpu_has_mmips) {
2322                         seg = get_fs();
2323                         if (!user_mode(regs))
2324                                 set_fs(KERNEL_DS);
2325                         emulate_load_store_microMIPS(regs,
2326                                 (void __user *)regs->cp0_badvaddr);
2327                         set_fs(seg);
2328 
2329                         return;
2330                 }
2331 
2332                 if (cpu_has_mips16) {
2333                         seg = get_fs();
2334                         if (!user_mode(regs))
2335                                 set_fs(KERNEL_DS);
2336                         emulate_load_store_MIPS16e(regs,
2337                                 (void __user *)regs->cp0_badvaddr);
2338                         set_fs(seg);
2339 
2340                         return;
2341         }
2342 
2343                 goto sigbus;
2344         }
2345 
2346         if (unaligned_action == UNALIGNED_ACTION_SHOW)
2347                 show_registers(regs);
2348         pc = (unsigned int __user *)exception_epc(regs);
2349 
2350         seg = get_fs();
2351         if (!user_mode(regs))
2352                 set_fs(KERNEL_DS);
2353         emulate_load_store_insn(regs, (void __user *)regs->cp0_badvaddr, pc);
2354         set_fs(seg);
2355 
2356         return;
2357 
2358 sigbus:
2359         die_if_kernel("Kernel unaligned instruction access", regs);
2360         force_sig(SIGBUS, current);
2361 
2362         /*
2363          * XXX On return from the signal handler we should advance the epc
2364          */
2365         exception_exit(prev_state);
2366 }
2367 
2368 #ifdef CONFIG_DEBUG_FS
2369 static int __init debugfs_unaligned(void)
2370 {
2371         struct dentry *d;
2372 
2373         if (!mips_debugfs_dir)
2374                 return -ENODEV;
2375         d = debugfs_create_u32("unaligned_instructions", S_IRUGO,
2376                                mips_debugfs_dir, &unaligned_instructions);
2377         if (!d)
2378                 return -ENOMEM;
2379         d = debugfs_create_u32("unaligned_action", S_IRUGO | S_IWUSR,
2380                                mips_debugfs_dir, &unaligned_action);
2381         if (!d)
2382                 return -ENOMEM;
2383         return 0;
2384 }
2385 arch_initcall(debugfs_unaligned);
2386 #endif
2387 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp