~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/ia64/kernel/unaligned.c

Version: ~ [ linux-5.8-rc5 ] ~ [ linux-5.7.8 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.51 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.132 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.188 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.230 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.230 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  * Architecture-specific unaligned trap handling.
  4  *
  5  * Copyright (C) 1999-2002, 2004 Hewlett-Packard Co
  6  *      Stephane Eranian <eranian@hpl.hp.com>
  7  *      David Mosberger-Tang <davidm@hpl.hp.com>
  8  *
  9  * 2002/12/09   Fix rotating register handling (off-by-1 error, missing fr-rotation).  Fix
 10  *              get_rse_reg() to not leak kernel bits to user-level (reading an out-of-frame
 11  *              stacked register returns an undefined value; it does NOT trigger a
 12  *              "rsvd register fault").
 13  * 2001/10/11   Fix unaligned access to rotating registers in s/w pipelined loops.
 14  * 2001/08/13   Correct size of extended floats (float_fsz) from 16 to 10 bytes.
 15  * 2001/01/17   Add support emulation of unaligned kernel accesses.
 16  */
 17 #include <linux/jiffies.h>
 18 #include <linux/kernel.h>
 19 #include <linux/sched/signal.h>
 20 #include <linux/tty.h>
 21 #include <linux/extable.h>
 22 #include <linux/ratelimit.h>
 23 #include <linux/uaccess.h>
 24 
 25 #include <asm/intrinsics.h>
 26 #include <asm/processor.h>
 27 #include <asm/rse.h>
 28 #include <asm/exception.h>
 29 #include <asm/unaligned.h>
 30 
 31 extern int die_if_kernel(char *str, struct pt_regs *regs, long err);
 32 
 33 #undef DEBUG_UNALIGNED_TRAP
 34 
 35 #ifdef DEBUG_UNALIGNED_TRAP
 36 # define DPRINT(a...)   do { printk("%s %u: ", __func__, __LINE__); printk (a); } while (0)
 37 # define DDUMP(str,vp,len)      dump(str, vp, len)
 38 
 39 static void
 40 dump (const char *str, void *vp, size_t len)
 41 {
 42         unsigned char *cp = vp;
 43         int i;
 44 
 45         printk("%s", str);
 46         for (i = 0; i < len; ++i)
 47                 printk (" %02x", *cp++);
 48         printk("\n");
 49 }
 50 #else
 51 # define DPRINT(a...)
 52 # define DDUMP(str,vp,len)
 53 #endif
 54 
 55 #define IA64_FIRST_STACKED_GR   32
 56 #define IA64_FIRST_ROTATING_FR  32
 57 #define SIGN_EXT9               0xffffffffffffff00ul
 58 
 59 /*
 60  *  sysctl settable hook which tells the kernel whether to honor the
 61  *  IA64_THREAD_UAC_NOPRINT prctl.  Because this is user settable, we want
 62  *  to allow the super user to enable/disable this for security reasons
 63  *  (i.e. don't allow attacker to fill up logs with unaligned accesses).
 64  */
 65 int no_unaligned_warning;
 66 int unaligned_dump_stack;
 67 
 68 /*
 69  * For M-unit:
 70  *
 71  *  opcode |   m  |   x6    |
 72  * --------|------|---------|
 73  * [40-37] | [36] | [35:30] |
 74  * --------|------|---------|
 75  *     4   |   1  |    6    | = 11 bits
 76  * --------------------------
 77  * However bits [31:30] are not directly useful to distinguish between
 78  * load/store so we can use [35:32] instead, which gives the following
 79  * mask ([40:32]) using 9 bits. The 'e' comes from the fact that we defer
 80  * checking the m-bit until later in the load/store emulation.
 81  */
 82 #define IA64_OPCODE_MASK        0x1ef
 83 #define IA64_OPCODE_SHIFT       32
 84 
 85 /*
 86  * Table C-28 Integer Load/Store
 87  *
 88  * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
 89  *
 90  * ld8.fill, st8.fill  MUST be aligned because the RNATs are based on
 91  * the address (bits [8:3]), so we must failed.
 92  */
 93 #define LD_OP            0x080
 94 #define LDS_OP           0x081
 95 #define LDA_OP           0x082
 96 #define LDSA_OP          0x083
 97 #define LDBIAS_OP        0x084
 98 #define LDACQ_OP         0x085
 99 /* 0x086, 0x087 are not relevant */
100 #define LDCCLR_OP        0x088
101 #define LDCNC_OP         0x089
102 #define LDCCLRACQ_OP     0x08a
103 #define ST_OP            0x08c
104 #define STREL_OP         0x08d
105 /* 0x08e,0x8f are not relevant */
106 
107 /*
108  * Table C-29 Integer Load +Reg
109  *
110  * we use the ld->m (bit [36:36]) field to determine whether or not we have
111  * a load/store of this form.
112  */
113 
114 /*
115  * Table C-30 Integer Load/Store +Imm
116  *
117  * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
118  *
119  * ld8.fill, st8.fill  must be aligned because the Nat register are based on
120  * the address, so we must fail and the program must be fixed.
121  */
122 #define LD_IMM_OP            0x0a0
123 #define LDS_IMM_OP           0x0a1
124 #define LDA_IMM_OP           0x0a2
125 #define LDSA_IMM_OP          0x0a3
126 #define LDBIAS_IMM_OP        0x0a4
127 #define LDACQ_IMM_OP         0x0a5
128 /* 0x0a6, 0xa7 are not relevant */
129 #define LDCCLR_IMM_OP        0x0a8
130 #define LDCNC_IMM_OP         0x0a9
131 #define LDCCLRACQ_IMM_OP     0x0aa
132 #define ST_IMM_OP            0x0ac
133 #define STREL_IMM_OP         0x0ad
134 /* 0x0ae,0xaf are not relevant */
135 
136 /*
137  * Table C-32 Floating-point Load/Store
138  */
139 #define LDF_OP           0x0c0
140 #define LDFS_OP          0x0c1
141 #define LDFA_OP          0x0c2
142 #define LDFSA_OP         0x0c3
143 /* 0x0c6 is irrelevant */
144 #define LDFCCLR_OP       0x0c8
145 #define LDFCNC_OP        0x0c9
146 /* 0x0cb is irrelevant  */
147 #define STF_OP           0x0cc
148 
149 /*
150  * Table C-33 Floating-point Load +Reg
151  *
152  * we use the ld->m (bit [36:36]) field to determine whether or not we have
153  * a load/store of this form.
154  */
155 
156 /*
157  * Table C-34 Floating-point Load/Store +Imm
158  */
159 #define LDF_IMM_OP       0x0e0
160 #define LDFS_IMM_OP      0x0e1
161 #define LDFA_IMM_OP      0x0e2
162 #define LDFSA_IMM_OP     0x0e3
163 /* 0x0e6 is irrelevant */
164 #define LDFCCLR_IMM_OP   0x0e8
165 #define LDFCNC_IMM_OP    0x0e9
166 #define STF_IMM_OP       0x0ec
167 
168 typedef struct {
169         unsigned long    qp:6;  /* [0:5]   */
170         unsigned long    r1:7;  /* [6:12]  */
171         unsigned long   imm:7;  /* [13:19] */
172         unsigned long    r3:7;  /* [20:26] */
173         unsigned long     x:1;  /* [27:27] */
174         unsigned long  hint:2;  /* [28:29] */
175         unsigned long x6_sz:2;  /* [30:31] */
176         unsigned long x6_op:4;  /* [32:35], x6 = x6_sz|x6_op */
177         unsigned long     m:1;  /* [36:36] */
178         unsigned long    op:4;  /* [37:40] */
179         unsigned long   pad:23; /* [41:63] */
180 } load_store_t;
181 
182 
183 typedef enum {
184         UPD_IMMEDIATE,  /* ldXZ r1=[r3],imm(9) */
185         UPD_REG         /* ldXZ r1=[r3],r2     */
186 } update_t;
187 
188 /*
189  * We use tables to keep track of the offsets of registers in the saved state.
190  * This way we save having big switch/case statements.
191  *
192  * We use bit 0 to indicate switch_stack or pt_regs.
193  * The offset is simply shifted by 1 bit.
194  * A 2-byte value should be enough to hold any kind of offset
195  *
196  * In case the calling convention changes (and thus pt_regs/switch_stack)
197  * simply use RSW instead of RPT or vice-versa.
198  */
199 
200 #define RPO(x)  ((size_t) &((struct pt_regs *)0)->x)
201 #define RSO(x)  ((size_t) &((struct switch_stack *)0)->x)
202 
203 #define RPT(x)          (RPO(x) << 1)
204 #define RSW(x)          (1| RSO(x)<<1)
205 
206 #define GR_OFFS(x)      (gr_info[x]>>1)
207 #define GR_IN_SW(x)     (gr_info[x] & 0x1)
208 
209 #define FR_OFFS(x)      (fr_info[x]>>1)
210 #define FR_IN_SW(x)     (fr_info[x] & 0x1)
211 
212 static u16 gr_info[32]={
213         0,                      /* r0 is read-only : WE SHOULD NEVER GET THIS */
214 
215         RPT(r1), RPT(r2), RPT(r3),
216 
217         RSW(r4), RSW(r5), RSW(r6), RSW(r7),
218 
219         RPT(r8), RPT(r9), RPT(r10), RPT(r11),
220         RPT(r12), RPT(r13), RPT(r14), RPT(r15),
221 
222         RPT(r16), RPT(r17), RPT(r18), RPT(r19),
223         RPT(r20), RPT(r21), RPT(r22), RPT(r23),
224         RPT(r24), RPT(r25), RPT(r26), RPT(r27),
225         RPT(r28), RPT(r29), RPT(r30), RPT(r31)
226 };
227 
228 static u16 fr_info[32]={
229         0,                      /* constant : WE SHOULD NEVER GET THIS */
230         0,                      /* constant : WE SHOULD NEVER GET THIS */
231 
232         RSW(f2), RSW(f3), RSW(f4), RSW(f5),
233 
234         RPT(f6), RPT(f7), RPT(f8), RPT(f9),
235         RPT(f10), RPT(f11),
236 
237         RSW(f12), RSW(f13), RSW(f14),
238         RSW(f15), RSW(f16), RSW(f17), RSW(f18), RSW(f19),
239         RSW(f20), RSW(f21), RSW(f22), RSW(f23), RSW(f24),
240         RSW(f25), RSW(f26), RSW(f27), RSW(f28), RSW(f29),
241         RSW(f30), RSW(f31)
242 };
243 
244 /* Invalidate ALAT entry for integer register REGNO.  */
245 static void
246 invala_gr (int regno)
247 {
248 #       define F(reg)   case reg: ia64_invala_gr(reg); break
249 
250         switch (regno) {
251                 F(  0); F(  1); F(  2); F(  3); F(  4); F(  5); F(  6); F(  7);
252                 F(  8); F(  9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15);
253                 F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23);
254                 F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31);
255                 F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39);
256                 F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47);
257                 F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55);
258                 F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63);
259                 F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71);
260                 F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79);
261                 F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87);
262                 F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95);
263                 F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103);
264                 F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111);
265                 F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119);
266                 F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127);
267         }
268 #       undef F
269 }
270 
271 /* Invalidate ALAT entry for floating-point register REGNO.  */
272 static void
273 invala_fr (int regno)
274 {
275 #       define F(reg)   case reg: ia64_invala_fr(reg); break
276 
277         switch (regno) {
278                 F(  0); F(  1); F(  2); F(  3); F(  4); F(  5); F(  6); F(  7);
279                 F(  8); F(  9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15);
280                 F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23);
281                 F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31);
282                 F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39);
283                 F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47);
284                 F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55);
285                 F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63);
286                 F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71);
287                 F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79);
288                 F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87);
289                 F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95);
290                 F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103);
291                 F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111);
292                 F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119);
293                 F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127);
294         }
295 #       undef F
296 }
297 
298 static inline unsigned long
299 rotate_reg (unsigned long sor, unsigned long rrb, unsigned long reg)
300 {
301         reg += rrb;
302         if (reg >= sor)
303                 reg -= sor;
304         return reg;
305 }
306 
307 static void
308 set_rse_reg (struct pt_regs *regs, unsigned long r1, unsigned long val, int nat)
309 {
310         struct switch_stack *sw = (struct switch_stack *) regs - 1;
311         unsigned long *bsp, *bspstore, *addr, *rnat_addr, *ubs_end;
312         unsigned long *kbs = (void *) current + IA64_RBS_OFFSET;
313         unsigned long rnats, nat_mask;
314         unsigned long on_kbs;
315         long sof = (regs->cr_ifs) & 0x7f;
316         long sor = 8 * ((regs->cr_ifs >> 14) & 0xf);
317         long rrb_gr = (regs->cr_ifs >> 18) & 0x7f;
318         long ridx = r1 - 32;
319 
320         if (ridx >= sof) {
321                 /* this should never happen, as the "rsvd register fault" has higher priority */
322                 DPRINT("ignoring write to r%lu; only %lu registers are allocated!\n", r1, sof);
323                 return;
324         }
325 
326         if (ridx < sor)
327                 ridx = rotate_reg(sor, rrb_gr, ridx);
328 
329         DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n",
330                r1, sw->ar_bspstore, regs->ar_bspstore, sof, (regs->cr_ifs >> 7) & 0x7f, ridx);
331 
332         on_kbs = ia64_rse_num_regs(kbs, (unsigned long *) sw->ar_bspstore);
333         addr = ia64_rse_skip_regs((unsigned long *) sw->ar_bspstore, -sof + ridx);
334         if (addr >= kbs) {
335                 /* the register is on the kernel backing store: easy... */
336                 rnat_addr = ia64_rse_rnat_addr(addr);
337                 if ((unsigned long) rnat_addr >= sw->ar_bspstore)
338                         rnat_addr = &sw->ar_rnat;
339                 nat_mask = 1UL << ia64_rse_slot_num(addr);
340 
341                 *addr = val;
342                 if (nat)
343                         *rnat_addr |=  nat_mask;
344                 else
345                         *rnat_addr &= ~nat_mask;
346                 return;
347         }
348 
349         if (!user_stack(current, regs)) {
350                 DPRINT("ignoring kernel write to r%lu; register isn't on the kernel RBS!", r1);
351                 return;
352         }
353 
354         bspstore = (unsigned long *)regs->ar_bspstore;
355         ubs_end = ia64_rse_skip_regs(bspstore, on_kbs);
356         bsp     = ia64_rse_skip_regs(ubs_end, -sof);
357         addr    = ia64_rse_skip_regs(bsp, ridx);
358 
359         DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end, (void *) bsp, (void *) addr);
360 
361         ia64_poke(current, sw, (unsigned long) ubs_end, (unsigned long) addr, val);
362 
363         rnat_addr = ia64_rse_rnat_addr(addr);
364 
365         ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, &rnats);
366         DPRINT("rnat @%p = 0x%lx nat=%d old nat=%ld\n",
367                (void *) rnat_addr, rnats, nat, (rnats >> ia64_rse_slot_num(addr)) & 1);
368 
369         nat_mask = 1UL << ia64_rse_slot_num(addr);
370         if (nat)
371                 rnats |=  nat_mask;
372         else
373                 rnats &= ~nat_mask;
374         ia64_poke(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, rnats);
375 
376         DPRINT("rnat changed to @%p = 0x%lx\n", (void *) rnat_addr, rnats);
377 }
378 
379 
380 static void
381 get_rse_reg (struct pt_regs *regs, unsigned long r1, unsigned long *val, int *nat)
382 {
383         struct switch_stack *sw = (struct switch_stack *) regs - 1;
384         unsigned long *bsp, *addr, *rnat_addr, *ubs_end, *bspstore;
385         unsigned long *kbs = (void *) current + IA64_RBS_OFFSET;
386         unsigned long rnats, nat_mask;
387         unsigned long on_kbs;
388         long sof = (regs->cr_ifs) & 0x7f;
389         long sor = 8 * ((regs->cr_ifs >> 14) & 0xf);
390         long rrb_gr = (regs->cr_ifs >> 18) & 0x7f;
391         long ridx = r1 - 32;
392 
393         if (ridx >= sof) {
394                 /* read of out-of-frame register returns an undefined value; 0 in our case.  */
395                 DPRINT("ignoring read from r%lu; only %lu registers are allocated!\n", r1, sof);
396                 goto fail;
397         }
398 
399         if (ridx < sor)
400                 ridx = rotate_reg(sor, rrb_gr, ridx);
401 
402         DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n",
403                r1, sw->ar_bspstore, regs->ar_bspstore, sof, (regs->cr_ifs >> 7) & 0x7f, ridx);
404 
405         on_kbs = ia64_rse_num_regs(kbs, (unsigned long *) sw->ar_bspstore);
406         addr = ia64_rse_skip_regs((unsigned long *) sw->ar_bspstore, -sof + ridx);
407         if (addr >= kbs) {
408                 /* the register is on the kernel backing store: easy... */
409                 *val = *addr;
410                 if (nat) {
411                         rnat_addr = ia64_rse_rnat_addr(addr);
412                         if ((unsigned long) rnat_addr >= sw->ar_bspstore)
413                                 rnat_addr = &sw->ar_rnat;
414                         nat_mask = 1UL << ia64_rse_slot_num(addr);
415                         *nat = (*rnat_addr & nat_mask) != 0;
416                 }
417                 return;
418         }
419 
420         if (!user_stack(current, regs)) {
421                 DPRINT("ignoring kernel read of r%lu; register isn't on the RBS!", r1);
422                 goto fail;
423         }
424 
425         bspstore = (unsigned long *)regs->ar_bspstore;
426         ubs_end = ia64_rse_skip_regs(bspstore, on_kbs);
427         bsp     = ia64_rse_skip_regs(ubs_end, -sof);
428         addr    = ia64_rse_skip_regs(bsp, ridx);
429 
430         DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end, (void *) bsp, (void *) addr);
431 
432         ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) addr, val);
433 
434         if (nat) {
435                 rnat_addr = ia64_rse_rnat_addr(addr);
436                 nat_mask = 1UL << ia64_rse_slot_num(addr);
437 
438                 DPRINT("rnat @%p = 0x%lx\n", (void *) rnat_addr, rnats);
439 
440                 ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, &rnats);
441                 *nat = (rnats & nat_mask) != 0;
442         }
443         return;
444 
445   fail:
446         *val = 0;
447         if (nat)
448                 *nat = 0;
449         return;
450 }
451 
452 
453 static void
454 setreg (unsigned long regnum, unsigned long val, int nat, struct pt_regs *regs)
455 {
456         struct switch_stack *sw = (struct switch_stack *) regs - 1;
457         unsigned long addr;
458         unsigned long bitmask;
459         unsigned long *unat;
460 
461         /*
462          * First takes care of stacked registers
463          */
464         if (regnum >= IA64_FIRST_STACKED_GR) {
465                 set_rse_reg(regs, regnum, val, nat);
466                 return;
467         }
468 
469         /*
470          * Using r0 as a target raises a General Exception fault which has higher priority
471          * than the Unaligned Reference fault.
472          */
473 
474         /*
475          * Now look at registers in [0-31] range and init correct UNAT
476          */
477         if (GR_IN_SW(regnum)) {
478                 addr = (unsigned long)sw;
479                 unat = &sw->ar_unat;
480         } else {
481                 addr = (unsigned long)regs;
482                 unat = &sw->caller_unat;
483         }
484         DPRINT("tmp_base=%lx switch_stack=%s offset=%d\n",
485                addr, unat==&sw->ar_unat ? "yes":"no", GR_OFFS(regnum));
486         /*
487          * add offset from base of struct
488          * and do it !
489          */
490         addr += GR_OFFS(regnum);
491 
492         *(unsigned long *)addr = val;
493 
494         /*
495          * We need to clear the corresponding UNAT bit to fully emulate the load
496          * UNAT bit_pos = GR[r3]{8:3} form EAS-2.4
497          */
498         bitmask   = 1UL << (addr >> 3 & 0x3f);
499         DPRINT("*0x%lx=0x%lx NaT=%d prev_unat @%p=%lx\n", addr, val, nat, (void *) unat, *unat);
500         if (nat) {
501                 *unat |= bitmask;
502         } else {
503                 *unat &= ~bitmask;
504         }
505         DPRINT("*0x%lx=0x%lx NaT=%d new unat: %p=%lx\n", addr, val, nat, (void *) unat,*unat);
506 }
507 
508 /*
509  * Return the (rotated) index for floating point register REGNUM (REGNUM must be in the
510  * range from 32-127, result is in the range from 0-95.
511  */
512 static inline unsigned long
513 fph_index (struct pt_regs *regs, long regnum)
514 {
515         unsigned long rrb_fr = (regs->cr_ifs >> 25) & 0x7f;
516         return rotate_reg(96, rrb_fr, (regnum - IA64_FIRST_ROTATING_FR));
517 }
518 
519 static void
520 setfpreg (unsigned long regnum, struct ia64_fpreg *fpval, struct pt_regs *regs)
521 {
522         struct switch_stack *sw = (struct switch_stack *)regs - 1;
523         unsigned long addr;
524 
525         /*
526          * From EAS-2.5: FPDisableFault has higher priority than Unaligned
527          * Fault. Thus, when we get here, we know the partition is enabled.
528          * To update f32-f127, there are three choices:
529          *
530          *      (1) save f32-f127 to thread.fph and update the values there
531          *      (2) use a gigantic switch statement to directly access the registers
532          *      (3) generate code on the fly to update the desired register
533          *
534          * For now, we are using approach (1).
535          */
536         if (regnum >= IA64_FIRST_ROTATING_FR) {
537                 ia64_sync_fph(current);
538                 current->thread.fph[fph_index(regs, regnum)] = *fpval;
539         } else {
540                 /*
541                  * pt_regs or switch_stack ?
542                  */
543                 if (FR_IN_SW(regnum)) {
544                         addr = (unsigned long)sw;
545                 } else {
546                         addr = (unsigned long)regs;
547                 }
548 
549                 DPRINT("tmp_base=%lx offset=%d\n", addr, FR_OFFS(regnum));
550 
551                 addr += FR_OFFS(regnum);
552                 *(struct ia64_fpreg *)addr = *fpval;
553 
554                 /*
555                  * mark the low partition as being used now
556                  *
557                  * It is highly unlikely that this bit is not already set, but
558                  * let's do it for safety.
559                  */
560                 regs->cr_ipsr |= IA64_PSR_MFL;
561         }
562 }
563 
564 /*
565  * Those 2 inline functions generate the spilled versions of the constant floating point
566  * registers which can be used with stfX
567  */
568 static inline void
569 float_spill_f0 (struct ia64_fpreg *final)
570 {
571         ia64_stf_spill(final, 0);
572 }
573 
574 static inline void
575 float_spill_f1 (struct ia64_fpreg *final)
576 {
577         ia64_stf_spill(final, 1);
578 }
579 
580 static void
581 getfpreg (unsigned long regnum, struct ia64_fpreg *fpval, struct pt_regs *regs)
582 {
583         struct switch_stack *sw = (struct switch_stack *) regs - 1;
584         unsigned long addr;
585 
586         /*
587          * From EAS-2.5: FPDisableFault has higher priority than
588          * Unaligned Fault. Thus, when we get here, we know the partition is
589          * enabled.
590          *
591          * When regnum > 31, the register is still live and we need to force a save
592          * to current->thread.fph to get access to it.  See discussion in setfpreg()
593          * for reasons and other ways of doing this.
594          */
595         if (regnum >= IA64_FIRST_ROTATING_FR) {
596                 ia64_flush_fph(current);
597                 *fpval = current->thread.fph[fph_index(regs, regnum)];
598         } else {
599                 /*
600                  * f0 = 0.0, f1= 1.0. Those registers are constant and are thus
601                  * not saved, we must generate their spilled form on the fly
602                  */
603                 switch(regnum) {
604                 case 0:
605                         float_spill_f0(fpval);
606                         break;
607                 case 1:
608                         float_spill_f1(fpval);
609                         break;
610                 default:
611                         /*
612                          * pt_regs or switch_stack ?
613                          */
614                         addr =  FR_IN_SW(regnum) ? (unsigned long)sw
615                                                  : (unsigned long)regs;
616 
617                         DPRINT("is_sw=%d tmp_base=%lx offset=0x%x\n",
618                                FR_IN_SW(regnum), addr, FR_OFFS(regnum));
619 
620                         addr  += FR_OFFS(regnum);
621                         *fpval = *(struct ia64_fpreg *)addr;
622                 }
623         }
624 }
625 
626 
627 static void
628 getreg (unsigned long regnum, unsigned long *val, int *nat, struct pt_regs *regs)
629 {
630         struct switch_stack *sw = (struct switch_stack *) regs - 1;
631         unsigned long addr, *unat;
632 
633         if (regnum >= IA64_FIRST_STACKED_GR) {
634                 get_rse_reg(regs, regnum, val, nat);
635                 return;
636         }
637 
638         /*
639          * take care of r0 (read-only always evaluate to 0)
640          */
641         if (regnum == 0) {
642                 *val = 0;
643                 if (nat)
644                         *nat = 0;
645                 return;
646         }
647 
648         /*
649          * Now look at registers in [0-31] range and init correct UNAT
650          */
651         if (GR_IN_SW(regnum)) {
652                 addr = (unsigned long)sw;
653                 unat = &sw->ar_unat;
654         } else {
655                 addr = (unsigned long)regs;
656                 unat = &sw->caller_unat;
657         }
658 
659         DPRINT("addr_base=%lx offset=0x%x\n", addr,  GR_OFFS(regnum));
660 
661         addr += GR_OFFS(regnum);
662 
663         *val  = *(unsigned long *)addr;
664 
665         /*
666          * do it only when requested
667          */
668         if (nat)
669                 *nat  = (*unat >> (addr >> 3 & 0x3f)) & 0x1UL;
670 }
671 
672 static void
673 emulate_load_updates (update_t type, load_store_t ld, struct pt_regs *regs, unsigned long ifa)
674 {
675         /*
676          * IMPORTANT:
677          * Given the way we handle unaligned speculative loads, we should
678          * not get to this point in the code but we keep this sanity check,
679          * just in case.
680          */
681         if (ld.x6_op == 1 || ld.x6_op == 3) {
682                 printk(KERN_ERR "%s: register update on speculative load, error\n", __func__);
683                 if (die_if_kernel("unaligned reference on speculative load with register update\n",
684                                   regs, 30))
685                         return;
686         }
687 
688 
689         /*
690          * at this point, we know that the base register to update is valid i.e.,
691          * it's not r0
692          */
693         if (type == UPD_IMMEDIATE) {
694                 unsigned long imm;
695 
696                 /*
697                  * Load +Imm: ldXZ r1=[r3],imm(9)
698                  *
699                  *
700                  * form imm9: [13:19] contain the first 7 bits
701                  */
702                 imm = ld.x << 7 | ld.imm;
703 
704                 /*
705                  * sign extend (1+8bits) if m set
706                  */
707                 if (ld.m) imm |= SIGN_EXT9;
708 
709                 /*
710                  * ifa == r3 and we know that the NaT bit on r3 was clear so
711                  * we can directly use ifa.
712                  */
713                 ifa += imm;
714 
715                 setreg(ld.r3, ifa, 0, regs);
716 
717                 DPRINT("ld.x=%d ld.m=%d imm=%ld r3=0x%lx\n", ld.x, ld.m, imm, ifa);
718 
719         } else if (ld.m) {
720                 unsigned long r2;
721                 int nat_r2;
722 
723                 /*
724                  * Load +Reg Opcode: ldXZ r1=[r3],r2
725                  *
726                  * Note: that we update r3 even in the case of ldfX.a
727                  * (where the load does not happen)
728                  *
729                  * The way the load algorithm works, we know that r3 does not
730                  * have its NaT bit set (would have gotten NaT consumption
731                  * before getting the unaligned fault). So we can use ifa
732                  * which equals r3 at this point.
733                  *
734                  * IMPORTANT:
735                  * The above statement holds ONLY because we know that we
736                  * never reach this code when trying to do a ldX.s.
737                  * If we ever make it to here on an ldfX.s then
738                  */
739                 getreg(ld.imm, &r2, &nat_r2, regs);
740 
741                 ifa += r2;
742 
743                 /*
744                  * propagate Nat r2 -> r3
745                  */
746                 setreg(ld.r3, ifa, nat_r2, regs);
747 
748                 DPRINT("imm=%d r2=%ld r3=0x%lx nat_r2=%d\n",ld.imm, r2, ifa, nat_r2);
749         }
750 }
751 
752 
753 static int
754 emulate_load_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
755 {
756         unsigned int len = 1 << ld.x6_sz;
757         unsigned long val = 0;
758 
759         /*
760          * r0, as target, doesn't need to be checked because Illegal Instruction
761          * faults have higher priority than unaligned faults.
762          *
763          * r0 cannot be found as the base as it would never generate an
764          * unaligned reference.
765          */
766 
767         /*
768          * ldX.a we will emulate load and also invalidate the ALAT entry.
769          * See comment below for explanation on how we handle ldX.a
770          */
771 
772         if (len != 2 && len != 4 && len != 8) {
773                 DPRINT("unknown size: x6=%d\n", ld.x6_sz);
774                 return -1;
775         }
776         /* this assumes little-endian byte-order: */
777         if (copy_from_user(&val, (void __user *) ifa, len))
778                 return -1;
779         setreg(ld.r1, val, 0, regs);
780 
781         /*
782          * check for updates on any kind of loads
783          */
784         if (ld.op == 0x5 || ld.m)
785                 emulate_load_updates(ld.op == 0x5 ? UPD_IMMEDIATE: UPD_REG, ld, regs, ifa);
786 
787         /*
788          * handling of various loads (based on EAS2.4):
789          *
790          * ldX.acq (ordered load):
791          *      - acquire semantics would have been used, so force fence instead.
792          *
793          * ldX.c.clr (check load and clear):
794          *      - if we get to this handler, it's because the entry was not in the ALAT.
795          *        Therefore the operation reverts to a normal load
796          *
797          * ldX.c.nc (check load no clear):
798          *      - same as previous one
799          *
800          * ldX.c.clr.acq (ordered check load and clear):
801          *      - same as above for c.clr part. The load needs to have acquire semantics. So
802          *        we use the fence semantics which is stronger and thus ensures correctness.
803          *
804          * ldX.a (advanced load):
805          *      - suppose ldX.a r1=[r3]. If we get to the unaligned trap it's because the
806          *        address doesn't match requested size alignment. This means that we would
807          *        possibly need more than one load to get the result.
808          *
809          *        The load part can be handled just like a normal load, however the difficult
810          *        part is to get the right thing into the ALAT. The critical piece of information
811          *        in the base address of the load & size. To do that, a ld.a must be executed,
812          *        clearly any address can be pushed into the table by using ld1.a r1=[r3]. Now
813          *        if we use the same target register, we will be okay for the check.a instruction.
814          *        If we look at the store, basically a stX [r3]=r1 checks the ALAT  for any entry
815          *        which would overlap within [r3,r3+X] (the size of the load was store in the
816          *        ALAT). If such an entry is found the entry is invalidated. But this is not good
817          *        enough, take the following example:
818          *              r3=3
819          *              ld4.a r1=[r3]
820          *
821          *        Could be emulated by doing:
822          *              ld1.a r1=[r3],1
823          *              store to temporary;
824          *              ld1.a r1=[r3],1
825          *              store & shift to temporary;
826          *              ld1.a r1=[r3],1
827          *              store & shift to temporary;
828          *              ld1.a r1=[r3]
829          *              store & shift to temporary;
830          *              r1=temporary
831          *
832          *        So in this case, you would get the right value is r1 but the wrong info in
833          *        the ALAT.  Notice that you could do it in reverse to finish with address 3
834          *        but you would still get the size wrong.  To get the size right, one needs to
835          *        execute exactly the same kind of load. You could do it from a aligned
836          *        temporary location, but you would get the address wrong.
837          *
838          *        So no matter what, it is not possible to emulate an advanced load
839          *        correctly. But is that really critical ?
840          *
841          *        We will always convert ld.a into a normal load with ALAT invalidated.  This
842          *        will enable compiler to do optimization where certain code path after ld.a
843          *        is not required to have ld.c/chk.a, e.g., code path with no intervening stores.
844          *
845          *        If there is a store after the advanced load, one must either do a ld.c.* or
846          *        chk.a.* to reuse the value stored in the ALAT. Both can "fail" (meaning no
847          *        entry found in ALAT), and that's perfectly ok because:
848          *
849          *              - ld.c.*, if the entry is not present a  normal load is executed
850          *              - chk.a.*, if the entry is not present, execution jumps to recovery code
851          *
852          *        In either case, the load can be potentially retried in another form.
853          *
854          *        ALAT must be invalidated for the register (so that chk.a or ld.c don't pick
855          *        up a stale entry later). The register base update MUST also be performed.
856          */
857 
858         /*
859          * when the load has the .acq completer then
860          * use ordering fence.
861          */
862         if (ld.x6_op == 0x5 || ld.x6_op == 0xa)
863                 mb();
864 
865         /*
866          * invalidate ALAT entry in case of advanced load
867          */
868         if (ld.x6_op == 0x2)
869                 invala_gr(ld.r1);
870 
871         return 0;
872 }
873 
874 static int
875 emulate_store_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
876 {
877         unsigned long r2;
878         unsigned int len = 1 << ld.x6_sz;
879 
880         /*
881          * if we get to this handler, Nat bits on both r3 and r2 have already
882          * been checked. so we don't need to do it
883          *
884          * extract the value to be stored
885          */
886         getreg(ld.imm, &r2, NULL, regs);
887 
888         /*
889          * we rely on the macros in unaligned.h for now i.e.,
890          * we let the compiler figure out how to read memory gracefully.
891          *
892          * We need this switch/case because the way the inline function
893          * works. The code is optimized by the compiler and looks like
894          * a single switch/case.
895          */
896         DPRINT("st%d [%lx]=%lx\n", len, ifa, r2);
897 
898         if (len != 2 && len != 4 && len != 8) {
899                 DPRINT("unknown size: x6=%d\n", ld.x6_sz);
900                 return -1;
901         }
902 
903         /* this assumes little-endian byte-order: */
904         if (copy_to_user((void __user *) ifa, &r2, len))
905                 return -1;
906 
907         /*
908          * stX [r3]=r2,imm(9)
909          *
910          * NOTE:
911          * ld.r3 can never be r0, because r0 would not generate an
912          * unaligned access.
913          */
914         if (ld.op == 0x5) {
915                 unsigned long imm;
916 
917                 /*
918                  * form imm9: [12:6] contain first 7bits
919                  */
920                 imm = ld.x << 7 | ld.r1;
921                 /*
922                  * sign extend (8bits) if m set
923                  */
924                 if (ld.m) imm |= SIGN_EXT9;
925                 /*
926                  * ifa == r3 (NaT is necessarily cleared)
927                  */
928                 ifa += imm;
929 
930                 DPRINT("imm=%lx r3=%lx\n", imm, ifa);
931 
932                 setreg(ld.r3, ifa, 0, regs);
933         }
934         /*
935          * we don't have alat_invalidate_multiple() so we need
936          * to do the complete flush :-<<
937          */
938         ia64_invala();
939 
940         /*
941          * stX.rel: use fence instead of release
942          */
943         if (ld.x6_op == 0xd)
944                 mb();
945 
946         return 0;
947 }
948 
949 /*
950  * floating point operations sizes in bytes
951  */
952 static const unsigned char float_fsz[4]={
953         10, /* extended precision (e) */
954         8,  /* integer (8)            */
955         4,  /* single precision (s)   */
956         8   /* double precision (d)   */
957 };
958 
959 static inline void
960 mem2float_extended (struct ia64_fpreg *init, struct ia64_fpreg *final)
961 {
962         ia64_ldfe(6, init);
963         ia64_stop();
964         ia64_stf_spill(final, 6);
965 }
966 
967 static inline void
968 mem2float_integer (struct ia64_fpreg *init, struct ia64_fpreg *final)
969 {
970         ia64_ldf8(6, init);
971         ia64_stop();
972         ia64_stf_spill(final, 6);
973 }
974 
975 static inline void
976 mem2float_single (struct ia64_fpreg *init, struct ia64_fpreg *final)
977 {
978         ia64_ldfs(6, init);
979         ia64_stop();
980         ia64_stf_spill(final, 6);
981 }
982 
983 static inline void
984 mem2float_double (struct ia64_fpreg *init, struct ia64_fpreg *final)
985 {
986         ia64_ldfd(6, init);
987         ia64_stop();
988         ia64_stf_spill(final, 6);
989 }
990 
991 static inline void
992 float2mem_extended (struct ia64_fpreg *init, struct ia64_fpreg *final)
993 {
994         ia64_ldf_fill(6, init);
995         ia64_stop();
996         ia64_stfe(final, 6);
997 }
998 
999 static inline void
1000 float2mem_integer (struct ia64_fpreg *init, struct ia64_fpreg *final)
1001 {
1002         ia64_ldf_fill(6, init);
1003         ia64_stop();
1004         ia64_stf8(final, 6);
1005 }
1006 
1007 static inline void
1008 float2mem_single (struct ia64_fpreg *init, struct ia64_fpreg *final)
1009 {
1010         ia64_ldf_fill(6, init);
1011         ia64_stop();
1012         ia64_stfs(final, 6);
1013 }
1014 
1015 static inline void
1016 float2mem_double (struct ia64_fpreg *init, struct ia64_fpreg *final)
1017 {
1018         ia64_ldf_fill(6, init);
1019         ia64_stop();
1020         ia64_stfd(final, 6);
1021 }
1022 
1023 static int
1024 emulate_load_floatpair (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
1025 {
1026         struct ia64_fpreg fpr_init[2];
1027         struct ia64_fpreg fpr_final[2];
1028         unsigned long len = float_fsz[ld.x6_sz];
1029 
1030         /*
1031          * fr0 & fr1 don't need to be checked because Illegal Instruction faults have
1032          * higher priority than unaligned faults.
1033          *
1034          * r0 cannot be found as the base as it would never generate an unaligned
1035          * reference.
1036          */
1037 
1038         /*
1039          * make sure we get clean buffers
1040          */
1041         memset(&fpr_init, 0, sizeof(fpr_init));
1042         memset(&fpr_final, 0, sizeof(fpr_final));
1043 
1044         /*
1045          * ldfpX.a: we don't try to emulate anything but we must
1046          * invalidate the ALAT entry and execute updates, if any.
1047          */
1048         if (ld.x6_op != 0x2) {
1049                 /*
1050                  * This assumes little-endian byte-order.  Note that there is no "ldfpe"
1051                  * instruction:
1052                  */
1053                 if (copy_from_user(&fpr_init[0], (void __user *) ifa, len)
1054                     || copy_from_user(&fpr_init[1], (void __user *) (ifa + len), len))
1055                         return -1;
1056 
1057                 DPRINT("ld.r1=%d ld.imm=%d x6_sz=%d\n", ld.r1, ld.imm, ld.x6_sz);
1058                 DDUMP("frp_init =", &fpr_init, 2*len);
1059                 /*
1060                  * XXX fixme
1061                  * Could optimize inlines by using ldfpX & 2 spills
1062                  */
1063                 switch( ld.x6_sz ) {
1064                         case 0:
1065                                 mem2float_extended(&fpr_init[0], &fpr_final[0]);
1066                                 mem2float_extended(&fpr_init[1], &fpr_final[1]);
1067                                 break;
1068                         case 1:
1069                                 mem2float_integer(&fpr_init[0], &fpr_final[0]);
1070                                 mem2float_integer(&fpr_init[1], &fpr_final[1]);
1071                                 break;
1072                         case 2:
1073                                 mem2float_single(&fpr_init[0], &fpr_final[0]);
1074                                 mem2float_single(&fpr_init[1], &fpr_final[1]);
1075                                 break;
1076                         case 3:
1077                                 mem2float_double(&fpr_init[0], &fpr_final[0]);
1078                                 mem2float_double(&fpr_init[1], &fpr_final[1]);
1079                                 break;
1080                 }
1081                 DDUMP("fpr_final =", &fpr_final, 2*len);
1082                 /*
1083                  * XXX fixme
1084                  *
1085                  * A possible optimization would be to drop fpr_final and directly
1086                  * use the storage from the saved context i.e., the actual final
1087                  * destination (pt_regs, switch_stack or thread structure).
1088                  */
1089                 setfpreg(ld.r1, &fpr_final[0], regs);
1090                 setfpreg(ld.imm, &fpr_final[1], regs);
1091         }
1092 
1093         /*
1094          * Check for updates: only immediate updates are available for this
1095          * instruction.
1096          */
1097         if (ld.m) {
1098                 /*
1099                  * the immediate is implicit given the ldsz of the operation:
1100                  * single: 8 (2x4) and for  all others it's 16 (2x8)
1101                  */
1102                 ifa += len<<1;
1103 
1104                 /*
1105                  * IMPORTANT:
1106                  * the fact that we force the NaT of r3 to zero is ONLY valid
1107                  * as long as we don't come here with a ldfpX.s.
1108                  * For this reason we keep this sanity check
1109                  */
1110                 if (ld.x6_op == 1 || ld.x6_op == 3)
1111                         printk(KERN_ERR "%s: register update on speculative load pair, error\n",
1112                                __func__);
1113 
1114                 setreg(ld.r3, ifa, 0, regs);
1115         }
1116 
1117         /*
1118          * Invalidate ALAT entries, if any, for both registers.
1119          */
1120         if (ld.x6_op == 0x2) {
1121                 invala_fr(ld.r1);
1122                 invala_fr(ld.imm);
1123         }
1124         return 0;
1125 }
1126 
1127 
1128 static int
1129 emulate_load_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
1130 {
1131         struct ia64_fpreg fpr_init;
1132         struct ia64_fpreg fpr_final;
1133         unsigned long len = float_fsz[ld.x6_sz];
1134 
1135         /*
1136          * fr0 & fr1 don't need to be checked because Illegal Instruction
1137          * faults have higher priority than unaligned faults.
1138          *
1139          * r0 cannot be found as the base as it would never generate an
1140          * unaligned reference.
1141          */
1142 
1143         /*
1144          * make sure we get clean buffers
1145          */
1146         memset(&fpr_init,0, sizeof(fpr_init));
1147         memset(&fpr_final,0, sizeof(fpr_final));
1148 
1149         /*
1150          * ldfX.a we don't try to emulate anything but we must
1151          * invalidate the ALAT entry.
1152          * See comments in ldX for descriptions on how the various loads are handled.
1153          */
1154         if (ld.x6_op != 0x2) {
1155                 if (copy_from_user(&fpr_init, (void __user *) ifa, len))
1156                         return -1;
1157 
1158                 DPRINT("ld.r1=%d x6_sz=%d\n", ld.r1, ld.x6_sz);
1159                 DDUMP("fpr_init =", &fpr_init, len);
1160                 /*
1161                  * we only do something for x6_op={0,8,9}
1162                  */
1163                 switch( ld.x6_sz ) {
1164                         case 0:
1165                                 mem2float_extended(&fpr_init, &fpr_final);
1166                                 break;
1167                         case 1:
1168                                 mem2float_integer(&fpr_init, &fpr_final);
1169                                 break;
1170                         case 2:
1171                                 mem2float_single(&fpr_init, &fpr_final);
1172                                 break;
1173                         case 3:
1174                                 mem2float_double(&fpr_init, &fpr_final);
1175                                 break;
1176                 }
1177                 DDUMP("fpr_final =", &fpr_final, len);
1178                 /*
1179                  * XXX fixme
1180                  *
1181                  * A possible optimization would be to drop fpr_final and directly
1182                  * use the storage from the saved context i.e., the actual final
1183                  * destination (pt_regs, switch_stack or thread structure).
1184                  */
1185                 setfpreg(ld.r1, &fpr_final, regs);
1186         }
1187 
1188         /*
1189          * check for updates on any loads
1190          */
1191         if (ld.op == 0x7 || ld.m)
1192                 emulate_load_updates(ld.op == 0x7 ? UPD_IMMEDIATE: UPD_REG, ld, regs, ifa);
1193 
1194         /*
1195          * invalidate ALAT entry in case of advanced floating point loads
1196          */
1197         if (ld.x6_op == 0x2)
1198                 invala_fr(ld.r1);
1199 
1200         return 0;
1201 }
1202 
1203 
1204 static int
1205 emulate_store_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
1206 {
1207         struct ia64_fpreg fpr_init;
1208         struct ia64_fpreg fpr_final;
1209         unsigned long len = float_fsz[ld.x6_sz];
1210 
1211         /*
1212          * make sure we get clean buffers
1213          */
1214         memset(&fpr_init,0, sizeof(fpr_init));
1215         memset(&fpr_final,0, sizeof(fpr_final));
1216 
1217         /*
1218          * if we get to this handler, Nat bits on both r3 and r2 have already
1219          * been checked. so we don't need to do it
1220          *
1221          * extract the value to be stored
1222          */
1223         getfpreg(ld.imm, &fpr_init, regs);
1224         /*
1225          * during this step, we extract the spilled registers from the saved
1226          * context i.e., we refill. Then we store (no spill) to temporary
1227          * aligned location
1228          */
1229         switch( ld.x6_sz ) {
1230                 case 0:
1231                         float2mem_extended(&fpr_init, &fpr_final);
1232                         break;
1233                 case 1:
1234                         float2mem_integer(&fpr_init, &fpr_final);
1235                         break;
1236                 case 2:
1237                         float2mem_single(&fpr_init, &fpr_final);
1238                         break;
1239                 case 3:
1240                         float2mem_double(&fpr_init, &fpr_final);
1241                         break;
1242         }
1243         DPRINT("ld.r1=%d x6_sz=%d\n", ld.r1, ld.x6_sz);
1244         DDUMP("fpr_init =", &fpr_init, len);
1245         DDUMP("fpr_final =", &fpr_final, len);
1246 
1247         if (copy_to_user((void __user *) ifa, &fpr_final, len))
1248                 return -1;
1249 
1250         /*
1251          * stfX [r3]=r2,imm(9)
1252          *
1253          * NOTE:
1254          * ld.r3 can never be r0, because r0 would not generate an
1255          * unaligned access.
1256          */
1257         if (ld.op == 0x7) {
1258                 unsigned long imm;
1259 
1260                 /*
1261                  * form imm9: [12:6] contain first 7bits
1262                  */
1263                 imm = ld.x << 7 | ld.r1;
1264                 /*
1265                  * sign extend (8bits) if m set
1266                  */
1267                 if (ld.m)
1268                         imm |= SIGN_EXT9;
1269                 /*
1270                  * ifa == r3 (NaT is necessarily cleared)
1271                  */
1272                 ifa += imm;
1273 
1274                 DPRINT("imm=%lx r3=%lx\n", imm, ifa);
1275 
1276                 setreg(ld.r3, ifa, 0, regs);
1277         }
1278         /*
1279          * we don't have alat_invalidate_multiple() so we need
1280          * to do the complete flush :-<<
1281          */
1282         ia64_invala();
1283 
1284         return 0;
1285 }
1286 
1287 /*
1288  * Make sure we log the unaligned access, so that user/sysadmin can notice it and
1289  * eventually fix the program.  However, we don't want to do that for every access so we
1290  * pace it with jiffies.
1291  */
1292 static DEFINE_RATELIMIT_STATE(logging_rate_limit, 5 * HZ, 5);
1293 
1294 void
1295 ia64_handle_unaligned (unsigned long ifa, struct pt_regs *regs)
1296 {
1297         struct ia64_psr *ipsr = ia64_psr(regs);
1298         mm_segment_t old_fs = get_fs();
1299         unsigned long bundle[2];
1300         unsigned long opcode;
1301         const struct exception_table_entry *eh = NULL;
1302         union {
1303                 unsigned long l;
1304                 load_store_t insn;
1305         } u;
1306         int ret = -1;
1307 
1308         if (ia64_psr(regs)->be) {
1309                 /* we don't support big-endian accesses */
1310                 if (die_if_kernel("big-endian unaligned accesses are not supported", regs, 0))
1311                         return;
1312                 goto force_sigbus;
1313         }
1314 
1315         /*
1316          * Treat kernel accesses for which there is an exception handler entry the same as
1317          * user-level unaligned accesses.  Otherwise, a clever program could trick this
1318          * handler into reading an arbitrary kernel addresses...
1319          */
1320         if (!user_mode(regs))
1321                 eh = search_exception_tables(regs->cr_iip + ia64_psr(regs)->ri);
1322         if (user_mode(regs) || eh) {
1323                 if ((current->thread.flags & IA64_THREAD_UAC_SIGBUS) != 0)
1324                         goto force_sigbus;
1325 
1326                 if (!no_unaligned_warning &&
1327                     !(current->thread.flags & IA64_THREAD_UAC_NOPRINT) &&
1328                     __ratelimit(&logging_rate_limit))
1329                 {
1330                         char buf[200];  /* comm[] is at most 16 bytes... */
1331                         size_t len;
1332 
1333                         len = sprintf(buf, "%s(%d): unaligned access to 0x%016lx, "
1334                                       "ip=0x%016lx\n\r", current->comm,
1335                                       task_pid_nr(current),
1336                                       ifa, regs->cr_iip + ipsr->ri);
1337                         /*
1338                          * Don't call tty_write_message() if we're in the kernel; we might
1339                          * be holding locks...
1340                          */
1341                         if (user_mode(regs)) {
1342                                 struct tty_struct *tty = get_current_tty();
1343                                 tty_write_message(tty, buf);
1344                                 tty_kref_put(tty);
1345                         }
1346                         buf[len-1] = '\0';      /* drop '\r' */
1347                         /* watch for command names containing %s */
1348                         printk(KERN_WARNING "%s", buf);
1349                 } else {
1350                         if (no_unaligned_warning) {
1351                                 printk_once(KERN_WARNING "%s(%d) encountered an "
1352                                        "unaligned exception which required\n"
1353                                        "kernel assistance, which degrades "
1354                                        "the performance of the application.\n"
1355                                        "Unaligned exception warnings have "
1356                                        "been disabled by the system "
1357                                        "administrator\n"
1358                                        "echo 0 > /proc/sys/kernel/ignore-"
1359                                        "unaligned-usertrap to re-enable\n",
1360                                        current->comm, task_pid_nr(current));
1361                         }
1362                 }
1363         } else {
1364                 if (__ratelimit(&logging_rate_limit)) {
1365                         printk(KERN_WARNING "kernel unaligned access to 0x%016lx, ip=0x%016lx\n",
1366                                ifa, regs->cr_iip + ipsr->ri);
1367                         if (unaligned_dump_stack)
1368                                 dump_stack();
1369                 }
1370                 set_fs(KERNEL_DS);
1371         }
1372 
1373         DPRINT("iip=%lx ifa=%lx isr=%lx (ei=%d, sp=%d)\n",
1374                regs->cr_iip, ifa, regs->cr_ipsr, ipsr->ri, ipsr->it);
1375 
1376         if (__copy_from_user(bundle, (void __user *) regs->cr_iip, 16))
1377                 goto failure;
1378 
1379         /*
1380          * extract the instruction from the bundle given the slot number
1381          */
1382         switch (ipsr->ri) {
1383               default:
1384               case 0: u.l = (bundle[0] >>  5); break;
1385               case 1: u.l = (bundle[0] >> 46) | (bundle[1] << 18); break;
1386               case 2: u.l = (bundle[1] >> 23); break;
1387         }
1388         opcode = (u.l >> IA64_OPCODE_SHIFT) & IA64_OPCODE_MASK;
1389 
1390         DPRINT("opcode=%lx ld.qp=%d ld.r1=%d ld.imm=%d ld.r3=%d ld.x=%d ld.hint=%d "
1391                "ld.x6=0x%x ld.m=%d ld.op=%d\n", opcode, u.insn.qp, u.insn.r1, u.insn.imm,
1392                u.insn.r3, u.insn.x, u.insn.hint, u.insn.x6_sz, u.insn.m, u.insn.op);
1393 
1394         /*
1395          * IMPORTANT:
1396          * Notice that the switch statement DOES not cover all possible instructions
1397          * that DO generate unaligned references. This is made on purpose because for some
1398          * instructions it DOES NOT make sense to try and emulate the access. Sometimes it
1399          * is WRONG to try and emulate. Here is a list of instruction we don't emulate i.e.,
1400          * the program will get a signal and die:
1401          *
1402          *      load/store:
1403          *              - ldX.spill
1404          *              - stX.spill
1405          *      Reason: RNATs are based on addresses
1406          *              - ld16
1407          *              - st16
1408          *      Reason: ld16 and st16 are supposed to occur in a single
1409          *              memory op
1410          *
1411          *      synchronization:
1412          *              - cmpxchg
1413          *              - fetchadd
1414          *              - xchg
1415          *      Reason: ATOMIC operations cannot be emulated properly using multiple
1416          *              instructions.
1417          *
1418          *      speculative loads:
1419          *              - ldX.sZ
1420          *      Reason: side effects, code must be ready to deal with failure so simpler
1421          *              to let the load fail.
1422          * ---------------------------------------------------------------------------------
1423          * XXX fixme
1424          *
1425          * I would like to get rid of this switch case and do something
1426          * more elegant.
1427          */
1428         switch (opcode) {
1429               case LDS_OP:
1430               case LDSA_OP:
1431                 if (u.insn.x)
1432                         /* oops, really a semaphore op (cmpxchg, etc) */
1433                         goto failure;
1434                 /* no break */
1435               case LDS_IMM_OP:
1436               case LDSA_IMM_OP:
1437               case LDFS_OP:
1438               case LDFSA_OP:
1439               case LDFS_IMM_OP:
1440                 /*
1441                  * The instruction will be retried with deferred exceptions turned on, and
1442                  * we should get Nat bit installed
1443                  *
1444                  * IMPORTANT: When PSR_ED is set, the register & immediate update forms
1445                  * are actually executed even though the operation failed. So we don't
1446                  * need to take care of this.
1447                  */
1448                 DPRINT("forcing PSR_ED\n");
1449                 regs->cr_ipsr |= IA64_PSR_ED;
1450                 goto done;
1451 
1452               case LD_OP:
1453               case LDA_OP:
1454               case LDBIAS_OP:
1455               case LDACQ_OP:
1456               case LDCCLR_OP:
1457               case LDCNC_OP:
1458               case LDCCLRACQ_OP:
1459                 if (u.insn.x)
1460                         /* oops, really a semaphore op (cmpxchg, etc) */
1461                         goto failure;
1462                 /* no break */
1463               case LD_IMM_OP:
1464               case LDA_IMM_OP:
1465               case LDBIAS_IMM_OP:
1466               case LDACQ_IMM_OP:
1467               case LDCCLR_IMM_OP:
1468               case LDCNC_IMM_OP:
1469               case LDCCLRACQ_IMM_OP:
1470                 ret = emulate_load_int(ifa, u.insn, regs);
1471                 break;
1472 
1473               case ST_OP:
1474               case STREL_OP:
1475                 if (u.insn.x)
1476                         /* oops, really a semaphore op (cmpxchg, etc) */
1477                         goto failure;
1478                 /* no break */
1479               case ST_IMM_OP:
1480               case STREL_IMM_OP:
1481                 ret = emulate_store_int(ifa, u.insn, regs);
1482                 break;
1483 
1484               case LDF_OP:
1485               case LDFA_OP:
1486               case LDFCCLR_OP:
1487               case LDFCNC_OP:
1488                 if (u.insn.x)
1489                         ret = emulate_load_floatpair(ifa, u.insn, regs);
1490                 else
1491                         ret = emulate_load_float(ifa, u.insn, regs);
1492                 break;
1493 
1494               case LDF_IMM_OP:
1495               case LDFA_IMM_OP:
1496               case LDFCCLR_IMM_OP:
1497               case LDFCNC_IMM_OP:
1498                 ret = emulate_load_float(ifa, u.insn, regs);
1499                 break;
1500 
1501               case STF_OP:
1502               case STF_IMM_OP:
1503                 ret = emulate_store_float(ifa, u.insn, regs);
1504                 break;
1505 
1506               default:
1507                 goto failure;
1508         }
1509         DPRINT("ret=%d\n", ret);
1510         if (ret)
1511                 goto failure;
1512 
1513         if (ipsr->ri == 2)
1514                 /*
1515                  * given today's architecture this case is not likely to happen because a
1516                  * memory access instruction (M) can never be in the last slot of a
1517                  * bundle. But let's keep it for now.
1518                  */
1519                 regs->cr_iip += 16;
1520         ipsr->ri = (ipsr->ri + 1) & 0x3;
1521 
1522         DPRINT("ipsr->ri=%d iip=%lx\n", ipsr->ri, regs->cr_iip);
1523   done:
1524         set_fs(old_fs);         /* restore original address limit */
1525         return;
1526 
1527   failure:
1528         /* something went wrong... */
1529         if (!user_mode(regs)) {
1530                 if (eh) {
1531                         ia64_handle_exception(regs, eh);
1532                         goto done;
1533                 }
1534                 if (die_if_kernel("error during unaligned kernel access\n", regs, ret))
1535                         return;
1536                 /* NOT_REACHED */
1537         }
1538   force_sigbus:
1539         force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *) ifa,
1540                         0, 0, 0);
1541         goto done;
1542 }
1543 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp