~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/ia64/kernel/unaligned.c

Version: ~ [ linux-5.3-rc5 ] ~ [ linux-5.2.9 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.67 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.139 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.189 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.189 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.72 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Architecture-specific unaligned trap handling.
  3  *
  4  * Copyright (C) 1999-2002, 2004 Hewlett-Packard Co
  5  *      Stephane Eranian <eranian@hpl.hp.com>
  6  *      David Mosberger-Tang <davidm@hpl.hp.com>
  7  *
  8  * 2002/12/09   Fix rotating register handling (off-by-1 error, missing fr-rotation).  Fix
  9  *              get_rse_reg() to not leak kernel bits to user-level (reading an out-of-frame
 10  *              stacked register returns an undefined value; it does NOT trigger a
 11  *              "rsvd register fault").
 12  * 2001/10/11   Fix unaligned access to rotating registers in s/w pipelined loops.
 13  * 2001/08/13   Correct size of extended floats (float_fsz) from 16 to 10 bytes.
 14  * 2001/01/17   Add support emulation of unaligned kernel accesses.
 15  */
 16 #include <linux/jiffies.h>
 17 #include <linux/kernel.h>
 18 #include <linux/sched.h>
 19 #include <linux/tty.h>
 20 #include <linux/ratelimit.h>
 21 
 22 #include <asm/intrinsics.h>
 23 #include <asm/processor.h>
 24 #include <asm/rse.h>
 25 #include <asm/uaccess.h>
 26 #include <asm/unaligned.h>
 27 
 28 extern int die_if_kernel(char *str, struct pt_regs *regs, long err);
 29 
 30 #undef DEBUG_UNALIGNED_TRAP
 31 
 32 #ifdef DEBUG_UNALIGNED_TRAP
 33 # define DPRINT(a...)   do { printk("%s %u: ", __func__, __LINE__); printk (a); } while (0)
 34 # define DDUMP(str,vp,len)      dump(str, vp, len)
 35 
 36 static void
 37 dump (const char *str, void *vp, size_t len)
 38 {
 39         unsigned char *cp = vp;
 40         int i;
 41 
 42         printk("%s", str);
 43         for (i = 0; i < len; ++i)
 44                 printk (" %02x", *cp++);
 45         printk("\n");
 46 }
 47 #else
 48 # define DPRINT(a...)
 49 # define DDUMP(str,vp,len)
 50 #endif
 51 
 52 #define IA64_FIRST_STACKED_GR   32
 53 #define IA64_FIRST_ROTATING_FR  32
 54 #define SIGN_EXT9               0xffffffffffffff00ul
 55 
 56 /*
 57  *  sysctl settable hook which tells the kernel whether to honor the
 58  *  IA64_THREAD_UAC_NOPRINT prctl.  Because this is user settable, we want
 59  *  to allow the super user to enable/disable this for security reasons
 60  *  (i.e. don't allow attacker to fill up logs with unaligned accesses).
 61  */
 62 int no_unaligned_warning;
 63 int unaligned_dump_stack;
 64 
 65 /*
 66  * For M-unit:
 67  *
 68  *  opcode |   m  |   x6    |
 69  * --------|------|---------|
 70  * [40-37] | [36] | [35:30] |
 71  * --------|------|---------|
 72  *     4   |   1  |    6    | = 11 bits
 73  * --------------------------
 74  * However bits [31:30] are not directly useful to distinguish between
 75  * load/store so we can use [35:32] instead, which gives the following
 76  * mask ([40:32]) using 9 bits. The 'e' comes from the fact that we defer
 77  * checking the m-bit until later in the load/store emulation.
 78  */
 79 #define IA64_OPCODE_MASK        0x1ef
 80 #define IA64_OPCODE_SHIFT       32
 81 
 82 /*
 83  * Table C-28 Integer Load/Store
 84  *
 85  * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
 86  *
 87  * ld8.fill, st8.fill  MUST be aligned because the RNATs are based on
 88  * the address (bits [8:3]), so we must failed.
 89  */
 90 #define LD_OP            0x080
 91 #define LDS_OP           0x081
 92 #define LDA_OP           0x082
 93 #define LDSA_OP          0x083
 94 #define LDBIAS_OP        0x084
 95 #define LDACQ_OP         0x085
 96 /* 0x086, 0x087 are not relevant */
 97 #define LDCCLR_OP        0x088
 98 #define LDCNC_OP         0x089
 99 #define LDCCLRACQ_OP     0x08a
100 #define ST_OP            0x08c
101 #define STREL_OP         0x08d
102 /* 0x08e,0x8f are not relevant */
103 
104 /*
105  * Table C-29 Integer Load +Reg
106  *
107  * we use the ld->m (bit [36:36]) field to determine whether or not we have
108  * a load/store of this form.
109  */
110 
111 /*
112  * Table C-30 Integer Load/Store +Imm
113  *
114  * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
115  *
116  * ld8.fill, st8.fill  must be aligned because the Nat register are based on
117  * the address, so we must fail and the program must be fixed.
118  */
119 #define LD_IMM_OP            0x0a0
120 #define LDS_IMM_OP           0x0a1
121 #define LDA_IMM_OP           0x0a2
122 #define LDSA_IMM_OP          0x0a3
123 #define LDBIAS_IMM_OP        0x0a4
124 #define LDACQ_IMM_OP         0x0a5
125 /* 0x0a6, 0xa7 are not relevant */
126 #define LDCCLR_IMM_OP        0x0a8
127 #define LDCNC_IMM_OP         0x0a9
128 #define LDCCLRACQ_IMM_OP     0x0aa
129 #define ST_IMM_OP            0x0ac
130 #define STREL_IMM_OP         0x0ad
131 /* 0x0ae,0xaf are not relevant */
132 
133 /*
134  * Table C-32 Floating-point Load/Store
135  */
136 #define LDF_OP           0x0c0
137 #define LDFS_OP          0x0c1
138 #define LDFA_OP          0x0c2
139 #define LDFSA_OP         0x0c3
140 /* 0x0c6 is irrelevant */
141 #define LDFCCLR_OP       0x0c8
142 #define LDFCNC_OP        0x0c9
143 /* 0x0cb is irrelevant  */
144 #define STF_OP           0x0cc
145 
146 /*
147  * Table C-33 Floating-point Load +Reg
148  *
149  * we use the ld->m (bit [36:36]) field to determine whether or not we have
150  * a load/store of this form.
151  */
152 
153 /*
154  * Table C-34 Floating-point Load/Store +Imm
155  */
156 #define LDF_IMM_OP       0x0e0
157 #define LDFS_IMM_OP      0x0e1
158 #define LDFA_IMM_OP      0x0e2
159 #define LDFSA_IMM_OP     0x0e3
160 /* 0x0e6 is irrelevant */
161 #define LDFCCLR_IMM_OP   0x0e8
162 #define LDFCNC_IMM_OP    0x0e9
163 #define STF_IMM_OP       0x0ec
164 
165 typedef struct {
166         unsigned long    qp:6;  /* [0:5]   */
167         unsigned long    r1:7;  /* [6:12]  */
168         unsigned long   imm:7;  /* [13:19] */
169         unsigned long    r3:7;  /* [20:26] */
170         unsigned long     x:1;  /* [27:27] */
171         unsigned long  hint:2;  /* [28:29] */
172         unsigned long x6_sz:2;  /* [30:31] */
173         unsigned long x6_op:4;  /* [32:35], x6 = x6_sz|x6_op */
174         unsigned long     m:1;  /* [36:36] */
175         unsigned long    op:4;  /* [37:40] */
176         unsigned long   pad:23; /* [41:63] */
177 } load_store_t;
178 
179 
180 typedef enum {
181         UPD_IMMEDIATE,  /* ldXZ r1=[r3],imm(9) */
182         UPD_REG         /* ldXZ r1=[r3],r2     */
183 } update_t;
184 
185 /*
186  * We use tables to keep track of the offsets of registers in the saved state.
187  * This way we save having big switch/case statements.
188  *
189  * We use bit 0 to indicate switch_stack or pt_regs.
190  * The offset is simply shifted by 1 bit.
191  * A 2-byte value should be enough to hold any kind of offset
192  *
193  * In case the calling convention changes (and thus pt_regs/switch_stack)
194  * simply use RSW instead of RPT or vice-versa.
195  */
196 
197 #define RPO(x)  ((size_t) &((struct pt_regs *)0)->x)
198 #define RSO(x)  ((size_t) &((struct switch_stack *)0)->x)
199 
200 #define RPT(x)          (RPO(x) << 1)
201 #define RSW(x)          (1| RSO(x)<<1)
202 
203 #define GR_OFFS(x)      (gr_info[x]>>1)
204 #define GR_IN_SW(x)     (gr_info[x] & 0x1)
205 
206 #define FR_OFFS(x)      (fr_info[x]>>1)
207 #define FR_IN_SW(x)     (fr_info[x] & 0x1)
208 
209 static u16 gr_info[32]={
210         0,                      /* r0 is read-only : WE SHOULD NEVER GET THIS */
211 
212         RPT(r1), RPT(r2), RPT(r3),
213 
214         RSW(r4), RSW(r5), RSW(r6), RSW(r7),
215 
216         RPT(r8), RPT(r9), RPT(r10), RPT(r11),
217         RPT(r12), RPT(r13), RPT(r14), RPT(r15),
218 
219         RPT(r16), RPT(r17), RPT(r18), RPT(r19),
220         RPT(r20), RPT(r21), RPT(r22), RPT(r23),
221         RPT(r24), RPT(r25), RPT(r26), RPT(r27),
222         RPT(r28), RPT(r29), RPT(r30), RPT(r31)
223 };
224 
225 static u16 fr_info[32]={
226         0,                      /* constant : WE SHOULD NEVER GET THIS */
227         0,                      /* constant : WE SHOULD NEVER GET THIS */
228 
229         RSW(f2), RSW(f3), RSW(f4), RSW(f5),
230 
231         RPT(f6), RPT(f7), RPT(f8), RPT(f9),
232         RPT(f10), RPT(f11),
233 
234         RSW(f12), RSW(f13), RSW(f14),
235         RSW(f15), RSW(f16), RSW(f17), RSW(f18), RSW(f19),
236         RSW(f20), RSW(f21), RSW(f22), RSW(f23), RSW(f24),
237         RSW(f25), RSW(f26), RSW(f27), RSW(f28), RSW(f29),
238         RSW(f30), RSW(f31)
239 };
240 
241 /* Invalidate ALAT entry for integer register REGNO.  */
242 static void
243 invala_gr (int regno)
244 {
245 #       define F(reg)   case reg: ia64_invala_gr(reg); break
246 
247         switch (regno) {
248                 F(  0); F(  1); F(  2); F(  3); F(  4); F(  5); F(  6); F(  7);
249                 F(  8); F(  9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15);
250                 F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23);
251                 F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31);
252                 F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39);
253                 F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47);
254                 F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55);
255                 F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63);
256                 F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71);
257                 F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79);
258                 F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87);
259                 F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95);
260                 F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103);
261                 F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111);
262                 F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119);
263                 F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127);
264         }
265 #       undef F
266 }
267 
268 /* Invalidate ALAT entry for floating-point register REGNO.  */
269 static void
270 invala_fr (int regno)
271 {
272 #       define F(reg)   case reg: ia64_invala_fr(reg); break
273 
274         switch (regno) {
275                 F(  0); F(  1); F(  2); F(  3); F(  4); F(  5); F(  6); F(  7);
276                 F(  8); F(  9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15);
277                 F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23);
278                 F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31);
279                 F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39);
280                 F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47);
281                 F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55);
282                 F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63);
283                 F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71);
284                 F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79);
285                 F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87);
286                 F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95);
287                 F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103);
288                 F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111);
289                 F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119);
290                 F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127);
291         }
292 #       undef F
293 }
294 
295 static inline unsigned long
296 rotate_reg (unsigned long sor, unsigned long rrb, unsigned long reg)
297 {
298         reg += rrb;
299         if (reg >= sor)
300                 reg -= sor;
301         return reg;
302 }
303 
304 static void
305 set_rse_reg (struct pt_regs *regs, unsigned long r1, unsigned long val, int nat)
306 {
307         struct switch_stack *sw = (struct switch_stack *) regs - 1;
308         unsigned long *bsp, *bspstore, *addr, *rnat_addr, *ubs_end;
309         unsigned long *kbs = (void *) current + IA64_RBS_OFFSET;
310         unsigned long rnats, nat_mask;
311         unsigned long on_kbs;
312         long sof = (regs->cr_ifs) & 0x7f;
313         long sor = 8 * ((regs->cr_ifs >> 14) & 0xf);
314         long rrb_gr = (regs->cr_ifs >> 18) & 0x7f;
315         long ridx = r1 - 32;
316 
317         if (ridx >= sof) {
318                 /* this should never happen, as the "rsvd register fault" has higher priority */
319                 DPRINT("ignoring write to r%lu; only %lu registers are allocated!\n", r1, sof);
320                 return;
321         }
322 
323         if (ridx < sor)
324                 ridx = rotate_reg(sor, rrb_gr, ridx);
325 
326         DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n",
327                r1, sw->ar_bspstore, regs->ar_bspstore, sof, (regs->cr_ifs >> 7) & 0x7f, ridx);
328 
329         on_kbs = ia64_rse_num_regs(kbs, (unsigned long *) sw->ar_bspstore);
330         addr = ia64_rse_skip_regs((unsigned long *) sw->ar_bspstore, -sof + ridx);
331         if (addr >= kbs) {
332                 /* the register is on the kernel backing store: easy... */
333                 rnat_addr = ia64_rse_rnat_addr(addr);
334                 if ((unsigned long) rnat_addr >= sw->ar_bspstore)
335                         rnat_addr = &sw->ar_rnat;
336                 nat_mask = 1UL << ia64_rse_slot_num(addr);
337 
338                 *addr = val;
339                 if (nat)
340                         *rnat_addr |=  nat_mask;
341                 else
342                         *rnat_addr &= ~nat_mask;
343                 return;
344         }
345 
346         if (!user_stack(current, regs)) {
347                 DPRINT("ignoring kernel write to r%lu; register isn't on the kernel RBS!", r1);
348                 return;
349         }
350 
351         bspstore = (unsigned long *)regs->ar_bspstore;
352         ubs_end = ia64_rse_skip_regs(bspstore, on_kbs);
353         bsp     = ia64_rse_skip_regs(ubs_end, -sof);
354         addr    = ia64_rse_skip_regs(bsp, ridx);
355 
356         DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end, (void *) bsp, (void *) addr);
357 
358         ia64_poke(current, sw, (unsigned long) ubs_end, (unsigned long) addr, val);
359 
360         rnat_addr = ia64_rse_rnat_addr(addr);
361 
362         ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, &rnats);
363         DPRINT("rnat @%p = 0x%lx nat=%d old nat=%ld\n",
364                (void *) rnat_addr, rnats, nat, (rnats >> ia64_rse_slot_num(addr)) & 1);
365 
366         nat_mask = 1UL << ia64_rse_slot_num(addr);
367         if (nat)
368                 rnats |=  nat_mask;
369         else
370                 rnats &= ~nat_mask;
371         ia64_poke(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, rnats);
372 
373         DPRINT("rnat changed to @%p = 0x%lx\n", (void *) rnat_addr, rnats);
374 }
375 
376 
377 static void
378 get_rse_reg (struct pt_regs *regs, unsigned long r1, unsigned long *val, int *nat)
379 {
380         struct switch_stack *sw = (struct switch_stack *) regs - 1;
381         unsigned long *bsp, *addr, *rnat_addr, *ubs_end, *bspstore;
382         unsigned long *kbs = (void *) current + IA64_RBS_OFFSET;
383         unsigned long rnats, nat_mask;
384         unsigned long on_kbs;
385         long sof = (regs->cr_ifs) & 0x7f;
386         long sor = 8 * ((regs->cr_ifs >> 14) & 0xf);
387         long rrb_gr = (regs->cr_ifs >> 18) & 0x7f;
388         long ridx = r1 - 32;
389 
390         if (ridx >= sof) {
391                 /* read of out-of-frame register returns an undefined value; 0 in our case.  */
392                 DPRINT("ignoring read from r%lu; only %lu registers are allocated!\n", r1, sof);
393                 goto fail;
394         }
395 
396         if (ridx < sor)
397                 ridx = rotate_reg(sor, rrb_gr, ridx);
398 
399         DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n",
400                r1, sw->ar_bspstore, regs->ar_bspstore, sof, (regs->cr_ifs >> 7) & 0x7f, ridx);
401 
402         on_kbs = ia64_rse_num_regs(kbs, (unsigned long *) sw->ar_bspstore);
403         addr = ia64_rse_skip_regs((unsigned long *) sw->ar_bspstore, -sof + ridx);
404         if (addr >= kbs) {
405                 /* the register is on the kernel backing store: easy... */
406                 *val = *addr;
407                 if (nat) {
408                         rnat_addr = ia64_rse_rnat_addr(addr);
409                         if ((unsigned long) rnat_addr >= sw->ar_bspstore)
410                                 rnat_addr = &sw->ar_rnat;
411                         nat_mask = 1UL << ia64_rse_slot_num(addr);
412                         *nat = (*rnat_addr & nat_mask) != 0;
413                 }
414                 return;
415         }
416 
417         if (!user_stack(current, regs)) {
418                 DPRINT("ignoring kernel read of r%lu; register isn't on the RBS!", r1);
419                 goto fail;
420         }
421 
422         bspstore = (unsigned long *)regs->ar_bspstore;
423         ubs_end = ia64_rse_skip_regs(bspstore, on_kbs);
424         bsp     = ia64_rse_skip_regs(ubs_end, -sof);
425         addr    = ia64_rse_skip_regs(bsp, ridx);
426 
427         DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end, (void *) bsp, (void *) addr);
428 
429         ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) addr, val);
430 
431         if (nat) {
432                 rnat_addr = ia64_rse_rnat_addr(addr);
433                 nat_mask = 1UL << ia64_rse_slot_num(addr);
434 
435                 DPRINT("rnat @%p = 0x%lx\n", (void *) rnat_addr, rnats);
436 
437                 ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, &rnats);
438                 *nat = (rnats & nat_mask) != 0;
439         }
440         return;
441 
442   fail:
443         *val = 0;
444         if (nat)
445                 *nat = 0;
446         return;
447 }
448 
449 
450 static void
451 setreg (unsigned long regnum, unsigned long val, int nat, struct pt_regs *regs)
452 {
453         struct switch_stack *sw = (struct switch_stack *) regs - 1;
454         unsigned long addr;
455         unsigned long bitmask;
456         unsigned long *unat;
457 
458         /*
459          * First takes care of stacked registers
460          */
461         if (regnum >= IA64_FIRST_STACKED_GR) {
462                 set_rse_reg(regs, regnum, val, nat);
463                 return;
464         }
465 
466         /*
467          * Using r0 as a target raises a General Exception fault which has higher priority
468          * than the Unaligned Reference fault.
469          */
470 
471         /*
472          * Now look at registers in [0-31] range and init correct UNAT
473          */
474         if (GR_IN_SW(regnum)) {
475                 addr = (unsigned long)sw;
476                 unat = &sw->ar_unat;
477         } else {
478                 addr = (unsigned long)regs;
479                 unat = &sw->caller_unat;
480         }
481         DPRINT("tmp_base=%lx switch_stack=%s offset=%d\n",
482                addr, unat==&sw->ar_unat ? "yes":"no", GR_OFFS(regnum));
483         /*
484          * add offset from base of struct
485          * and do it !
486          */
487         addr += GR_OFFS(regnum);
488 
489         *(unsigned long *)addr = val;
490 
491         /*
492          * We need to clear the corresponding UNAT bit to fully emulate the load
493          * UNAT bit_pos = GR[r3]{8:3} form EAS-2.4
494          */
495         bitmask   = 1UL << (addr >> 3 & 0x3f);
496         DPRINT("*0x%lx=0x%lx NaT=%d prev_unat @%p=%lx\n", addr, val, nat, (void *) unat, *unat);
497         if (nat) {
498                 *unat |= bitmask;
499         } else {
500                 *unat &= ~bitmask;
501         }
502         DPRINT("*0x%lx=0x%lx NaT=%d new unat: %p=%lx\n", addr, val, nat, (void *) unat,*unat);
503 }
504 
505 /*
506  * Return the (rotated) index for floating point register REGNUM (REGNUM must be in the
507  * range from 32-127, result is in the range from 0-95.
508  */
509 static inline unsigned long
510 fph_index (struct pt_regs *regs, long regnum)
511 {
512         unsigned long rrb_fr = (regs->cr_ifs >> 25) & 0x7f;
513         return rotate_reg(96, rrb_fr, (regnum - IA64_FIRST_ROTATING_FR));
514 }
515 
516 static void
517 setfpreg (unsigned long regnum, struct ia64_fpreg *fpval, struct pt_regs *regs)
518 {
519         struct switch_stack *sw = (struct switch_stack *)regs - 1;
520         unsigned long addr;
521 
522         /*
523          * From EAS-2.5: FPDisableFault has higher priority than Unaligned
524          * Fault. Thus, when we get here, we know the partition is enabled.
525          * To update f32-f127, there are three choices:
526          *
527          *      (1) save f32-f127 to thread.fph and update the values there
528          *      (2) use a gigantic switch statement to directly access the registers
529          *      (3) generate code on the fly to update the desired register
530          *
531          * For now, we are using approach (1).
532          */
533         if (regnum >= IA64_FIRST_ROTATING_FR) {
534                 ia64_sync_fph(current);
535                 current->thread.fph[fph_index(regs, regnum)] = *fpval;
536         } else {
537                 /*
538                  * pt_regs or switch_stack ?
539                  */
540                 if (FR_IN_SW(regnum)) {
541                         addr = (unsigned long)sw;
542                 } else {
543                         addr = (unsigned long)regs;
544                 }
545 
546                 DPRINT("tmp_base=%lx offset=%d\n", addr, FR_OFFS(regnum));
547 
548                 addr += FR_OFFS(regnum);
549                 *(struct ia64_fpreg *)addr = *fpval;
550 
551                 /*
552                  * mark the low partition as being used now
553                  *
554                  * It is highly unlikely that this bit is not already set, but
555                  * let's do it for safety.
556                  */
557                 regs->cr_ipsr |= IA64_PSR_MFL;
558         }
559 }
560 
561 /*
562  * Those 2 inline functions generate the spilled versions of the constant floating point
563  * registers which can be used with stfX
564  */
565 static inline void
566 float_spill_f0 (struct ia64_fpreg *final)
567 {
568         ia64_stf_spill(final, 0);
569 }
570 
571 static inline void
572 float_spill_f1 (struct ia64_fpreg *final)
573 {
574         ia64_stf_spill(final, 1);
575 }
576 
577 static void
578 getfpreg (unsigned long regnum, struct ia64_fpreg *fpval, struct pt_regs *regs)
579 {
580         struct switch_stack *sw = (struct switch_stack *) regs - 1;
581         unsigned long addr;
582 
583         /*
584          * From EAS-2.5: FPDisableFault has higher priority than
585          * Unaligned Fault. Thus, when we get here, we know the partition is
586          * enabled.
587          *
588          * When regnum > 31, the register is still live and we need to force a save
589          * to current->thread.fph to get access to it.  See discussion in setfpreg()
590          * for reasons and other ways of doing this.
591          */
592         if (regnum >= IA64_FIRST_ROTATING_FR) {
593                 ia64_flush_fph(current);
594                 *fpval = current->thread.fph[fph_index(regs, regnum)];
595         } else {
596                 /*
597                  * f0 = 0.0, f1= 1.0. Those registers are constant and are thus
598                  * not saved, we must generate their spilled form on the fly
599                  */
600                 switch(regnum) {
601                 case 0:
602                         float_spill_f0(fpval);
603                         break;
604                 case 1:
605                         float_spill_f1(fpval);
606                         break;
607                 default:
608                         /*
609                          * pt_regs or switch_stack ?
610                          */
611                         addr =  FR_IN_SW(regnum) ? (unsigned long)sw
612                                                  : (unsigned long)regs;
613 
614                         DPRINT("is_sw=%d tmp_base=%lx offset=0x%x\n",
615                                FR_IN_SW(regnum), addr, FR_OFFS(regnum));
616 
617                         addr  += FR_OFFS(regnum);
618                         *fpval = *(struct ia64_fpreg *)addr;
619                 }
620         }
621 }
622 
623 
624 static void
625 getreg (unsigned long regnum, unsigned long *val, int *nat, struct pt_regs *regs)
626 {
627         struct switch_stack *sw = (struct switch_stack *) regs - 1;
628         unsigned long addr, *unat;
629 
630         if (regnum >= IA64_FIRST_STACKED_GR) {
631                 get_rse_reg(regs, regnum, val, nat);
632                 return;
633         }
634 
635         /*
636          * take care of r0 (read-only always evaluate to 0)
637          */
638         if (regnum == 0) {
639                 *val = 0;
640                 if (nat)
641                         *nat = 0;
642                 return;
643         }
644 
645         /*
646          * Now look at registers in [0-31] range and init correct UNAT
647          */
648         if (GR_IN_SW(regnum)) {
649                 addr = (unsigned long)sw;
650                 unat = &sw->ar_unat;
651         } else {
652                 addr = (unsigned long)regs;
653                 unat = &sw->caller_unat;
654         }
655 
656         DPRINT("addr_base=%lx offset=0x%x\n", addr,  GR_OFFS(regnum));
657 
658         addr += GR_OFFS(regnum);
659 
660         *val  = *(unsigned long *)addr;
661 
662         /*
663          * do it only when requested
664          */
665         if (nat)
666                 *nat  = (*unat >> (addr >> 3 & 0x3f)) & 0x1UL;
667 }
668 
669 static void
670 emulate_load_updates (update_t type, load_store_t ld, struct pt_regs *regs, unsigned long ifa)
671 {
672         /*
673          * IMPORTANT:
674          * Given the way we handle unaligned speculative loads, we should
675          * not get to this point in the code but we keep this sanity check,
676          * just in case.
677          */
678         if (ld.x6_op == 1 || ld.x6_op == 3) {
679                 printk(KERN_ERR "%s: register update on speculative load, error\n", __func__);
680                 if (die_if_kernel("unaligned reference on speculative load with register update\n",
681                                   regs, 30))
682                         return;
683         }
684 
685 
686         /*
687          * at this point, we know that the base register to update is valid i.e.,
688          * it's not r0
689          */
690         if (type == UPD_IMMEDIATE) {
691                 unsigned long imm;
692 
693                 /*
694                  * Load +Imm: ldXZ r1=[r3],imm(9)
695                  *
696                  *
697                  * form imm9: [13:19] contain the first 7 bits
698                  */
699                 imm = ld.x << 7 | ld.imm;
700 
701                 /*
702                  * sign extend (1+8bits) if m set
703                  */
704                 if (ld.m) imm |= SIGN_EXT9;
705 
706                 /*
707                  * ifa == r3 and we know that the NaT bit on r3 was clear so
708                  * we can directly use ifa.
709                  */
710                 ifa += imm;
711 
712                 setreg(ld.r3, ifa, 0, regs);
713 
714                 DPRINT("ld.x=%d ld.m=%d imm=%ld r3=0x%lx\n", ld.x, ld.m, imm, ifa);
715 
716         } else if (ld.m) {
717                 unsigned long r2;
718                 int nat_r2;
719 
720                 /*
721                  * Load +Reg Opcode: ldXZ r1=[r3],r2
722                  *
723                  * Note: that we update r3 even in the case of ldfX.a
724                  * (where the load does not happen)
725                  *
726                  * The way the load algorithm works, we know that r3 does not
727                  * have its NaT bit set (would have gotten NaT consumption
728                  * before getting the unaligned fault). So we can use ifa
729                  * which equals r3 at this point.
730                  *
731                  * IMPORTANT:
732                  * The above statement holds ONLY because we know that we
733                  * never reach this code when trying to do a ldX.s.
734                  * If we ever make it to here on an ldfX.s then
735                  */
736                 getreg(ld.imm, &r2, &nat_r2, regs);
737 
738                 ifa += r2;
739 
740                 /*
741                  * propagate Nat r2 -> r3
742                  */
743                 setreg(ld.r3, ifa, nat_r2, regs);
744 
745                 DPRINT("imm=%d r2=%ld r3=0x%lx nat_r2=%d\n",ld.imm, r2, ifa, nat_r2);
746         }
747 }
748 
749 
750 static int
751 emulate_load_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
752 {
753         unsigned int len = 1 << ld.x6_sz;
754         unsigned long val = 0;
755 
756         /*
757          * r0, as target, doesn't need to be checked because Illegal Instruction
758          * faults have higher priority than unaligned faults.
759          *
760          * r0 cannot be found as the base as it would never generate an
761          * unaligned reference.
762          */
763 
764         /*
765          * ldX.a we will emulate load and also invalidate the ALAT entry.
766          * See comment below for explanation on how we handle ldX.a
767          */
768 
769         if (len != 2 && len != 4 && len != 8) {
770                 DPRINT("unknown size: x6=%d\n", ld.x6_sz);
771                 return -1;
772         }
773         /* this assumes little-endian byte-order: */
774         if (copy_from_user(&val, (void __user *) ifa, len))
775                 return -1;
776         setreg(ld.r1, val, 0, regs);
777 
778         /*
779          * check for updates on any kind of loads
780          */
781         if (ld.op == 0x5 || ld.m)
782                 emulate_load_updates(ld.op == 0x5 ? UPD_IMMEDIATE: UPD_REG, ld, regs, ifa);
783 
784         /*
785          * handling of various loads (based on EAS2.4):
786          *
787          * ldX.acq (ordered load):
788          *      - acquire semantics would have been used, so force fence instead.
789          *
790          * ldX.c.clr (check load and clear):
791          *      - if we get to this handler, it's because the entry was not in the ALAT.
792          *        Therefore the operation reverts to a normal load
793          *
794          * ldX.c.nc (check load no clear):
795          *      - same as previous one
796          *
797          * ldX.c.clr.acq (ordered check load and clear):
798          *      - same as above for c.clr part. The load needs to have acquire semantics. So
799          *        we use the fence semantics which is stronger and thus ensures correctness.
800          *
801          * ldX.a (advanced load):
802          *      - suppose ldX.a r1=[r3]. If we get to the unaligned trap it's because the
803          *        address doesn't match requested size alignment. This means that we would
804          *        possibly need more than one load to get the result.
805          *
806          *        The load part can be handled just like a normal load, however the difficult
807          *        part is to get the right thing into the ALAT. The critical piece of information
808          *        in the base address of the load & size. To do that, a ld.a must be executed,
809          *        clearly any address can be pushed into the table by using ld1.a r1=[r3]. Now
810          *        if we use the same target register, we will be okay for the check.a instruction.
811          *        If we look at the store, basically a stX [r3]=r1 checks the ALAT  for any entry
812          *        which would overlap within [r3,r3+X] (the size of the load was store in the
813          *        ALAT). If such an entry is found the entry is invalidated. But this is not good
814          *        enough, take the following example:
815          *              r3=3
816          *              ld4.a r1=[r3]
817          *
818          *        Could be emulated by doing:
819          *              ld1.a r1=[r3],1
820          *              store to temporary;
821          *              ld1.a r1=[r3],1
822          *              store & shift to temporary;
823          *              ld1.a r1=[r3],1
824          *              store & shift to temporary;
825          *              ld1.a r1=[r3]
826          *              store & shift to temporary;
827          *              r1=temporary
828          *
829          *        So in this case, you would get the right value is r1 but the wrong info in
830          *        the ALAT.  Notice that you could do it in reverse to finish with address 3
831          *        but you would still get the size wrong.  To get the size right, one needs to
832          *        execute exactly the same kind of load. You could do it from a aligned
833          *        temporary location, but you would get the address wrong.
834          *
835          *        So no matter what, it is not possible to emulate an advanced load
836          *        correctly. But is that really critical ?
837          *
838          *        We will always convert ld.a into a normal load with ALAT invalidated.  This
839          *        will enable compiler to do optimization where certain code path after ld.a
840          *        is not required to have ld.c/chk.a, e.g., code path with no intervening stores.
841          *
842          *        If there is a store after the advanced load, one must either do a ld.c.* or
843          *        chk.a.* to reuse the value stored in the ALAT. Both can "fail" (meaning no
844          *        entry found in ALAT), and that's perfectly ok because:
845          *
846          *              - ld.c.*, if the entry is not present a  normal load is executed
847          *              - chk.a.*, if the entry is not present, execution jumps to recovery code
848          *
849          *        In either case, the load can be potentially retried in another form.
850          *
851          *        ALAT must be invalidated for the register (so that chk.a or ld.c don't pick
852          *        up a stale entry later). The register base update MUST also be performed.
853          */
854 
855         /*
856          * when the load has the .acq completer then
857          * use ordering fence.
858          */
859         if (ld.x6_op == 0x5 || ld.x6_op == 0xa)
860                 mb();
861 
862         /*
863          * invalidate ALAT entry in case of advanced load
864          */
865         if (ld.x6_op == 0x2)
866                 invala_gr(ld.r1);
867 
868         return 0;
869 }
870 
871 static int
872 emulate_store_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
873 {
874         unsigned long r2;
875         unsigned int len = 1 << ld.x6_sz;
876 
877         /*
878          * if we get to this handler, Nat bits on both r3 and r2 have already
879          * been checked. so we don't need to do it
880          *
881          * extract the value to be stored
882          */
883         getreg(ld.imm, &r2, NULL, regs);
884 
885         /*
886          * we rely on the macros in unaligned.h for now i.e.,
887          * we let the compiler figure out how to read memory gracefully.
888          *
889          * We need this switch/case because the way the inline function
890          * works. The code is optimized by the compiler and looks like
891          * a single switch/case.
892          */
893         DPRINT("st%d [%lx]=%lx\n", len, ifa, r2);
894 
895         if (len != 2 && len != 4 && len != 8) {
896                 DPRINT("unknown size: x6=%d\n", ld.x6_sz);
897                 return -1;
898         }
899 
900         /* this assumes little-endian byte-order: */
901         if (copy_to_user((void __user *) ifa, &r2, len))
902                 return -1;
903 
904         /*
905          * stX [r3]=r2,imm(9)
906          *
907          * NOTE:
908          * ld.r3 can never be r0, because r0 would not generate an
909          * unaligned access.
910          */
911         if (ld.op == 0x5) {
912                 unsigned long imm;
913 
914                 /*
915                  * form imm9: [12:6] contain first 7bits
916                  */
917                 imm = ld.x << 7 | ld.r1;
918                 /*
919                  * sign extend (8bits) if m set
920                  */
921                 if (ld.m) imm |= SIGN_EXT9;
922                 /*
923                  * ifa == r3 (NaT is necessarily cleared)
924                  */
925                 ifa += imm;
926 
927                 DPRINT("imm=%lx r3=%lx\n", imm, ifa);
928 
929                 setreg(ld.r3, ifa, 0, regs);
930         }
931         /*
932          * we don't have alat_invalidate_multiple() so we need
933          * to do the complete flush :-<<
934          */
935         ia64_invala();
936 
937         /*
938          * stX.rel: use fence instead of release
939          */
940         if (ld.x6_op == 0xd)
941                 mb();
942 
943         return 0;
944 }
945 
946 /*
947  * floating point operations sizes in bytes
948  */
949 static const unsigned char float_fsz[4]={
950         10, /* extended precision (e) */
951         8,  /* integer (8)            */
952         4,  /* single precision (s)   */
953         8   /* double precision (d)   */
954 };
955 
956 static inline void
957 mem2float_extended (struct ia64_fpreg *init, struct ia64_fpreg *final)
958 {
959         ia64_ldfe(6, init);
960         ia64_stop();
961         ia64_stf_spill(final, 6);
962 }
963 
964 static inline void
965 mem2float_integer (struct ia64_fpreg *init, struct ia64_fpreg *final)
966 {
967         ia64_ldf8(6, init);
968         ia64_stop();
969         ia64_stf_spill(final, 6);
970 }
971 
972 static inline void
973 mem2float_single (struct ia64_fpreg *init, struct ia64_fpreg *final)
974 {
975         ia64_ldfs(6, init);
976         ia64_stop();
977         ia64_stf_spill(final, 6);
978 }
979 
980 static inline void
981 mem2float_double (struct ia64_fpreg *init, struct ia64_fpreg *final)
982 {
983         ia64_ldfd(6, init);
984         ia64_stop();
985         ia64_stf_spill(final, 6);
986 }
987 
988 static inline void
989 float2mem_extended (struct ia64_fpreg *init, struct ia64_fpreg *final)
990 {
991         ia64_ldf_fill(6, init);
992         ia64_stop();
993         ia64_stfe(final, 6);
994 }
995 
996 static inline void
997 float2mem_integer (struct ia64_fpreg *init, struct ia64_fpreg *final)
998 {
999         ia64_ldf_fill(6, init);
1000         ia64_stop();
1001         ia64_stf8(final, 6);
1002 }
1003 
1004 static inline void
1005 float2mem_single (struct ia64_fpreg *init, struct ia64_fpreg *final)
1006 {
1007         ia64_ldf_fill(6, init);
1008         ia64_stop();
1009         ia64_stfs(final, 6);
1010 }
1011 
1012 static inline void
1013 float2mem_double (struct ia64_fpreg *init, struct ia64_fpreg *final)
1014 {
1015         ia64_ldf_fill(6, init);
1016         ia64_stop();
1017         ia64_stfd(final, 6);
1018 }
1019 
1020 static int
1021 emulate_load_floatpair (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
1022 {
1023         struct ia64_fpreg fpr_init[2];
1024         struct ia64_fpreg fpr_final[2];
1025         unsigned long len = float_fsz[ld.x6_sz];
1026 
1027         /*
1028          * fr0 & fr1 don't need to be checked because Illegal Instruction faults have
1029          * higher priority than unaligned faults.
1030          *
1031          * r0 cannot be found as the base as it would never generate an unaligned
1032          * reference.
1033          */
1034 
1035         /*
1036          * make sure we get clean buffers
1037          */
1038         memset(&fpr_init, 0, sizeof(fpr_init));
1039         memset(&fpr_final, 0, sizeof(fpr_final));
1040 
1041         /*
1042          * ldfpX.a: we don't try to emulate anything but we must
1043          * invalidate the ALAT entry and execute updates, if any.
1044          */
1045         if (ld.x6_op != 0x2) {
1046                 /*
1047                  * This assumes little-endian byte-order.  Note that there is no "ldfpe"
1048                  * instruction:
1049                  */
1050                 if (copy_from_user(&fpr_init[0], (void __user *) ifa, len)
1051                     || copy_from_user(&fpr_init[1], (void __user *) (ifa + len), len))
1052                         return -1;
1053 
1054                 DPRINT("ld.r1=%d ld.imm=%d x6_sz=%d\n", ld.r1, ld.imm, ld.x6_sz);
1055                 DDUMP("frp_init =", &fpr_init, 2*len);
1056                 /*
1057                  * XXX fixme
1058                  * Could optimize inlines by using ldfpX & 2 spills
1059                  */
1060                 switch( ld.x6_sz ) {
1061                         case 0:
1062                                 mem2float_extended(&fpr_init[0], &fpr_final[0]);
1063                                 mem2float_extended(&fpr_init[1], &fpr_final[1]);
1064                                 break;
1065                         case 1:
1066                                 mem2float_integer(&fpr_init[0], &fpr_final[0]);
1067                                 mem2float_integer(&fpr_init[1], &fpr_final[1]);
1068                                 break;
1069                         case 2:
1070                                 mem2float_single(&fpr_init[0], &fpr_final[0]);
1071                                 mem2float_single(&fpr_init[1], &fpr_final[1]);
1072                                 break;
1073                         case 3:
1074                                 mem2float_double(&fpr_init[0], &fpr_final[0]);
1075                                 mem2float_double(&fpr_init[1], &fpr_final[1]);
1076                                 break;
1077                 }
1078                 DDUMP("fpr_final =", &fpr_final, 2*len);
1079                 /*
1080                  * XXX fixme
1081                  *
1082                  * A possible optimization would be to drop fpr_final and directly
1083                  * use the storage from the saved context i.e., the actual final
1084                  * destination (pt_regs, switch_stack or thread structure).
1085                  */
1086                 setfpreg(ld.r1, &fpr_final[0], regs);
1087                 setfpreg(ld.imm, &fpr_final[1], regs);
1088         }
1089 
1090         /*
1091          * Check for updates: only immediate updates are available for this
1092          * instruction.
1093          */
1094         if (ld.m) {
1095                 /*
1096                  * the immediate is implicit given the ldsz of the operation:
1097                  * single: 8 (2x4) and for  all others it's 16 (2x8)
1098                  */
1099                 ifa += len<<1;
1100 
1101                 /*
1102                  * IMPORTANT:
1103                  * the fact that we force the NaT of r3 to zero is ONLY valid
1104                  * as long as we don't come here with a ldfpX.s.
1105                  * For this reason we keep this sanity check
1106                  */
1107                 if (ld.x6_op == 1 || ld.x6_op == 3)
1108                         printk(KERN_ERR "%s: register update on speculative load pair, error\n",
1109                                __func__);
1110 
1111                 setreg(ld.r3, ifa, 0, regs);
1112         }
1113 
1114         /*
1115          * Invalidate ALAT entries, if any, for both registers.
1116          */
1117         if (ld.x6_op == 0x2) {
1118                 invala_fr(ld.r1);
1119                 invala_fr(ld.imm);
1120         }
1121         return 0;
1122 }
1123 
1124 
1125 static int
1126 emulate_load_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
1127 {
1128         struct ia64_fpreg fpr_init;
1129         struct ia64_fpreg fpr_final;
1130         unsigned long len = float_fsz[ld.x6_sz];
1131 
1132         /*
1133          * fr0 & fr1 don't need to be checked because Illegal Instruction
1134          * faults have higher priority than unaligned faults.
1135          *
1136          * r0 cannot be found as the base as it would never generate an
1137          * unaligned reference.
1138          */
1139 
1140         /*
1141          * make sure we get clean buffers
1142          */
1143         memset(&fpr_init,0, sizeof(fpr_init));
1144         memset(&fpr_final,0, sizeof(fpr_final));
1145 
1146         /*
1147          * ldfX.a we don't try to emulate anything but we must
1148          * invalidate the ALAT entry.
1149          * See comments in ldX for descriptions on how the various loads are handled.
1150          */
1151         if (ld.x6_op != 0x2) {
1152                 if (copy_from_user(&fpr_init, (void __user *) ifa, len))
1153                         return -1;
1154 
1155                 DPRINT("ld.r1=%d x6_sz=%d\n", ld.r1, ld.x6_sz);
1156                 DDUMP("fpr_init =", &fpr_init, len);
1157                 /*
1158                  * we only do something for x6_op={0,8,9}
1159                  */
1160                 switch( ld.x6_sz ) {
1161                         case 0:
1162                                 mem2float_extended(&fpr_init, &fpr_final);
1163                                 break;
1164                         case 1:
1165                                 mem2float_integer(&fpr_init, &fpr_final);
1166                                 break;
1167                         case 2:
1168                                 mem2float_single(&fpr_init, &fpr_final);
1169                                 break;
1170                         case 3:
1171                                 mem2float_double(&fpr_init, &fpr_final);
1172                                 break;
1173                 }
1174                 DDUMP("fpr_final =", &fpr_final, len);
1175                 /*
1176                  * XXX fixme
1177                  *
1178                  * A possible optimization would be to drop fpr_final and directly
1179                  * use the storage from the saved context i.e., the actual final
1180                  * destination (pt_regs, switch_stack or thread structure).
1181                  */
1182                 setfpreg(ld.r1, &fpr_final, regs);
1183         }
1184 
1185         /*
1186          * check for updates on any loads
1187          */
1188         if (ld.op == 0x7 || ld.m)
1189                 emulate_load_updates(ld.op == 0x7 ? UPD_IMMEDIATE: UPD_REG, ld, regs, ifa);
1190 
1191         /*
1192          * invalidate ALAT entry in case of advanced floating point loads
1193          */
1194         if (ld.x6_op == 0x2)
1195                 invala_fr(ld.r1);
1196 
1197         return 0;
1198 }
1199 
1200 
1201 static int
1202 emulate_store_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
1203 {
1204         struct ia64_fpreg fpr_init;
1205         struct ia64_fpreg fpr_final;
1206         unsigned long len = float_fsz[ld.x6_sz];
1207 
1208         /*
1209          * make sure we get clean buffers
1210          */
1211         memset(&fpr_init,0, sizeof(fpr_init));
1212         memset(&fpr_final,0, sizeof(fpr_final));
1213 
1214         /*
1215          * if we get to this handler, Nat bits on both r3 and r2 have already
1216          * been checked. so we don't need to do it
1217          *
1218          * extract the value to be stored
1219          */
1220         getfpreg(ld.imm, &fpr_init, regs);
1221         /*
1222          * during this step, we extract the spilled registers from the saved
1223          * context i.e., we refill. Then we store (no spill) to temporary
1224          * aligned location
1225          */
1226         switch( ld.x6_sz ) {
1227                 case 0:
1228                         float2mem_extended(&fpr_init, &fpr_final);
1229                         break;
1230                 case 1:
1231                         float2mem_integer(&fpr_init, &fpr_final);
1232                         break;
1233                 case 2:
1234                         float2mem_single(&fpr_init, &fpr_final);
1235                         break;
1236                 case 3:
1237                         float2mem_double(&fpr_init, &fpr_final);
1238                         break;
1239         }
1240         DPRINT("ld.r1=%d x6_sz=%d\n", ld.r1, ld.x6_sz);
1241         DDUMP("fpr_init =", &fpr_init, len);
1242         DDUMP("fpr_final =", &fpr_final, len);
1243 
1244         if (copy_to_user((void __user *) ifa, &fpr_final, len))
1245                 return -1;
1246 
1247         /*
1248          * stfX [r3]=r2,imm(9)
1249          *
1250          * NOTE:
1251          * ld.r3 can never be r0, because r0 would not generate an
1252          * unaligned access.
1253          */
1254         if (ld.op == 0x7) {
1255                 unsigned long imm;
1256 
1257                 /*
1258                  * form imm9: [12:6] contain first 7bits
1259                  */
1260                 imm = ld.x << 7 | ld.r1;
1261                 /*
1262                  * sign extend (8bits) if m set
1263                  */
1264                 if (ld.m)
1265                         imm |= SIGN_EXT9;
1266                 /*
1267                  * ifa == r3 (NaT is necessarily cleared)
1268                  */
1269                 ifa += imm;
1270 
1271                 DPRINT("imm=%lx r3=%lx\n", imm, ifa);
1272 
1273                 setreg(ld.r3, ifa, 0, regs);
1274         }
1275         /*
1276          * we don't have alat_invalidate_multiple() so we need
1277          * to do the complete flush :-<<
1278          */
1279         ia64_invala();
1280 
1281         return 0;
1282 }
1283 
1284 /*
1285  * Make sure we log the unaligned access, so that user/sysadmin can notice it and
1286  * eventually fix the program.  However, we don't want to do that for every access so we
1287  * pace it with jiffies.
1288  */
1289 static DEFINE_RATELIMIT_STATE(logging_rate_limit, 5 * HZ, 5);
1290 
1291 void
1292 ia64_handle_unaligned (unsigned long ifa, struct pt_regs *regs)
1293 {
1294         struct ia64_psr *ipsr = ia64_psr(regs);
1295         mm_segment_t old_fs = get_fs();
1296         unsigned long bundle[2];
1297         unsigned long opcode;
1298         struct siginfo si;
1299         const struct exception_table_entry *eh = NULL;
1300         union {
1301                 unsigned long l;
1302                 load_store_t insn;
1303         } u;
1304         int ret = -1;
1305 
1306         if (ia64_psr(regs)->be) {
1307                 /* we don't support big-endian accesses */
1308                 if (die_if_kernel("big-endian unaligned accesses are not supported", regs, 0))
1309                         return;
1310                 goto force_sigbus;
1311         }
1312 
1313         /*
1314          * Treat kernel accesses for which there is an exception handler entry the same as
1315          * user-level unaligned accesses.  Otherwise, a clever program could trick this
1316          * handler into reading an arbitrary kernel addresses...
1317          */
1318         if (!user_mode(regs))
1319                 eh = search_exception_tables(regs->cr_iip + ia64_psr(regs)->ri);
1320         if (user_mode(regs) || eh) {
1321                 if ((current->thread.flags & IA64_THREAD_UAC_SIGBUS) != 0)
1322                         goto force_sigbus;
1323 
1324                 if (!no_unaligned_warning &&
1325                     !(current->thread.flags & IA64_THREAD_UAC_NOPRINT) &&
1326                     __ratelimit(&logging_rate_limit))
1327                 {
1328                         char buf[200];  /* comm[] is at most 16 bytes... */
1329                         size_t len;
1330 
1331                         len = sprintf(buf, "%s(%d): unaligned access to 0x%016lx, "
1332                                       "ip=0x%016lx\n\r", current->comm,
1333                                       task_pid_nr(current),
1334                                       ifa, regs->cr_iip + ipsr->ri);
1335                         /*
1336                          * Don't call tty_write_message() if we're in the kernel; we might
1337                          * be holding locks...
1338                          */
1339                         if (user_mode(regs))
1340                                 tty_write_message(current->signal->tty, buf);
1341                         buf[len-1] = '\0';      /* drop '\r' */
1342                         /* watch for command names containing %s */
1343                         printk(KERN_WARNING "%s", buf);
1344                 } else {
1345                         if (no_unaligned_warning) {
1346                                 printk_once(KERN_WARNING "%s(%d) encountered an "
1347                                        "unaligned exception which required\n"
1348                                        "kernel assistance, which degrades "
1349                                        "the performance of the application.\n"
1350                                        "Unaligned exception warnings have "
1351                                        "been disabled by the system "
1352                                        "administrator\n"
1353                                        "echo 0 > /proc/sys/kernel/ignore-"
1354                                        "unaligned-usertrap to re-enable\n",
1355                                        current->comm, task_pid_nr(current));
1356                         }
1357                 }
1358         } else {
1359                 if (__ratelimit(&logging_rate_limit)) {
1360                         printk(KERN_WARNING "kernel unaligned access to 0x%016lx, ip=0x%016lx\n",
1361                                ifa, regs->cr_iip + ipsr->ri);
1362                         if (unaligned_dump_stack)
1363                                 dump_stack();
1364                 }
1365                 set_fs(KERNEL_DS);
1366         }
1367 
1368         DPRINT("iip=%lx ifa=%lx isr=%lx (ei=%d, sp=%d)\n",
1369                regs->cr_iip, ifa, regs->cr_ipsr, ipsr->ri, ipsr->it);
1370 
1371         if (__copy_from_user(bundle, (void __user *) regs->cr_iip, 16))
1372                 goto failure;
1373 
1374         /*
1375          * extract the instruction from the bundle given the slot number
1376          */
1377         switch (ipsr->ri) {
1378               case 0: u.l = (bundle[0] >>  5); break;
1379               case 1: u.l = (bundle[0] >> 46) | (bundle[1] << 18); break;
1380               case 2: u.l = (bundle[1] >> 23); break;
1381         }
1382         opcode = (u.l >> IA64_OPCODE_SHIFT) & IA64_OPCODE_MASK;
1383 
1384         DPRINT("opcode=%lx ld.qp=%d ld.r1=%d ld.imm=%d ld.r3=%d ld.x=%d ld.hint=%d "
1385                "ld.x6=0x%x ld.m=%d ld.op=%d\n", opcode, u.insn.qp, u.insn.r1, u.insn.imm,
1386                u.insn.r3, u.insn.x, u.insn.hint, u.insn.x6_sz, u.insn.m, u.insn.op);
1387 
1388         /*
1389          * IMPORTANT:
1390          * Notice that the switch statement DOES not cover all possible instructions
1391          * that DO generate unaligned references. This is made on purpose because for some
1392          * instructions it DOES NOT make sense to try and emulate the access. Sometimes it
1393          * is WRONG to try and emulate. Here is a list of instruction we don't emulate i.e.,
1394          * the program will get a signal and die:
1395          *
1396          *      load/store:
1397          *              - ldX.spill
1398          *              - stX.spill
1399          *      Reason: RNATs are based on addresses
1400          *              - ld16
1401          *              - st16
1402          *      Reason: ld16 and st16 are supposed to occur in a single
1403          *              memory op
1404          *
1405          *      synchronization:
1406          *              - cmpxchg
1407          *              - fetchadd
1408          *              - xchg
1409          *      Reason: ATOMIC operations cannot be emulated properly using multiple
1410          *              instructions.
1411          *
1412          *      speculative loads:
1413          *              - ldX.sZ
1414          *      Reason: side effects, code must be ready to deal with failure so simpler
1415          *              to let the load fail.
1416          * ---------------------------------------------------------------------------------
1417          * XXX fixme
1418          *
1419          * I would like to get rid of this switch case and do something
1420          * more elegant.
1421          */
1422         switch (opcode) {
1423               case LDS_OP:
1424               case LDSA_OP:
1425                 if (u.insn.x)
1426                         /* oops, really a semaphore op (cmpxchg, etc) */
1427                         goto failure;
1428                 /* no break */
1429               case LDS_IMM_OP:
1430               case LDSA_IMM_OP:
1431               case LDFS_OP:
1432               case LDFSA_OP:
1433               case LDFS_IMM_OP:
1434                 /*
1435                  * The instruction will be retried with deferred exceptions turned on, and
1436                  * we should get Nat bit installed
1437                  *
1438                  * IMPORTANT: When PSR_ED is set, the register & immediate update forms
1439                  * are actually executed even though the operation failed. So we don't
1440                  * need to take care of this.
1441                  */
1442                 DPRINT("forcing PSR_ED\n");
1443                 regs->cr_ipsr |= IA64_PSR_ED;
1444                 goto done;
1445 
1446               case LD_OP:
1447               case LDA_OP:
1448               case LDBIAS_OP:
1449               case LDACQ_OP:
1450               case LDCCLR_OP:
1451               case LDCNC_OP:
1452               case LDCCLRACQ_OP:
1453                 if (u.insn.x)
1454                         /* oops, really a semaphore op (cmpxchg, etc) */
1455                         goto failure;
1456                 /* no break */
1457               case LD_IMM_OP:
1458               case LDA_IMM_OP:
1459               case LDBIAS_IMM_OP:
1460               case LDACQ_IMM_OP:
1461               case LDCCLR_IMM_OP:
1462               case LDCNC_IMM_OP:
1463               case LDCCLRACQ_IMM_OP:
1464                 ret = emulate_load_int(ifa, u.insn, regs);
1465                 break;
1466 
1467               case ST_OP:
1468               case STREL_OP:
1469                 if (u.insn.x)
1470                         /* oops, really a semaphore op (cmpxchg, etc) */
1471                         goto failure;
1472                 /* no break */
1473               case ST_IMM_OP:
1474               case STREL_IMM_OP:
1475                 ret = emulate_store_int(ifa, u.insn, regs);
1476                 break;
1477 
1478               case LDF_OP:
1479               case LDFA_OP:
1480               case LDFCCLR_OP:
1481               case LDFCNC_OP:
1482                 if (u.insn.x)
1483                         ret = emulate_load_floatpair(ifa, u.insn, regs);
1484                 else
1485                         ret = emulate_load_float(ifa, u.insn, regs);
1486                 break;
1487 
1488               case LDF_IMM_OP:
1489               case LDFA_IMM_OP:
1490               case LDFCCLR_IMM_OP:
1491               case LDFCNC_IMM_OP:
1492                 ret = emulate_load_float(ifa, u.insn, regs);
1493                 break;
1494 
1495               case STF_OP:
1496               case STF_IMM_OP:
1497                 ret = emulate_store_float(ifa, u.insn, regs);
1498                 break;
1499 
1500               default:
1501                 goto failure;
1502         }
1503         DPRINT("ret=%d\n", ret);
1504         if (ret)
1505                 goto failure;
1506 
1507         if (ipsr->ri == 2)
1508                 /*
1509                  * given today's architecture this case is not likely to happen because a
1510                  * memory access instruction (M) can never be in the last slot of a
1511                  * bundle. But let's keep it for now.
1512                  */
1513                 regs->cr_iip += 16;
1514         ipsr->ri = (ipsr->ri + 1) & 0x3;
1515 
1516         DPRINT("ipsr->ri=%d iip=%lx\n", ipsr->ri, regs->cr_iip);
1517   done:
1518         set_fs(old_fs);         /* restore original address limit */
1519         return;
1520 
1521   failure:
1522         /* something went wrong... */
1523         if (!user_mode(regs)) {
1524                 if (eh) {
1525                         ia64_handle_exception(regs, eh);
1526                         goto done;
1527                 }
1528                 if (die_if_kernel("error during unaligned kernel access\n", regs, ret))
1529                         return;
1530                 /* NOT_REACHED */
1531         }
1532   force_sigbus:
1533         si.si_signo = SIGBUS;
1534         si.si_errno = 0;
1535         si.si_code = BUS_ADRALN;
1536         si.si_addr = (void __user *) ifa;
1537         si.si_flags = 0;
1538         si.si_isr = 0;
1539         si.si_imm = 0;
1540         force_sig_info(SIGBUS, &si, current);
1541         goto done;
1542 }
1543 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp