~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm/net/bpf_jit_32.c

Version: ~ [ linux-4.20-rc6 ] ~ [ linux-4.19.8 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.87 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.144 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.166 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.128 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.61 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.31.14 ] ~ [ linux-2.6.30.10 ] ~ [ linux-2.6.29.6 ] ~ [ linux-2.6.28.10 ] ~ [ linux-2.6.27.62 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Just-In-Time compiler for eBPF filters on 32bit ARM
  3  *
  4  * Copyright (c) 2017 Shubham Bansal <illusionist.neo@gmail.com>
  5  * Copyright (c) 2011 Mircea Gherzan <mgherzan@gmail.com>
  6  *
  7  * This program is free software; you can redistribute it and/or modify it
  8  * under the terms of the GNU General Public License as published by the
  9  * Free Software Foundation; version 2 of the License.
 10  */
 11 
 12 #include <linux/bpf.h>
 13 #include <linux/bitops.h>
 14 #include <linux/compiler.h>
 15 #include <linux/errno.h>
 16 #include <linux/filter.h>
 17 #include <linux/netdevice.h>
 18 #include <linux/string.h>
 19 #include <linux/slab.h>
 20 #include <linux/if_vlan.h>
 21 
 22 #include <asm/cacheflush.h>
 23 #include <asm/hwcap.h>
 24 #include <asm/opcodes.h>
 25 #include <asm/system_info.h>
 26 
 27 #include "bpf_jit_32.h"
 28 
 29 /*
 30  * eBPF prog stack layout:
 31  *
 32  *                         high
 33  * original ARM_SP =>     +-----+
 34  *                        |     | callee saved registers
 35  *                        +-----+ <= (BPF_FP + SCRATCH_SIZE)
 36  *                        | ... | eBPF JIT scratch space
 37  * eBPF fp register =>    +-----+
 38  *   (BPF_FP)             | ... | eBPF prog stack
 39  *                        +-----+
 40  *                        |RSVD | JIT scratchpad
 41  * current ARM_SP =>      +-----+ <= (BPF_FP - STACK_SIZE + SCRATCH_SIZE)
 42  *                        |     |
 43  *                        | ... | Function call stack
 44  *                        |     |
 45  *                        +-----+
 46  *                          low
 47  *
 48  * The callee saved registers depends on whether frame pointers are enabled.
 49  * With frame pointers (to be compliant with the ABI):
 50  *
 51  *                              high
 52  * original ARM_SP =>     +--------------+ \
 53  *                        |      pc      | |
 54  * current ARM_FP =>      +--------------+ } callee saved registers
 55  *                        |r4-r9,fp,ip,lr| |
 56  *                        +--------------+ /
 57  *                              low
 58  *
 59  * Without frame pointers:
 60  *
 61  *                              high
 62  * original ARM_SP =>     +--------------+
 63  *                        |  r4-r9,fp,lr | callee saved registers
 64  * current ARM_FP =>      +--------------+
 65  *                              low
 66  *
 67  * When popping registers off the stack at the end of a BPF function, we
 68  * reference them via the current ARM_FP register.
 69  */
 70 #define CALLEE_MASK     (1 << ARM_R4 | 1 << ARM_R5 | 1 << ARM_R6 | \
 71                          1 << ARM_R7 | 1 << ARM_R8 | 1 << ARM_R9 | \
 72                          1 << ARM_FP)
 73 #define CALLEE_PUSH_MASK (CALLEE_MASK | 1 << ARM_LR)
 74 #define CALLEE_POP_MASK  (CALLEE_MASK | 1 << ARM_PC)
 75 
 76 enum {
 77         /* Stack layout - these are offsets from (top of stack - 4) */
 78         BPF_R2_HI,
 79         BPF_R2_LO,
 80         BPF_R3_HI,
 81         BPF_R3_LO,
 82         BPF_R4_HI,
 83         BPF_R4_LO,
 84         BPF_R5_HI,
 85         BPF_R5_LO,
 86         BPF_R7_HI,
 87         BPF_R7_LO,
 88         BPF_R8_HI,
 89         BPF_R8_LO,
 90         BPF_R9_HI,
 91         BPF_R9_LO,
 92         BPF_FP_HI,
 93         BPF_FP_LO,
 94         BPF_TC_HI,
 95         BPF_TC_LO,
 96         BPF_AX_HI,
 97         BPF_AX_LO,
 98         /* Stack space for BPF_REG_2, BPF_REG_3, BPF_REG_4,
 99          * BPF_REG_5, BPF_REG_7, BPF_REG_8, BPF_REG_9,
100          * BPF_REG_FP and Tail call counts.
101          */
102         BPF_JIT_SCRATCH_REGS,
103 };
104 
105 /*
106  * Negative "register" values indicate the register is stored on the stack
107  * and are the offset from the top of the eBPF JIT scratch space.
108  */
109 #define STACK_OFFSET(k) (-4 - (k) * 4)
110 #define SCRATCH_SIZE    (BPF_JIT_SCRATCH_REGS * 4)
111 
112 #ifdef CONFIG_FRAME_POINTER
113 #define EBPF_SCRATCH_TO_ARM_FP(x) ((x) - 4 * hweight16(CALLEE_PUSH_MASK) - 4)
114 #else
115 #define EBPF_SCRATCH_TO_ARM_FP(x) (x)
116 #endif
117 
118 #define TMP_REG_1       (MAX_BPF_JIT_REG + 0)   /* TEMP Register 1 */
119 #define TMP_REG_2       (MAX_BPF_JIT_REG + 1)   /* TEMP Register 2 */
120 #define TCALL_CNT       (MAX_BPF_JIT_REG + 2)   /* Tail Call Count */
121 
122 #define FLAG_IMM_OVERFLOW       (1 << 0)
123 
124 /*
125  * Map eBPF registers to ARM 32bit registers or stack scratch space.
126  *
127  * 1. First argument is passed using the arm 32bit registers and rest of the
128  * arguments are passed on stack scratch space.
129  * 2. First callee-saved argument is mapped to arm 32 bit registers and rest
130  * arguments are mapped to scratch space on stack.
131  * 3. We need two 64 bit temp registers to do complex operations on eBPF
132  * registers.
133  *
134  * As the eBPF registers are all 64 bit registers and arm has only 32 bit
135  * registers, we have to map each eBPF registers with two arm 32 bit regs or
136  * scratch memory space and we have to build eBPF 64 bit register from those.
137  *
138  */
139 static const s8 bpf2a32[][2] = {
140         /* return value from in-kernel function, and exit value from eBPF */
141         [BPF_REG_0] = {ARM_R1, ARM_R0},
142         /* arguments from eBPF program to in-kernel function */
143         [BPF_REG_1] = {ARM_R3, ARM_R2},
144         /* Stored on stack scratch space */
145         [BPF_REG_2] = {STACK_OFFSET(BPF_R2_HI), STACK_OFFSET(BPF_R2_LO)},
146         [BPF_REG_3] = {STACK_OFFSET(BPF_R3_HI), STACK_OFFSET(BPF_R3_LO)},
147         [BPF_REG_4] = {STACK_OFFSET(BPF_R4_HI), STACK_OFFSET(BPF_R4_LO)},
148         [BPF_REG_5] = {STACK_OFFSET(BPF_R5_HI), STACK_OFFSET(BPF_R5_LO)},
149         /* callee saved registers that in-kernel function will preserve */
150         [BPF_REG_6] = {ARM_R5, ARM_R4},
151         /* Stored on stack scratch space */
152         [BPF_REG_7] = {STACK_OFFSET(BPF_R7_HI), STACK_OFFSET(BPF_R7_LO)},
153         [BPF_REG_8] = {STACK_OFFSET(BPF_R8_HI), STACK_OFFSET(BPF_R8_LO)},
154         [BPF_REG_9] = {STACK_OFFSET(BPF_R9_HI), STACK_OFFSET(BPF_R9_LO)},
155         /* Read only Frame Pointer to access Stack */
156         [BPF_REG_FP] = {STACK_OFFSET(BPF_FP_HI), STACK_OFFSET(BPF_FP_LO)},
157         /* Temporary Register for internal BPF JIT, can be used
158          * for constant blindings and others.
159          */
160         [TMP_REG_1] = {ARM_R7, ARM_R6},
161         [TMP_REG_2] = {ARM_R9, ARM_R8},
162         /* Tail call count. Stored on stack scratch space. */
163         [TCALL_CNT] = {STACK_OFFSET(BPF_TC_HI), STACK_OFFSET(BPF_TC_LO)},
164         /* temporary register for blinding constants.
165          * Stored on stack scratch space.
166          */
167         [BPF_REG_AX] = {STACK_OFFSET(BPF_AX_HI), STACK_OFFSET(BPF_AX_LO)},
168 };
169 
170 #define dst_lo  dst[1]
171 #define dst_hi  dst[0]
172 #define src_lo  src[1]
173 #define src_hi  src[0]
174 
175 /*
176  * JIT Context:
177  *
178  * prog                 :       bpf_prog
179  * idx                  :       index of current last JITed instruction.
180  * prologue_bytes       :       bytes used in prologue.
181  * epilogue_offset      :       offset of epilogue starting.
182  * offsets              :       array of eBPF instruction offsets in
183  *                              JITed code.
184  * target               :       final JITed code.
185  * epilogue_bytes       :       no of bytes used in epilogue.
186  * imm_count            :       no of immediate counts used for global
187  *                              variables.
188  * imms                 :       array of global variable addresses.
189  */
190 
191 struct jit_ctx {
192         const struct bpf_prog *prog;
193         unsigned int idx;
194         unsigned int prologue_bytes;
195         unsigned int epilogue_offset;
196         unsigned int cpu_architecture;
197         u32 flags;
198         u32 *offsets;
199         u32 *target;
200         u32 stack_size;
201 #if __LINUX_ARM_ARCH__ < 7
202         u16 epilogue_bytes;
203         u16 imm_count;
204         u32 *imms;
205 #endif
206 };
207 
208 /*
209  * Wrappers which handle both OABI and EABI and assures Thumb2 interworking
210  * (where the assembly routines like __aeabi_uidiv could cause problems).
211  */
212 static u32 jit_udiv32(u32 dividend, u32 divisor)
213 {
214         return dividend / divisor;
215 }
216 
217 static u32 jit_mod32(u32 dividend, u32 divisor)
218 {
219         return dividend % divisor;
220 }
221 
222 static inline void _emit(int cond, u32 inst, struct jit_ctx *ctx)
223 {
224         inst |= (cond << 28);
225         inst = __opcode_to_mem_arm(inst);
226 
227         if (ctx->target != NULL)
228                 ctx->target[ctx->idx] = inst;
229 
230         ctx->idx++;
231 }
232 
233 /*
234  * Emit an instruction that will be executed unconditionally.
235  */
236 static inline void emit(u32 inst, struct jit_ctx *ctx)
237 {
238         _emit(ARM_COND_AL, inst, ctx);
239 }
240 
241 /*
242  * This is rather horrid, but necessary to convert an integer constant
243  * to an immediate operand for the opcodes, and be able to detect at
244  * build time whether the constant can't be converted (iow, usable in
245  * BUILD_BUG_ON()).
246  */
247 #define imm12val(v, s) (rol32(v, (s)) | (s) << 7)
248 #define const_imm8m(x)                                  \
249         ({ int r;                                       \
250            u32 v = (x);                                 \
251            if (!(v & ~0x000000ff))                      \
252                 r = imm12val(v, 0);                     \
253            else if (!(v & ~0xc000003f))                 \
254                 r = imm12val(v, 2);                     \
255            else if (!(v & ~0xf000000f))                 \
256                 r = imm12val(v, 4);                     \
257            else if (!(v & ~0xfc000003))                 \
258                 r = imm12val(v, 6);                     \
259            else if (!(v & ~0xff000000))                 \
260                 r = imm12val(v, 8);                     \
261            else if (!(v & ~0x3fc00000))                 \
262                 r = imm12val(v, 10);                    \
263            else if (!(v & ~0x0ff00000))                 \
264                 r = imm12val(v, 12);                    \
265            else if (!(v & ~0x03fc0000))                 \
266                 r = imm12val(v, 14);                    \
267            else if (!(v & ~0x00ff0000))                 \
268                 r = imm12val(v, 16);                    \
269            else if (!(v & ~0x003fc000))                 \
270                 r = imm12val(v, 18);                    \
271            else if (!(v & ~0x000ff000))                 \
272                 r = imm12val(v, 20);                    \
273            else if (!(v & ~0x0003fc00))                 \
274                 r = imm12val(v, 22);                    \
275            else if (!(v & ~0x0000ff00))                 \
276                 r = imm12val(v, 24);                    \
277            else if (!(v & ~0x00003fc0))                 \
278                 r = imm12val(v, 26);                    \
279            else if (!(v & ~0x00000ff0))                 \
280                 r = imm12val(v, 28);                    \
281            else if (!(v & ~0x000003fc))                 \
282                 r = imm12val(v, 30);                    \
283            else                                         \
284                 r = -1;                                 \
285            r; })
286 
287 /*
288  * Checks if immediate value can be converted to imm12(12 bits) value.
289  */
290 static int imm8m(u32 x)
291 {
292         u32 rot;
293 
294         for (rot = 0; rot < 16; rot++)
295                 if ((x & ~ror32(0xff, 2 * rot)) == 0)
296                         return rol32(x, 2 * rot) | (rot << 8);
297         return -1;
298 }
299 
300 #define imm8m(x) (__builtin_constant_p(x) ? const_imm8m(x) : imm8m(x))
301 
302 static u32 arm_bpf_ldst_imm12(u32 op, u8 rt, u8 rn, s16 imm12)
303 {
304         op |= rt << 12 | rn << 16;
305         if (imm12 >= 0)
306                 op |= ARM_INST_LDST__U;
307         else
308                 imm12 = -imm12;
309         return op | (imm12 & ARM_INST_LDST__IMM12);
310 }
311 
312 static u32 arm_bpf_ldst_imm8(u32 op, u8 rt, u8 rn, s16 imm8)
313 {
314         op |= rt << 12 | rn << 16;
315         if (imm8 >= 0)
316                 op |= ARM_INST_LDST__U;
317         else
318                 imm8 = -imm8;
319         return op | (imm8 & 0xf0) << 4 | (imm8 & 0x0f);
320 }
321 
322 #define ARM_LDR_I(rt, rn, off)  arm_bpf_ldst_imm12(ARM_INST_LDR_I, rt, rn, off)
323 #define ARM_LDRB_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_LDRB_I, rt, rn, off)
324 #define ARM_LDRD_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_LDRD_I, rt, rn, off)
325 #define ARM_LDRH_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_LDRH_I, rt, rn, off)
326 
327 #define ARM_STR_I(rt, rn, off)  arm_bpf_ldst_imm12(ARM_INST_STR_I, rt, rn, off)
328 #define ARM_STRB_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_STRB_I, rt, rn, off)
329 #define ARM_STRD_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_STRD_I, rt, rn, off)
330 #define ARM_STRH_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_STRH_I, rt, rn, off)
331 
332 /*
333  * Initializes the JIT space with undefined instructions.
334  */
335 static void jit_fill_hole(void *area, unsigned int size)
336 {
337         u32 *ptr;
338         /* We are guaranteed to have aligned memory. */
339         for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
340                 *ptr++ = __opcode_to_mem_arm(ARM_INST_UDF);
341 }
342 
343 #if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5)
344 /* EABI requires the stack to be aligned to 64-bit boundaries */
345 #define STACK_ALIGNMENT 8
346 #else
347 /* Stack must be aligned to 32-bit boundaries */
348 #define STACK_ALIGNMENT 4
349 #endif
350 
351 /* total stack size used in JITed code */
352 #define _STACK_SIZE     (ctx->prog->aux->stack_depth + SCRATCH_SIZE)
353 #define STACK_SIZE      ALIGN(_STACK_SIZE, STACK_ALIGNMENT)
354 
355 #if __LINUX_ARM_ARCH__ < 7
356 
357 static u16 imm_offset(u32 k, struct jit_ctx *ctx)
358 {
359         unsigned int i = 0, offset;
360         u16 imm;
361 
362         /* on the "fake" run we just count them (duplicates included) */
363         if (ctx->target == NULL) {
364                 ctx->imm_count++;
365                 return 0;
366         }
367 
368         while ((i < ctx->imm_count) && ctx->imms[i]) {
369                 if (ctx->imms[i] == k)
370                         break;
371                 i++;
372         }
373 
374         if (ctx->imms[i] == 0)
375                 ctx->imms[i] = k;
376 
377         /* constants go just after the epilogue */
378         offset =  ctx->offsets[ctx->prog->len - 1] * 4;
379         offset += ctx->prologue_bytes;
380         offset += ctx->epilogue_bytes;
381         offset += i * 4;
382 
383         ctx->target[offset / 4] = k;
384 
385         /* PC in ARM mode == address of the instruction + 8 */
386         imm = offset - (8 + ctx->idx * 4);
387 
388         if (imm & ~0xfff) {
389                 /*
390                  * literal pool is too far, signal it into flags. we
391                  * can only detect it on the second pass unfortunately.
392                  */
393                 ctx->flags |= FLAG_IMM_OVERFLOW;
394                 return 0;
395         }
396 
397         return imm;
398 }
399 
400 #endif /* __LINUX_ARM_ARCH__ */
401 
402 static inline int bpf2a32_offset(int bpf_to, int bpf_from,
403                                  const struct jit_ctx *ctx) {
404         int to, from;
405 
406         if (ctx->target == NULL)
407                 return 0;
408         to = ctx->offsets[bpf_to];
409         from = ctx->offsets[bpf_from];
410 
411         return to - from - 1;
412 }
413 
414 /*
415  * Move an immediate that's not an imm8m to a core register.
416  */
417 static inline void emit_mov_i_no8m(const u8 rd, u32 val, struct jit_ctx *ctx)
418 {
419 #if __LINUX_ARM_ARCH__ < 7
420         emit(ARM_LDR_I(rd, ARM_PC, imm_offset(val, ctx)), ctx);
421 #else
422         emit(ARM_MOVW(rd, val & 0xffff), ctx);
423         if (val > 0xffff)
424                 emit(ARM_MOVT(rd, val >> 16), ctx);
425 #endif
426 }
427 
428 static inline void emit_mov_i(const u8 rd, u32 val, struct jit_ctx *ctx)
429 {
430         int imm12 = imm8m(val);
431 
432         if (imm12 >= 0)
433                 emit(ARM_MOV_I(rd, imm12), ctx);
434         else
435                 emit_mov_i_no8m(rd, val, ctx);
436 }
437 
438 static void emit_bx_r(u8 tgt_reg, struct jit_ctx *ctx)
439 {
440         if (elf_hwcap & HWCAP_THUMB)
441                 emit(ARM_BX(tgt_reg), ctx);
442         else
443                 emit(ARM_MOV_R(ARM_PC, tgt_reg), ctx);
444 }
445 
446 static inline void emit_blx_r(u8 tgt_reg, struct jit_ctx *ctx)
447 {
448 #if __LINUX_ARM_ARCH__ < 5
449         emit(ARM_MOV_R(ARM_LR, ARM_PC), ctx);
450         emit_bx_r(tgt_reg, ctx);
451 #else
452         emit(ARM_BLX_R(tgt_reg), ctx);
453 #endif
454 }
455 
456 static inline int epilogue_offset(const struct jit_ctx *ctx)
457 {
458         int to, from;
459         /* No need for 1st dummy run */
460         if (ctx->target == NULL)
461                 return 0;
462         to = ctx->epilogue_offset;
463         from = ctx->idx;
464 
465         return to - from - 2;
466 }
467 
468 static inline void emit_udivmod(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx, u8 op)
469 {
470         const s8 *tmp = bpf2a32[TMP_REG_1];
471 
472 #if __LINUX_ARM_ARCH__ == 7
473         if (elf_hwcap & HWCAP_IDIVA) {
474                 if (op == BPF_DIV)
475                         emit(ARM_UDIV(rd, rm, rn), ctx);
476                 else {
477                         emit(ARM_UDIV(ARM_IP, rm, rn), ctx);
478                         emit(ARM_MLS(rd, rn, ARM_IP, rm), ctx);
479                 }
480                 return;
481         }
482 #endif
483 
484         /*
485          * For BPF_ALU | BPF_DIV | BPF_K instructions
486          * As ARM_R1 and ARM_R0 contains 1st argument of bpf
487          * function, we need to save it on caller side to save
488          * it from getting destroyed within callee.
489          * After the return from the callee, we restore ARM_R0
490          * ARM_R1.
491          */
492         if (rn != ARM_R1) {
493                 emit(ARM_MOV_R(tmp[0], ARM_R1), ctx);
494                 emit(ARM_MOV_R(ARM_R1, rn), ctx);
495         }
496         if (rm != ARM_R0) {
497                 emit(ARM_MOV_R(tmp[1], ARM_R0), ctx);
498                 emit(ARM_MOV_R(ARM_R0, rm), ctx);
499         }
500 
501         /* Call appropriate function */
502         emit_mov_i(ARM_IP, op == BPF_DIV ?
503                    (u32)jit_udiv32 : (u32)jit_mod32, ctx);
504         emit_blx_r(ARM_IP, ctx);
505 
506         /* Save return value */
507         if (rd != ARM_R0)
508                 emit(ARM_MOV_R(rd, ARM_R0), ctx);
509 
510         /* Restore ARM_R0 and ARM_R1 */
511         if (rn != ARM_R1)
512                 emit(ARM_MOV_R(ARM_R1, tmp[0]), ctx);
513         if (rm != ARM_R0)
514                 emit(ARM_MOV_R(ARM_R0, tmp[1]), ctx);
515 }
516 
517 /* Is the translated BPF register on stack? */
518 static bool is_stacked(s8 reg)
519 {
520         return reg < 0;
521 }
522 
523 /* If a BPF register is on the stack (stk is true), load it to the
524  * supplied temporary register and return the temporary register
525  * for subsequent operations, otherwise just use the CPU register.
526  */
527 static s8 arm_bpf_get_reg32(s8 reg, s8 tmp, struct jit_ctx *ctx)
528 {
529         if (is_stacked(reg)) {
530                 emit(ARM_LDR_I(tmp, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx);
531                 reg = tmp;
532         }
533         return reg;
534 }
535 
536 static const s8 *arm_bpf_get_reg64(const s8 *reg, const s8 *tmp,
537                                    struct jit_ctx *ctx)
538 {
539         if (is_stacked(reg[1])) {
540                 if (__LINUX_ARM_ARCH__ >= 6 ||
541                     ctx->cpu_architecture >= CPU_ARCH_ARMv5TE) {
542                         emit(ARM_LDRD_I(tmp[1], ARM_FP,
543                                         EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx);
544                 } else {
545                         emit(ARM_LDR_I(tmp[1], ARM_FP,
546                                        EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx);
547                         emit(ARM_LDR_I(tmp[0], ARM_FP,
548                                        EBPF_SCRATCH_TO_ARM_FP(reg[0])), ctx);
549                 }
550                 reg = tmp;
551         }
552         return reg;
553 }
554 
555 /* If a BPF register is on the stack (stk is true), save the register
556  * back to the stack.  If the source register is not the same, then
557  * move it into the correct register.
558  */
559 static void arm_bpf_put_reg32(s8 reg, s8 src, struct jit_ctx *ctx)
560 {
561         if (is_stacked(reg))
562                 emit(ARM_STR_I(src, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx);
563         else if (reg != src)
564                 emit(ARM_MOV_R(reg, src), ctx);
565 }
566 
567 static void arm_bpf_put_reg64(const s8 *reg, const s8 *src,
568                               struct jit_ctx *ctx)
569 {
570         if (is_stacked(reg[1])) {
571                 if (__LINUX_ARM_ARCH__ >= 6 ||
572                     ctx->cpu_architecture >= CPU_ARCH_ARMv5TE) {
573                         emit(ARM_STRD_I(src[1], ARM_FP,
574                                        EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx);
575                 } else {
576                         emit(ARM_STR_I(src[1], ARM_FP,
577                                        EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx);
578                         emit(ARM_STR_I(src[0], ARM_FP,
579                                        EBPF_SCRATCH_TO_ARM_FP(reg[0])), ctx);
580                 }
581         } else {
582                 if (reg[1] != src[1])
583                         emit(ARM_MOV_R(reg[1], src[1]), ctx);
584                 if (reg[0] != src[0])
585                         emit(ARM_MOV_R(reg[0], src[0]), ctx);
586         }
587 }
588 
589 static inline void emit_a32_mov_i(const s8 dst, const u32 val,
590                                   struct jit_ctx *ctx)
591 {
592         const s8 *tmp = bpf2a32[TMP_REG_1];
593 
594         if (is_stacked(dst)) {
595                 emit_mov_i(tmp[1], val, ctx);
596                 arm_bpf_put_reg32(dst, tmp[1], ctx);
597         } else {
598                 emit_mov_i(dst, val, ctx);
599         }
600 }
601 
602 static void emit_a32_mov_i64(const s8 dst[], u64 val, struct jit_ctx *ctx)
603 {
604         const s8 *tmp = bpf2a32[TMP_REG_1];
605         const s8 *rd = is_stacked(dst_lo) ? tmp : dst;
606 
607         emit_mov_i(rd[1], (u32)val, ctx);
608         emit_mov_i(rd[0], val >> 32, ctx);
609 
610         arm_bpf_put_reg64(dst, rd, ctx);
611 }
612 
613 /* Sign extended move */
614 static inline void emit_a32_mov_se_i64(const bool is64, const s8 dst[],
615                                        const u32 val, struct jit_ctx *ctx) {
616         u64 val64 = val;
617 
618         if (is64 && (val & (1<<31)))
619                 val64 |= 0xffffffff00000000ULL;
620         emit_a32_mov_i64(dst, val64, ctx);
621 }
622 
623 static inline void emit_a32_add_r(const u8 dst, const u8 src,
624                               const bool is64, const bool hi,
625                               struct jit_ctx *ctx) {
626         /* 64 bit :
627          *      adds dst_lo, dst_lo, src_lo
628          *      adc dst_hi, dst_hi, src_hi
629          * 32 bit :
630          *      add dst_lo, dst_lo, src_lo
631          */
632         if (!hi && is64)
633                 emit(ARM_ADDS_R(dst, dst, src), ctx);
634         else if (hi && is64)
635                 emit(ARM_ADC_R(dst, dst, src), ctx);
636         else
637                 emit(ARM_ADD_R(dst, dst, src), ctx);
638 }
639 
640 static inline void emit_a32_sub_r(const u8 dst, const u8 src,
641                                   const bool is64, const bool hi,
642                                   struct jit_ctx *ctx) {
643         /* 64 bit :
644          *      subs dst_lo, dst_lo, src_lo
645          *      sbc dst_hi, dst_hi, src_hi
646          * 32 bit :
647          *      sub dst_lo, dst_lo, src_lo
648          */
649         if (!hi && is64)
650                 emit(ARM_SUBS_R(dst, dst, src), ctx);
651         else if (hi && is64)
652                 emit(ARM_SBC_R(dst, dst, src), ctx);
653         else
654                 emit(ARM_SUB_R(dst, dst, src), ctx);
655 }
656 
657 static inline void emit_alu_r(const u8 dst, const u8 src, const bool is64,
658                               const bool hi, const u8 op, struct jit_ctx *ctx){
659         switch (BPF_OP(op)) {
660         /* dst = dst + src */
661         case BPF_ADD:
662                 emit_a32_add_r(dst, src, is64, hi, ctx);
663                 break;
664         /* dst = dst - src */
665         case BPF_SUB:
666                 emit_a32_sub_r(dst, src, is64, hi, ctx);
667                 break;
668         /* dst = dst | src */
669         case BPF_OR:
670                 emit(ARM_ORR_R(dst, dst, src), ctx);
671                 break;
672         /* dst = dst & src */
673         case BPF_AND:
674                 emit(ARM_AND_R(dst, dst, src), ctx);
675                 break;
676         /* dst = dst ^ src */
677         case BPF_XOR:
678                 emit(ARM_EOR_R(dst, dst, src), ctx);
679                 break;
680         /* dst = dst * src */
681         case BPF_MUL:
682                 emit(ARM_MUL(dst, dst, src), ctx);
683                 break;
684         /* dst = dst << src */
685         case BPF_LSH:
686                 emit(ARM_LSL_R(dst, dst, src), ctx);
687                 break;
688         /* dst = dst >> src */
689         case BPF_RSH:
690                 emit(ARM_LSR_R(dst, dst, src), ctx);
691                 break;
692         /* dst = dst >> src (signed)*/
693         case BPF_ARSH:
694                 emit(ARM_MOV_SR(dst, dst, SRTYPE_ASR, src), ctx);
695                 break;
696         }
697 }
698 
699 /* ALU operation (32 bit)
700  * dst = dst (op) src
701  */
702 static inline void emit_a32_alu_r(const s8 dst, const s8 src,
703                                   struct jit_ctx *ctx, const bool is64,
704                                   const bool hi, const u8 op) {
705         const s8 *tmp = bpf2a32[TMP_REG_1];
706         s8 rn, rd;
707 
708         rn = arm_bpf_get_reg32(src, tmp[1], ctx);
709         rd = arm_bpf_get_reg32(dst, tmp[0], ctx);
710         /* ALU operation */
711         emit_alu_r(rd, rn, is64, hi, op, ctx);
712         arm_bpf_put_reg32(dst, rd, ctx);
713 }
714 
715 /* ALU operation (64 bit) */
716 static inline void emit_a32_alu_r64(const bool is64, const s8 dst[],
717                                   const s8 src[], struct jit_ctx *ctx,
718                                   const u8 op) {
719         const s8 *tmp = bpf2a32[TMP_REG_1];
720         const s8 *tmp2 = bpf2a32[TMP_REG_2];
721         const s8 *rd;
722 
723         rd = arm_bpf_get_reg64(dst, tmp, ctx);
724         if (is64) {
725                 const s8 *rs;
726 
727                 rs = arm_bpf_get_reg64(src, tmp2, ctx);
728 
729                 /* ALU operation */
730                 emit_alu_r(rd[1], rs[1], true, false, op, ctx);
731                 emit_alu_r(rd[0], rs[0], true, true, op, ctx);
732         } else {
733                 s8 rs;
734 
735                 rs = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
736 
737                 /* ALU operation */
738                 emit_alu_r(rd[1], rs, true, false, op, ctx);
739                 emit_a32_mov_i(rd[0], 0, ctx);
740         }
741 
742         arm_bpf_put_reg64(dst, rd, ctx);
743 }
744 
745 /* dst = src (4 bytes)*/
746 static inline void emit_a32_mov_r(const s8 dst, const s8 src,
747                                   struct jit_ctx *ctx) {
748         const s8 *tmp = bpf2a32[TMP_REG_1];
749         s8 rt;
750 
751         rt = arm_bpf_get_reg32(src, tmp[0], ctx);
752         arm_bpf_put_reg32(dst, rt, ctx);
753 }
754 
755 /* dst = src */
756 static inline void emit_a32_mov_r64(const bool is64, const s8 dst[],
757                                   const s8 src[],
758                                   struct jit_ctx *ctx) {
759         if (!is64) {
760                 emit_a32_mov_r(dst_lo, src_lo, ctx);
761                 /* Zero out high 4 bytes */
762                 emit_a32_mov_i(dst_hi, 0, ctx);
763         } else if (__LINUX_ARM_ARCH__ < 6 &&
764                    ctx->cpu_architecture < CPU_ARCH_ARMv5TE) {
765                 /* complete 8 byte move */
766                 emit_a32_mov_r(dst_lo, src_lo, ctx);
767                 emit_a32_mov_r(dst_hi, src_hi, ctx);
768         } else if (is_stacked(src_lo) && is_stacked(dst_lo)) {
769                 const u8 *tmp = bpf2a32[TMP_REG_1];
770 
771                 emit(ARM_LDRD_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(src_lo)), ctx);
772                 emit(ARM_STRD_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(dst_lo)), ctx);
773         } else if (is_stacked(src_lo)) {
774                 emit(ARM_LDRD_I(dst[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(src_lo)), ctx);
775         } else if (is_stacked(dst_lo)) {
776                 emit(ARM_STRD_I(src[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(dst_lo)), ctx);
777         } else {
778                 emit(ARM_MOV_R(dst[0], src[0]), ctx);
779                 emit(ARM_MOV_R(dst[1], src[1]), ctx);
780         }
781 }
782 
783 /* Shift operations */
784 static inline void emit_a32_alu_i(const s8 dst, const u32 val,
785                                 struct jit_ctx *ctx, const u8 op) {
786         const s8 *tmp = bpf2a32[TMP_REG_1];
787         s8 rd;
788 
789         rd = arm_bpf_get_reg32(dst, tmp[0], ctx);
790 
791         /* Do shift operation */
792         switch (op) {
793         case BPF_LSH:
794                 emit(ARM_LSL_I(rd, rd, val), ctx);
795                 break;
796         case BPF_RSH:
797                 emit(ARM_LSR_I(rd, rd, val), ctx);
798                 break;
799         case BPF_NEG:
800                 emit(ARM_RSB_I(rd, rd, val), ctx);
801                 break;
802         }
803 
804         arm_bpf_put_reg32(dst, rd, ctx);
805 }
806 
807 /* dst = ~dst (64 bit) */
808 static inline void emit_a32_neg64(const s8 dst[],
809                                 struct jit_ctx *ctx){
810         const s8 *tmp = bpf2a32[TMP_REG_1];
811         const s8 *rd;
812 
813         /* Setup Operand */
814         rd = arm_bpf_get_reg64(dst, tmp, ctx);
815 
816         /* Do Negate Operation */
817         emit(ARM_RSBS_I(rd[1], rd[1], 0), ctx);
818         emit(ARM_RSC_I(rd[0], rd[0], 0), ctx);
819 
820         arm_bpf_put_reg64(dst, rd, ctx);
821 }
822 
823 /* dst = dst << src */
824 static inline void emit_a32_lsh_r64(const s8 dst[], const s8 src[],
825                                     struct jit_ctx *ctx) {
826         const s8 *tmp = bpf2a32[TMP_REG_1];
827         const s8 *tmp2 = bpf2a32[TMP_REG_2];
828         const s8 *rd;
829         s8 rt;
830 
831         /* Setup Operands */
832         rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
833         rd = arm_bpf_get_reg64(dst, tmp, ctx);
834 
835         /* Do LSH operation */
836         emit(ARM_SUB_I(ARM_IP, rt, 32), ctx);
837         emit(ARM_RSB_I(tmp2[0], rt, 32), ctx);
838         emit(ARM_MOV_SR(ARM_LR, rd[0], SRTYPE_ASL, rt), ctx);
839         emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[1], SRTYPE_ASL, ARM_IP), ctx);
840         emit(ARM_ORR_SR(ARM_IP, ARM_LR, rd[1], SRTYPE_LSR, tmp2[0]), ctx);
841         emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_ASL, rt), ctx);
842 
843         arm_bpf_put_reg32(dst_lo, ARM_LR, ctx);
844         arm_bpf_put_reg32(dst_hi, ARM_IP, ctx);
845 }
846 
847 /* dst = dst >> src (signed)*/
848 static inline void emit_a32_arsh_r64(const s8 dst[], const s8 src[],
849                                      struct jit_ctx *ctx) {
850         const s8 *tmp = bpf2a32[TMP_REG_1];
851         const s8 *tmp2 = bpf2a32[TMP_REG_2];
852         const s8 *rd;
853         s8 rt;
854 
855         /* Setup Operands */
856         rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
857         rd = arm_bpf_get_reg64(dst, tmp, ctx);
858 
859         /* Do the ARSH operation */
860         emit(ARM_RSB_I(ARM_IP, rt, 32), ctx);
861         emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx);
862         emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx);
863         emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx);
864         _emit(ARM_COND_MI, ARM_B(0), ctx);
865         emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASR, tmp2[0]), ctx);
866         emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_ASR, rt), ctx);
867 
868         arm_bpf_put_reg32(dst_lo, ARM_LR, ctx);
869         arm_bpf_put_reg32(dst_hi, ARM_IP, ctx);
870 }
871 
872 /* dst = dst >> src */
873 static inline void emit_a32_rsh_r64(const s8 dst[], const s8 src[],
874                                     struct jit_ctx *ctx) {
875         const s8 *tmp = bpf2a32[TMP_REG_1];
876         const s8 *tmp2 = bpf2a32[TMP_REG_2];
877         const s8 *rd;
878         s8 rt;
879 
880         /* Setup Operands */
881         rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
882         rd = arm_bpf_get_reg64(dst, tmp, ctx);
883 
884         /* Do RSH operation */
885         emit(ARM_RSB_I(ARM_IP, rt, 32), ctx);
886         emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx);
887         emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx);
888         emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx);
889         emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_LSR, tmp2[0]), ctx);
890         emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_LSR, rt), ctx);
891 
892         arm_bpf_put_reg32(dst_lo, ARM_LR, ctx);
893         arm_bpf_put_reg32(dst_hi, ARM_IP, ctx);
894 }
895 
896 /* dst = dst << val */
897 static inline void emit_a32_lsh_i64(const s8 dst[],
898                                     const u32 val, struct jit_ctx *ctx){
899         const s8 *tmp = bpf2a32[TMP_REG_1];
900         const s8 *tmp2 = bpf2a32[TMP_REG_2];
901         const s8 *rd;
902 
903         /* Setup operands */
904         rd = arm_bpf_get_reg64(dst, tmp, ctx);
905 
906         /* Do LSH operation */
907         if (val < 32) {
908                 emit(ARM_MOV_SI(tmp2[0], rd[0], SRTYPE_ASL, val), ctx);
909                 emit(ARM_ORR_SI(rd[0], tmp2[0], rd[1], SRTYPE_LSR, 32 - val), ctx);
910                 emit(ARM_MOV_SI(rd[1], rd[1], SRTYPE_ASL, val), ctx);
911         } else {
912                 if (val == 32)
913                         emit(ARM_MOV_R(rd[0], rd[1]), ctx);
914                 else
915                         emit(ARM_MOV_SI(rd[0], rd[1], SRTYPE_ASL, val - 32), ctx);
916                 emit(ARM_EOR_R(rd[1], rd[1], rd[1]), ctx);
917         }
918 
919         arm_bpf_put_reg64(dst, rd, ctx);
920 }
921 
922 /* dst = dst >> val */
923 static inline void emit_a32_rsh_i64(const s8 dst[],
924                                     const u32 val, struct jit_ctx *ctx) {
925         const s8 *tmp = bpf2a32[TMP_REG_1];
926         const s8 *tmp2 = bpf2a32[TMP_REG_2];
927         const s8 *rd;
928 
929         /* Setup operands */
930         rd = arm_bpf_get_reg64(dst, tmp, ctx);
931 
932         /* Do LSR operation */
933         if (val < 32) {
934                 emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx);
935                 emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx);
936                 emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_LSR, val), ctx);
937         } else if (val == 32) {
938                 emit(ARM_MOV_R(rd[1], rd[0]), ctx);
939                 emit(ARM_MOV_I(rd[0], 0), ctx);
940         } else {
941                 emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_LSR, val - 32), ctx);
942                 emit(ARM_MOV_I(rd[0], 0), ctx);
943         }
944 
945         arm_bpf_put_reg64(dst, rd, ctx);
946 }
947 
948 /* dst = dst >> val (signed) */
949 static inline void emit_a32_arsh_i64(const s8 dst[],
950                                      const u32 val, struct jit_ctx *ctx){
951         const s8 *tmp = bpf2a32[TMP_REG_1];
952         const s8 *tmp2 = bpf2a32[TMP_REG_2];
953         const s8 *rd;
954 
955         /* Setup operands */
956         rd = arm_bpf_get_reg64(dst, tmp, ctx);
957 
958         /* Do ARSH operation */
959         if (val < 32) {
960                 emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx);
961                 emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx);
962                 emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, val), ctx);
963         } else if (val == 32) {
964                 emit(ARM_MOV_R(rd[1], rd[0]), ctx);
965                 emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx);
966         } else {
967                 emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_ASR, val - 32), ctx);
968                 emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx);
969         }
970 
971         arm_bpf_put_reg64(dst, rd, ctx);
972 }
973 
974 static inline void emit_a32_mul_r64(const s8 dst[], const s8 src[],
975                                     struct jit_ctx *ctx) {
976         const s8 *tmp = bpf2a32[TMP_REG_1];
977         const s8 *tmp2 = bpf2a32[TMP_REG_2];
978         const s8 *rd, *rt;
979 
980         /* Setup operands for multiplication */
981         rd = arm_bpf_get_reg64(dst, tmp, ctx);
982         rt = arm_bpf_get_reg64(src, tmp2, ctx);
983 
984         /* Do Multiplication */
985         emit(ARM_MUL(ARM_IP, rd[1], rt[0]), ctx);
986         emit(ARM_MUL(ARM_LR, rd[0], rt[1]), ctx);
987         emit(ARM_ADD_R(ARM_LR, ARM_IP, ARM_LR), ctx);
988 
989         emit(ARM_UMULL(ARM_IP, rd[0], rd[1], rt[1]), ctx);
990         emit(ARM_ADD_R(rd[0], ARM_LR, rd[0]), ctx);
991 
992         arm_bpf_put_reg32(dst_lo, ARM_IP, ctx);
993         arm_bpf_put_reg32(dst_hi, rd[0], ctx);
994 }
995 
996 /* *(size *)(dst + off) = src */
997 static inline void emit_str_r(const s8 dst, const s8 src[],
998                               s32 off, struct jit_ctx *ctx, const u8 sz){
999         const s8 *tmp = bpf2a32[TMP_REG_1];
1000         s32 off_max;
1001         s8 rd;
1002 
1003         rd = arm_bpf_get_reg32(dst, tmp[1], ctx);
1004 
1005         if (sz == BPF_H)
1006                 off_max = 0xff;
1007         else
1008                 off_max = 0xfff;
1009 
1010         if (off < 0 || off > off_max) {
1011                 emit_a32_mov_i(tmp[0], off, ctx);
1012                 emit(ARM_ADD_R(tmp[0], tmp[0], rd), ctx);
1013                 rd = tmp[0];
1014                 off = 0;
1015         }
1016         switch (sz) {
1017         case BPF_B:
1018                 /* Store a Byte */
1019                 emit(ARM_STRB_I(src_lo, rd, off), ctx);
1020                 break;
1021         case BPF_H:
1022                 /* Store a HalfWord */
1023                 emit(ARM_STRH_I(src_lo, rd, off), ctx);
1024                 break;
1025         case BPF_W:
1026                 /* Store a Word */
1027                 emit(ARM_STR_I(src_lo, rd, off), ctx);
1028                 break;
1029         case BPF_DW:
1030                 /* Store a Double Word */
1031                 emit(ARM_STR_I(src_lo, rd, off), ctx);
1032                 emit(ARM_STR_I(src_hi, rd, off + 4), ctx);
1033                 break;
1034         }
1035 }
1036 
1037 /* dst = *(size*)(src + off) */
1038 static inline void emit_ldx_r(const s8 dst[], const s8 src,
1039                               s32 off, struct jit_ctx *ctx, const u8 sz){
1040         const s8 *tmp = bpf2a32[TMP_REG_1];
1041         const s8 *rd = is_stacked(dst_lo) ? tmp : dst;
1042         s8 rm = src;
1043         s32 off_max;
1044 
1045         if (sz == BPF_H)
1046                 off_max = 0xff;
1047         else
1048                 off_max = 0xfff;
1049 
1050         if (off < 0 || off > off_max) {
1051                 emit_a32_mov_i(tmp[0], off, ctx);
1052                 emit(ARM_ADD_R(tmp[0], tmp[0], src), ctx);
1053                 rm = tmp[0];
1054                 off = 0;
1055         } else if (rd[1] == rm) {
1056                 emit(ARM_MOV_R(tmp[0], rm), ctx);
1057                 rm = tmp[0];
1058         }
1059         switch (sz) {
1060         case BPF_B:
1061                 /* Load a Byte */
1062                 emit(ARM_LDRB_I(rd[1], rm, off), ctx);
1063                 emit_a32_mov_i(rd[0], 0, ctx);
1064                 break;
1065         case BPF_H:
1066                 /* Load a HalfWord */
1067                 emit(ARM_LDRH_I(rd[1], rm, off), ctx);
1068                 emit_a32_mov_i(rd[0], 0, ctx);
1069                 break;
1070         case BPF_W:
1071                 /* Load a Word */
1072                 emit(ARM_LDR_I(rd[1], rm, off), ctx);
1073                 emit_a32_mov_i(rd[0], 0, ctx);
1074                 break;
1075         case BPF_DW:
1076                 /* Load a Double Word */
1077                 emit(ARM_LDR_I(rd[1], rm, off), ctx);
1078                 emit(ARM_LDR_I(rd[0], rm, off + 4), ctx);
1079                 break;
1080         }
1081         arm_bpf_put_reg64(dst, rd, ctx);
1082 }
1083 
1084 /* Arithmatic Operation */
1085 static inline void emit_ar_r(const u8 rd, const u8 rt, const u8 rm,
1086                              const u8 rn, struct jit_ctx *ctx, u8 op) {
1087         switch (op) {
1088         case BPF_JSET:
1089                 emit(ARM_AND_R(ARM_IP, rt, rn), ctx);
1090                 emit(ARM_AND_R(ARM_LR, rd, rm), ctx);
1091                 emit(ARM_ORRS_R(ARM_IP, ARM_LR, ARM_IP), ctx);
1092                 break;
1093         case BPF_JEQ:
1094         case BPF_JNE:
1095         case BPF_JGT:
1096         case BPF_JGE:
1097         case BPF_JLE:
1098         case BPF_JLT:
1099                 emit(ARM_CMP_R(rd, rm), ctx);
1100                 _emit(ARM_COND_EQ, ARM_CMP_R(rt, rn), ctx);
1101                 break;
1102         case BPF_JSLE:
1103         case BPF_JSGT:
1104                 emit(ARM_CMP_R(rn, rt), ctx);
1105                 emit(ARM_SBCS_R(ARM_IP, rm, rd), ctx);
1106                 break;
1107         case BPF_JSLT:
1108         case BPF_JSGE:
1109                 emit(ARM_CMP_R(rt, rn), ctx);
1110                 emit(ARM_SBCS_R(ARM_IP, rd, rm), ctx);
1111                 break;
1112         }
1113 }
1114 
1115 static int out_offset = -1; /* initialized on the first pass of build_body() */
1116 static int emit_bpf_tail_call(struct jit_ctx *ctx)
1117 {
1118 
1119         /* bpf_tail_call(void *prog_ctx, struct bpf_array *array, u64 index) */
1120         const s8 *r2 = bpf2a32[BPF_REG_2];
1121         const s8 *r3 = bpf2a32[BPF_REG_3];
1122         const s8 *tmp = bpf2a32[TMP_REG_1];
1123         const s8 *tmp2 = bpf2a32[TMP_REG_2];
1124         const s8 *tcc = bpf2a32[TCALL_CNT];
1125         const s8 *tc;
1126         const int idx0 = ctx->idx;
1127 #define cur_offset (ctx->idx - idx0)
1128 #define jmp_offset (out_offset - (cur_offset) - 2)
1129         u32 lo, hi;
1130         s8 r_array, r_index;
1131         int off;
1132 
1133         /* if (index >= array->map.max_entries)
1134          *      goto out;
1135          */
1136         BUILD_BUG_ON(offsetof(struct bpf_array, map.max_entries) >
1137                      ARM_INST_LDST__IMM12);
1138         off = offsetof(struct bpf_array, map.max_entries);
1139         r_array = arm_bpf_get_reg32(r2[1], tmp2[0], ctx);
1140         /* index is 32-bit for arrays */
1141         r_index = arm_bpf_get_reg32(r3[1], tmp2[1], ctx);
1142         /* array->map.max_entries */
1143         emit(ARM_LDR_I(tmp[1], r_array, off), ctx);
1144         /* index >= array->map.max_entries */
1145         emit(ARM_CMP_R(r_index, tmp[1]), ctx);
1146         _emit(ARM_COND_CS, ARM_B(jmp_offset), ctx);
1147 
1148         /* tmp2[0] = array, tmp2[1] = index */
1149 
1150         /* if (tail_call_cnt > MAX_TAIL_CALL_CNT)
1151          *      goto out;
1152          * tail_call_cnt++;
1153          */
1154         lo = (u32)MAX_TAIL_CALL_CNT;
1155         hi = (u32)((u64)MAX_TAIL_CALL_CNT >> 32);
1156         tc = arm_bpf_get_reg64(tcc, tmp, ctx);
1157         emit(ARM_CMP_I(tc[0], hi), ctx);
1158         _emit(ARM_COND_EQ, ARM_CMP_I(tc[1], lo), ctx);
1159         _emit(ARM_COND_HI, ARM_B(jmp_offset), ctx);
1160         emit(ARM_ADDS_I(tc[1], tc[1], 1), ctx);
1161         emit(ARM_ADC_I(tc[0], tc[0], 0), ctx);
1162         arm_bpf_put_reg64(tcc, tmp, ctx);
1163 
1164         /* prog = array->ptrs[index]
1165          * if (prog == NULL)
1166          *      goto out;
1167          */
1168         BUILD_BUG_ON(imm8m(offsetof(struct bpf_array, ptrs)) < 0);
1169         off = imm8m(offsetof(struct bpf_array, ptrs));
1170         emit(ARM_ADD_I(tmp[1], r_array, off), ctx);
1171         emit(ARM_LDR_R_SI(tmp[1], tmp[1], r_index, SRTYPE_ASL, 2), ctx);
1172         emit(ARM_CMP_I(tmp[1], 0), ctx);
1173         _emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx);
1174 
1175         /* goto *(prog->bpf_func + prologue_size); */
1176         BUILD_BUG_ON(offsetof(struct bpf_prog, bpf_func) >
1177                      ARM_INST_LDST__IMM12);
1178         off = offsetof(struct bpf_prog, bpf_func);
1179         emit(ARM_LDR_I(tmp[1], tmp[1], off), ctx);
1180         emit(ARM_ADD_I(tmp[1], tmp[1], ctx->prologue_bytes), ctx);
1181         emit_bx_r(tmp[1], ctx);
1182 
1183         /* out: */
1184         if (out_offset == -1)
1185                 out_offset = cur_offset;
1186         if (cur_offset != out_offset) {
1187                 pr_err_once("tail_call out_offset = %d, expected %d!\n",
1188                             cur_offset, out_offset);
1189                 return -1;
1190         }
1191         return 0;
1192 #undef cur_offset
1193 #undef jmp_offset
1194 }
1195 
1196 /* 0xabcd => 0xcdab */
1197 static inline void emit_rev16(const u8 rd, const u8 rn, struct jit_ctx *ctx)
1198 {
1199 #if __LINUX_ARM_ARCH__ < 6
1200         const s8 *tmp2 = bpf2a32[TMP_REG_2];
1201 
1202         emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx);
1203         emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 8), ctx);
1204         emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx);
1205         emit(ARM_ORR_SI(rd, tmp2[0], tmp2[1], SRTYPE_LSL, 8), ctx);
1206 #else /* ARMv6+ */
1207         emit(ARM_REV16(rd, rn), ctx);
1208 #endif
1209 }
1210 
1211 /* 0xabcdefgh => 0xghefcdab */
1212 static inline void emit_rev32(const u8 rd, const u8 rn, struct jit_ctx *ctx)
1213 {
1214 #if __LINUX_ARM_ARCH__ < 6
1215         const s8 *tmp2 = bpf2a32[TMP_REG_2];
1216 
1217         emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx);
1218         emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 24), ctx);
1219         emit(ARM_ORR_SI(ARM_IP, tmp2[0], tmp2[1], SRTYPE_LSL, 24), ctx);
1220 
1221         emit(ARM_MOV_SI(tmp2[1], rn, SRTYPE_LSR, 8), ctx);
1222         emit(ARM_AND_I(tmp2[1], tmp2[1], 0xff), ctx);
1223         emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 16), ctx);
1224         emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx);
1225         emit(ARM_MOV_SI(tmp2[0], tmp2[0], SRTYPE_LSL, 8), ctx);
1226         emit(ARM_ORR_SI(tmp2[0], tmp2[0], tmp2[1], SRTYPE_LSL, 16), ctx);
1227         emit(ARM_ORR_R(rd, ARM_IP, tmp2[0]), ctx);
1228 
1229 #else /* ARMv6+ */
1230         emit(ARM_REV(rd, rn), ctx);
1231 #endif
1232 }
1233 
1234 // push the scratch stack register on top of the stack
1235 static inline void emit_push_r64(const s8 src[], struct jit_ctx *ctx)
1236 {
1237         const s8 *tmp2 = bpf2a32[TMP_REG_2];
1238         const s8 *rt;
1239         u16 reg_set = 0;
1240 
1241         rt = arm_bpf_get_reg64(src, tmp2, ctx);
1242 
1243         reg_set = (1 << rt[1]) | (1 << rt[0]);
1244         emit(ARM_PUSH(reg_set), ctx);
1245 }
1246 
1247 static void build_prologue(struct jit_ctx *ctx)
1248 {
1249         const s8 r0 = bpf2a32[BPF_REG_0][1];
1250         const s8 r2 = bpf2a32[BPF_REG_1][1];
1251         const s8 r3 = bpf2a32[BPF_REG_1][0];
1252         const s8 r4 = bpf2a32[BPF_REG_6][1];
1253         const s8 fplo = bpf2a32[BPF_REG_FP][1];
1254         const s8 fphi = bpf2a32[BPF_REG_FP][0];
1255         const s8 *tcc = bpf2a32[TCALL_CNT];
1256 
1257         /* Save callee saved registers. */
1258 #ifdef CONFIG_FRAME_POINTER
1259         u16 reg_set = CALLEE_PUSH_MASK | 1 << ARM_IP | 1 << ARM_PC;
1260         emit(ARM_MOV_R(ARM_IP, ARM_SP), ctx);
1261         emit(ARM_PUSH(reg_set), ctx);
1262         emit(ARM_SUB_I(ARM_FP, ARM_IP, 4), ctx);
1263 #else
1264         emit(ARM_PUSH(CALLEE_PUSH_MASK), ctx);
1265         emit(ARM_MOV_R(ARM_FP, ARM_SP), ctx);
1266 #endif
1267         /* Save frame pointer for later */
1268         emit(ARM_SUB_I(ARM_IP, ARM_SP, SCRATCH_SIZE), ctx);
1269 
1270         ctx->stack_size = imm8m(STACK_SIZE);
1271 
1272         /* Set up function call stack */
1273         emit(ARM_SUB_I(ARM_SP, ARM_SP, ctx->stack_size), ctx);
1274 
1275         /* Set up BPF prog stack base register */
1276         emit_a32_mov_r(fplo, ARM_IP, ctx);
1277         emit_a32_mov_i(fphi, 0, ctx);
1278 
1279         /* mov r4, 0 */
1280         emit(ARM_MOV_I(r4, 0), ctx);
1281 
1282         /* Move BPF_CTX to BPF_R1 */
1283         emit(ARM_MOV_R(r3, r4), ctx);
1284         emit(ARM_MOV_R(r2, r0), ctx);
1285         /* Initialize Tail Count */
1286         emit(ARM_STR_I(r4, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(tcc[0])), ctx);
1287         emit(ARM_STR_I(r4, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(tcc[1])), ctx);
1288         /* end of prologue */
1289 }
1290 
1291 /* restore callee saved registers. */
1292 static void build_epilogue(struct jit_ctx *ctx)
1293 {
1294 #ifdef CONFIG_FRAME_POINTER
1295         /* When using frame pointers, some additional registers need to
1296          * be loaded. */
1297         u16 reg_set = CALLEE_POP_MASK | 1 << ARM_SP;
1298         emit(ARM_SUB_I(ARM_SP, ARM_FP, hweight16(reg_set) * 4), ctx);
1299         emit(ARM_LDM(ARM_SP, reg_set), ctx);
1300 #else
1301         /* Restore callee saved registers. */
1302         emit(ARM_MOV_R(ARM_SP, ARM_FP), ctx);
1303         emit(ARM_POP(CALLEE_POP_MASK), ctx);
1304 #endif
1305 }
1306 
1307 /*
1308  * Convert an eBPF instruction to native instruction, i.e
1309  * JITs an eBPF instruction.
1310  * Returns :
1311  *      0  - Successfully JITed an 8-byte eBPF instruction
1312  *      >0 - Successfully JITed a 16-byte eBPF instruction
1313  *      <0 - Failed to JIT.
1314  */
1315 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
1316 {
1317         const u8 code = insn->code;
1318         const s8 *dst = bpf2a32[insn->dst_reg];
1319         const s8 *src = bpf2a32[insn->src_reg];
1320         const s8 *tmp = bpf2a32[TMP_REG_1];
1321         const s8 *tmp2 = bpf2a32[TMP_REG_2];
1322         const s16 off = insn->off;
1323         const s32 imm = insn->imm;
1324         const int i = insn - ctx->prog->insnsi;
1325         const bool is64 = BPF_CLASS(code) == BPF_ALU64;
1326         const s8 *rd, *rs;
1327         s8 rd_lo, rt, rm, rn;
1328         s32 jmp_offset;
1329 
1330 #define check_imm(bits, imm) do {                               \
1331         if ((imm) >= (1 << ((bits) - 1)) ||                     \
1332             (imm) < -(1 << ((bits) - 1))) {                     \
1333                 pr_info("[%2d] imm=%d(0x%x) out of range\n",    \
1334                         i, imm, imm);                           \
1335                 return -EINVAL;                                 \
1336         }                                                       \
1337 } while (0)
1338 #define check_imm24(imm) check_imm(24, imm)
1339 
1340         switch (code) {
1341         /* ALU operations */
1342 
1343         /* dst = src */
1344         case BPF_ALU | BPF_MOV | BPF_K:
1345         case BPF_ALU | BPF_MOV | BPF_X:
1346         case BPF_ALU64 | BPF_MOV | BPF_K:
1347         case BPF_ALU64 | BPF_MOV | BPF_X:
1348                 switch (BPF_SRC(code)) {
1349                 case BPF_X:
1350                         emit_a32_mov_r64(is64, dst, src, ctx);
1351                         break;
1352                 case BPF_K:
1353                         /* Sign-extend immediate value to destination reg */
1354                         emit_a32_mov_se_i64(is64, dst, imm, ctx);
1355                         break;
1356                 }
1357                 break;
1358         /* dst = dst + src/imm */
1359         /* dst = dst - src/imm */
1360         /* dst = dst | src/imm */
1361         /* dst = dst & src/imm */
1362         /* dst = dst ^ src/imm */
1363         /* dst = dst * src/imm */
1364         /* dst = dst << src */
1365         /* dst = dst >> src */
1366         case BPF_ALU | BPF_ADD | BPF_K:
1367         case BPF_ALU | BPF_ADD | BPF_X:
1368         case BPF_ALU | BPF_SUB | BPF_K:
1369         case BPF_ALU | BPF_SUB | BPF_X:
1370         case BPF_ALU | BPF_OR | BPF_K:
1371         case BPF_ALU | BPF_OR | BPF_X:
1372         case BPF_ALU | BPF_AND | BPF_K:
1373         case BPF_ALU | BPF_AND | BPF_X:
1374         case BPF_ALU | BPF_XOR | BPF_K:
1375         case BPF_ALU | BPF_XOR | BPF_X:
1376         case BPF_ALU | BPF_MUL | BPF_K:
1377         case BPF_ALU | BPF_MUL | BPF_X:
1378         case BPF_ALU | BPF_LSH | BPF_X:
1379         case BPF_ALU | BPF_RSH | BPF_X:
1380         case BPF_ALU | BPF_ARSH | BPF_K:
1381         case BPF_ALU | BPF_ARSH | BPF_X:
1382         case BPF_ALU64 | BPF_ADD | BPF_K:
1383         case BPF_ALU64 | BPF_ADD | BPF_X:
1384         case BPF_ALU64 | BPF_SUB | BPF_K:
1385         case BPF_ALU64 | BPF_SUB | BPF_X:
1386         case BPF_ALU64 | BPF_OR | BPF_K:
1387         case BPF_ALU64 | BPF_OR | BPF_X:
1388         case BPF_ALU64 | BPF_AND | BPF_K:
1389         case BPF_ALU64 | BPF_AND | BPF_X:
1390         case BPF_ALU64 | BPF_XOR | BPF_K:
1391         case BPF_ALU64 | BPF_XOR | BPF_X:
1392                 switch (BPF_SRC(code)) {
1393                 case BPF_X:
1394                         emit_a32_alu_r64(is64, dst, src, ctx, BPF_OP(code));
1395                         break;
1396                 case BPF_K:
1397                         /* Move immediate value to the temporary register
1398                          * and then do the ALU operation on the temporary
1399                          * register as this will sign-extend the immediate
1400                          * value into temporary reg and then it would be
1401                          * safe to do the operation on it.
1402                          */
1403                         emit_a32_mov_se_i64(is64, tmp2, imm, ctx);
1404                         emit_a32_alu_r64(is64, dst, tmp2, ctx, BPF_OP(code));
1405                         break;
1406                 }
1407                 break;
1408         /* dst = dst / src(imm) */
1409         /* dst = dst % src(imm) */
1410         case BPF_ALU | BPF_DIV | BPF_K:
1411         case BPF_ALU | BPF_DIV | BPF_X:
1412         case BPF_ALU | BPF_MOD | BPF_K:
1413         case BPF_ALU | BPF_MOD | BPF_X:
1414                 rd_lo = arm_bpf_get_reg32(dst_lo, tmp2[1], ctx);
1415                 switch (BPF_SRC(code)) {
1416                 case BPF_X:
1417                         rt = arm_bpf_get_reg32(src_lo, tmp2[0], ctx);
1418                         break;
1419                 case BPF_K:
1420                         rt = tmp2[0];
1421                         emit_a32_mov_i(rt, imm, ctx);
1422                         break;
1423                 default:
1424                         rt = src_lo;
1425                         break;
1426                 }
1427                 emit_udivmod(rd_lo, rd_lo, rt, ctx, BPF_OP(code));
1428                 arm_bpf_put_reg32(dst_lo, rd_lo, ctx);
1429                 emit_a32_mov_i(dst_hi, 0, ctx);
1430                 break;
1431         case BPF_ALU64 | BPF_DIV | BPF_K:
1432         case BPF_ALU64 | BPF_DIV | BPF_X:
1433         case BPF_ALU64 | BPF_MOD | BPF_K:
1434         case BPF_ALU64 | BPF_MOD | BPF_X:
1435                 goto notyet;
1436         /* dst = dst >> imm */
1437         /* dst = dst << imm */
1438         case BPF_ALU | BPF_RSH | BPF_K:
1439         case BPF_ALU | BPF_LSH | BPF_K:
1440                 if (unlikely(imm > 31))
1441                         return -EINVAL;
1442                 if (imm)
1443                         emit_a32_alu_i(dst_lo, imm, ctx, BPF_OP(code));
1444                 emit_a32_mov_i(dst_hi, 0, ctx);
1445                 break;
1446         /* dst = dst << imm */
1447         case BPF_ALU64 | BPF_LSH | BPF_K:
1448                 if (unlikely(imm > 63))
1449                         return -EINVAL;
1450                 emit_a32_lsh_i64(dst, imm, ctx);
1451                 break;
1452         /* dst = dst >> imm */
1453         case BPF_ALU64 | BPF_RSH | BPF_K:
1454                 if (unlikely(imm > 63))
1455                         return -EINVAL;
1456                 emit_a32_rsh_i64(dst, imm, ctx);
1457                 break;
1458         /* dst = dst << src */
1459         case BPF_ALU64 | BPF_LSH | BPF_X:
1460                 emit_a32_lsh_r64(dst, src, ctx);
1461                 break;
1462         /* dst = dst >> src */
1463         case BPF_ALU64 | BPF_RSH | BPF_X:
1464                 emit_a32_rsh_r64(dst, src, ctx);
1465                 break;
1466         /* dst = dst >> src (signed) */
1467         case BPF_ALU64 | BPF_ARSH | BPF_X:
1468                 emit_a32_arsh_r64(dst, src, ctx);
1469                 break;
1470         /* dst = dst >> imm (signed) */
1471         case BPF_ALU64 | BPF_ARSH | BPF_K:
1472                 if (unlikely(imm > 63))
1473                         return -EINVAL;
1474                 emit_a32_arsh_i64(dst, imm, ctx);
1475                 break;
1476         /* dst = ~dst */
1477         case BPF_ALU | BPF_NEG:
1478                 emit_a32_alu_i(dst_lo, 0, ctx, BPF_OP(code));
1479                 emit_a32_mov_i(dst_hi, 0, ctx);
1480                 break;
1481         /* dst = ~dst (64 bit) */
1482         case BPF_ALU64 | BPF_NEG:
1483                 emit_a32_neg64(dst, ctx);
1484                 break;
1485         /* dst = dst * src/imm */
1486         case BPF_ALU64 | BPF_MUL | BPF_X:
1487         case BPF_ALU64 | BPF_MUL | BPF_K:
1488                 switch (BPF_SRC(code)) {
1489                 case BPF_X:
1490                         emit_a32_mul_r64(dst, src, ctx);
1491                         break;
1492                 case BPF_K:
1493                         /* Move immediate value to the temporary register
1494                          * and then do the multiplication on it as this
1495                          * will sign-extend the immediate value into temp
1496                          * reg then it would be safe to do the operation
1497                          * on it.
1498                          */
1499                         emit_a32_mov_se_i64(is64, tmp2, imm, ctx);
1500                         emit_a32_mul_r64(dst, tmp2, ctx);
1501                         break;
1502                 }
1503                 break;
1504         /* dst = htole(dst) */
1505         /* dst = htobe(dst) */
1506         case BPF_ALU | BPF_END | BPF_FROM_LE:
1507         case BPF_ALU | BPF_END | BPF_FROM_BE:
1508                 rd = arm_bpf_get_reg64(dst, tmp, ctx);
1509                 if (BPF_SRC(code) == BPF_FROM_LE)
1510                         goto emit_bswap_uxt;
1511                 switch (imm) {
1512                 case 16:
1513                         emit_rev16(rd[1], rd[1], ctx);
1514                         goto emit_bswap_uxt;
1515                 case 32:
1516                         emit_rev32(rd[1], rd[1], ctx);
1517                         goto emit_bswap_uxt;
1518                 case 64:
1519                         emit_rev32(ARM_LR, rd[1], ctx);
1520                         emit_rev32(rd[1], rd[0], ctx);
1521                         emit(ARM_MOV_R(rd[0], ARM_LR), ctx);
1522                         break;
1523                 }
1524                 goto exit;
1525 emit_bswap_uxt:
1526                 switch (imm) {
1527                 case 16:
1528                         /* zero-extend 16 bits into 64 bits */
1529 #if __LINUX_ARM_ARCH__ < 6
1530                         emit_a32_mov_i(tmp2[1], 0xffff, ctx);
1531                         emit(ARM_AND_R(rd[1], rd[1], tmp2[1]), ctx);
1532 #else /* ARMv6+ */
1533                         emit(ARM_UXTH(rd[1], rd[1]), ctx);
1534 #endif
1535                         emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx);
1536                         break;
1537                 case 32:
1538                         /* zero-extend 32 bits into 64 bits */
1539                         emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx);
1540                         break;
1541                 case 64:
1542                         /* nop */
1543                         break;
1544                 }
1545 exit:
1546                 arm_bpf_put_reg64(dst, rd, ctx);
1547                 break;
1548         /* dst = imm64 */
1549         case BPF_LD | BPF_IMM | BPF_DW:
1550         {
1551                 u64 val = (u32)imm | (u64)insn[1].imm << 32;
1552 
1553                 emit_a32_mov_i64(dst, val, ctx);
1554 
1555                 return 1;
1556         }
1557         /* LDX: dst = *(size *)(src + off) */
1558         case BPF_LDX | BPF_MEM | BPF_W:
1559         case BPF_LDX | BPF_MEM | BPF_H:
1560         case BPF_LDX | BPF_MEM | BPF_B:
1561         case BPF_LDX | BPF_MEM | BPF_DW:
1562                 rn = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
1563                 emit_ldx_r(dst, rn, off, ctx, BPF_SIZE(code));
1564                 break;
1565         /* ST: *(size *)(dst + off) = imm */
1566         case BPF_ST | BPF_MEM | BPF_W:
1567         case BPF_ST | BPF_MEM | BPF_H:
1568         case BPF_ST | BPF_MEM | BPF_B:
1569         case BPF_ST | BPF_MEM | BPF_DW:
1570                 switch (BPF_SIZE(code)) {
1571                 case BPF_DW:
1572                         /* Sign-extend immediate value into temp reg */
1573                         emit_a32_mov_se_i64(true, tmp2, imm, ctx);
1574                         break;
1575                 case BPF_W:
1576                 case BPF_H:
1577                 case BPF_B:
1578                         emit_a32_mov_i(tmp2[1], imm, ctx);
1579                         break;
1580                 }
1581                 emit_str_r(dst_lo, tmp2, off, ctx, BPF_SIZE(code));
1582                 break;
1583         /* STX XADD: lock *(u32 *)(dst + off) += src */
1584         case BPF_STX | BPF_XADD | BPF_W:
1585         /* STX XADD: lock *(u64 *)(dst + off) += src */
1586         case BPF_STX | BPF_XADD | BPF_DW:
1587                 goto notyet;
1588         /* STX: *(size *)(dst + off) = src */
1589         case BPF_STX | BPF_MEM | BPF_W:
1590         case BPF_STX | BPF_MEM | BPF_H:
1591         case BPF_STX | BPF_MEM | BPF_B:
1592         case BPF_STX | BPF_MEM | BPF_DW:
1593                 rs = arm_bpf_get_reg64(src, tmp2, ctx);
1594                 emit_str_r(dst_lo, rs, off, ctx, BPF_SIZE(code));
1595                 break;
1596         /* PC += off if dst == src */
1597         /* PC += off if dst > src */
1598         /* PC += off if dst >= src */
1599         /* PC += off if dst < src */
1600         /* PC += off if dst <= src */
1601         /* PC += off if dst != src */
1602         /* PC += off if dst > src (signed) */
1603         /* PC += off if dst >= src (signed) */
1604         /* PC += off if dst < src (signed) */
1605         /* PC += off if dst <= src (signed) */
1606         /* PC += off if dst & src */
1607         case BPF_JMP | BPF_JEQ | BPF_X:
1608         case BPF_JMP | BPF_JGT | BPF_X:
1609         case BPF_JMP | BPF_JGE | BPF_X:
1610         case BPF_JMP | BPF_JNE | BPF_X:
1611         case BPF_JMP | BPF_JSGT | BPF_X:
1612         case BPF_JMP | BPF_JSGE | BPF_X:
1613         case BPF_JMP | BPF_JSET | BPF_X:
1614         case BPF_JMP | BPF_JLE | BPF_X:
1615         case BPF_JMP | BPF_JLT | BPF_X:
1616         case BPF_JMP | BPF_JSLT | BPF_X:
1617         case BPF_JMP | BPF_JSLE | BPF_X:
1618                 /* Setup source registers */
1619                 rm = arm_bpf_get_reg32(src_hi, tmp2[0], ctx);
1620                 rn = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
1621                 goto go_jmp;
1622         /* PC += off if dst == imm */
1623         /* PC += off if dst > imm */
1624         /* PC += off if dst >= imm */
1625         /* PC += off if dst < imm */
1626         /* PC += off if dst <= imm */
1627         /* PC += off if dst != imm */
1628         /* PC += off if dst > imm (signed) */
1629         /* PC += off if dst >= imm (signed) */
1630         /* PC += off if dst < imm (signed) */
1631         /* PC += off if dst <= imm (signed) */
1632         /* PC += off if dst & imm */
1633         case BPF_JMP | BPF_JEQ | BPF_K:
1634         case BPF_JMP | BPF_JGT | BPF_K:
1635         case BPF_JMP | BPF_JGE | BPF_K:
1636         case BPF_JMP | BPF_JNE | BPF_K:
1637         case BPF_JMP | BPF_JSGT | BPF_K:
1638         case BPF_JMP | BPF_JSGE | BPF_K:
1639         case BPF_JMP | BPF_JSET | BPF_K:
1640         case BPF_JMP | BPF_JLT | BPF_K:
1641         case BPF_JMP | BPF_JLE | BPF_K:
1642         case BPF_JMP | BPF_JSLT | BPF_K:
1643         case BPF_JMP | BPF_JSLE | BPF_K:
1644                 if (off == 0)
1645                         break;
1646                 rm = tmp2[0];
1647                 rn = tmp2[1];
1648                 /* Sign-extend immediate value */
1649                 emit_a32_mov_se_i64(true, tmp2, imm, ctx);
1650 go_jmp:
1651                 /* Setup destination register */
1652                 rd = arm_bpf_get_reg64(dst, tmp, ctx);
1653 
1654                 /* Check for the condition */
1655                 emit_ar_r(rd[0], rd[1], rm, rn, ctx, BPF_OP(code));
1656 
1657                 /* Setup JUMP instruction */
1658                 jmp_offset = bpf2a32_offset(i+off, i, ctx);
1659                 switch (BPF_OP(code)) {
1660                 case BPF_JNE:
1661                 case BPF_JSET:
1662                         _emit(ARM_COND_NE, ARM_B(jmp_offset), ctx);
1663                         break;
1664                 case BPF_JEQ:
1665                         _emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx);
1666                         break;
1667                 case BPF_JGT:
1668                         _emit(ARM_COND_HI, ARM_B(jmp_offset), ctx);
1669                         break;
1670                 case BPF_JGE:
1671                         _emit(ARM_COND_CS, ARM_B(jmp_offset), ctx);
1672                         break;
1673                 case BPF_JSGT:
1674                         _emit(ARM_COND_LT, ARM_B(jmp_offset), ctx);
1675                         break;
1676                 case BPF_JSGE:
1677                         _emit(ARM_COND_GE, ARM_B(jmp_offset), ctx);
1678                         break;
1679                 case BPF_JLE:
1680                         _emit(ARM_COND_LS, ARM_B(jmp_offset), ctx);
1681                         break;
1682                 case BPF_JLT:
1683                         _emit(ARM_COND_CC, ARM_B(jmp_offset), ctx);
1684                         break;
1685                 case BPF_JSLT:
1686                         _emit(ARM_COND_LT, ARM_B(jmp_offset), ctx);
1687                         break;
1688                 case BPF_JSLE:
1689                         _emit(ARM_COND_GE, ARM_B(jmp_offset), ctx);
1690                         break;
1691                 }
1692                 break;
1693         /* JMP OFF */
1694         case BPF_JMP | BPF_JA:
1695         {
1696                 if (off == 0)
1697                         break;
1698                 jmp_offset = bpf2a32_offset(i+off, i, ctx);
1699                 check_imm24(jmp_offset);
1700                 emit(ARM_B(jmp_offset), ctx);
1701                 break;
1702         }
1703         /* tail call */
1704         case BPF_JMP | BPF_TAIL_CALL:
1705                 if (emit_bpf_tail_call(ctx))
1706                         return -EFAULT;
1707                 break;
1708         /* function call */
1709         case BPF_JMP | BPF_CALL:
1710         {
1711                 const s8 *r0 = bpf2a32[BPF_REG_0];
1712                 const s8 *r1 = bpf2a32[BPF_REG_1];
1713                 const s8 *r2 = bpf2a32[BPF_REG_2];
1714                 const s8 *r3 = bpf2a32[BPF_REG_3];
1715                 const s8 *r4 = bpf2a32[BPF_REG_4];
1716                 const s8 *r5 = bpf2a32[BPF_REG_5];
1717                 const u32 func = (u32)__bpf_call_base + (u32)imm;
1718 
1719                 emit_a32_mov_r64(true, r0, r1, ctx);
1720                 emit_a32_mov_r64(true, r1, r2, ctx);
1721                 emit_push_r64(r5, ctx);
1722                 emit_push_r64(r4, ctx);
1723                 emit_push_r64(r3, ctx);
1724 
1725                 emit_a32_mov_i(tmp[1], func, ctx);
1726                 emit_blx_r(tmp[1], ctx);
1727 
1728                 emit(ARM_ADD_I(ARM_SP, ARM_SP, imm8m(24)), ctx); // callee clean
1729                 break;
1730         }
1731         /* function return */
1732         case BPF_JMP | BPF_EXIT:
1733                 /* Optimization: when last instruction is EXIT
1734                  * simply fallthrough to epilogue.
1735                  */
1736                 if (i == ctx->prog->len - 1)
1737                         break;
1738                 jmp_offset = epilogue_offset(ctx);
1739                 check_imm24(jmp_offset);
1740                 emit(ARM_B(jmp_offset), ctx);
1741                 break;
1742 notyet:
1743                 pr_info_once("*** NOT YET: opcode %02x ***\n", code);
1744                 return -EFAULT;
1745         default:
1746                 pr_err_once("unknown opcode %02x\n", code);
1747                 return -EINVAL;
1748         }
1749 
1750         if (ctx->flags & FLAG_IMM_OVERFLOW)
1751                 /*
1752                  * this instruction generated an overflow when
1753                  * trying to access the literal pool, so
1754                  * delegate this filter to the kernel interpreter.
1755                  */
1756                 return -1;
1757         return 0;
1758 }
1759 
1760 static int build_body(struct jit_ctx *ctx)
1761 {
1762         const struct bpf_prog *prog = ctx->prog;
1763         unsigned int i;
1764 
1765         for (i = 0; i < prog->len; i++) {
1766                 const struct bpf_insn *insn = &(prog->insnsi[i]);
1767                 int ret;
1768 
1769                 ret = build_insn(insn, ctx);
1770 
1771                 /* It's used with loading the 64 bit immediate value. */
1772                 if (ret > 0) {
1773                         i++;
1774                         if (ctx->target == NULL)
1775                                 ctx->offsets[i] = ctx->idx;
1776                         continue;
1777                 }
1778 
1779                 if (ctx->target == NULL)
1780                         ctx->offsets[i] = ctx->idx;
1781 
1782                 /* If unsuccesfull, return with error code */
1783                 if (ret)
1784                         return ret;
1785         }
1786         return 0;
1787 }
1788 
1789 static int validate_code(struct jit_ctx *ctx)
1790 {
1791         int i;
1792 
1793         for (i = 0; i < ctx->idx; i++) {
1794                 if (ctx->target[i] == __opcode_to_mem_arm(ARM_INST_UDF))
1795                         return -1;
1796         }
1797 
1798         return 0;
1799 }
1800 
1801 void bpf_jit_compile(struct bpf_prog *prog)
1802 {
1803         /* Nothing to do here. We support Internal BPF. */
1804 }
1805 
1806 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1807 {
1808         struct bpf_prog *tmp, *orig_prog = prog;
1809         struct bpf_binary_header *header;
1810         bool tmp_blinded = false;
1811         struct jit_ctx ctx;
1812         unsigned int tmp_idx;
1813         unsigned int image_size;
1814         u8 *image_ptr;
1815 
1816         /* If BPF JIT was not enabled then we must fall back to
1817          * the interpreter.
1818          */
1819         if (!prog->jit_requested)
1820                 return orig_prog;
1821 
1822         /* If constant blinding was enabled and we failed during blinding
1823          * then we must fall back to the interpreter. Otherwise, we save
1824          * the new JITed code.
1825          */
1826         tmp = bpf_jit_blind_constants(prog);
1827 
1828         if (IS_ERR(tmp))
1829                 return orig_prog;
1830         if (tmp != prog) {
1831                 tmp_blinded = true;
1832                 prog = tmp;
1833         }
1834 
1835         memset(&ctx, 0, sizeof(ctx));
1836         ctx.prog = prog;
1837         ctx.cpu_architecture = cpu_architecture();
1838 
1839         /* Not able to allocate memory for offsets[] , then
1840          * we must fall back to the interpreter
1841          */
1842         ctx.offsets = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
1843         if (ctx.offsets == NULL) {
1844                 prog = orig_prog;
1845                 goto out;
1846         }
1847 
1848         /* 1) fake pass to find in the length of the JITed code,
1849          * to compute ctx->offsets and other context variables
1850          * needed to compute final JITed code.
1851          * Also, calculate random starting pointer/start of JITed code
1852          * which is prefixed by random number of fault instructions.
1853          *
1854          * If the first pass fails then there is no chance of it
1855          * being successful in the second pass, so just fall back
1856          * to the interpreter.
1857          */
1858         if (build_body(&ctx)) {
1859                 prog = orig_prog;
1860                 goto out_off;
1861         }
1862 
1863         tmp_idx = ctx.idx;
1864         build_prologue(&ctx);
1865         ctx.prologue_bytes = (ctx.idx - tmp_idx) * 4;
1866 
1867         ctx.epilogue_offset = ctx.idx;
1868 
1869 #if __LINUX_ARM_ARCH__ < 7
1870         tmp_idx = ctx.idx;
1871         build_epilogue(&ctx);
1872         ctx.epilogue_bytes = (ctx.idx - tmp_idx) * 4;
1873 
1874         ctx.idx += ctx.imm_count;
1875         if (ctx.imm_count) {
1876                 ctx.imms = kcalloc(ctx.imm_count, sizeof(u32), GFP_KERNEL);
1877                 if (ctx.imms == NULL) {
1878                         prog = orig_prog;
1879                         goto out_off;
1880                 }
1881         }
1882 #else
1883         /* there's nothing about the epilogue on ARMv7 */
1884         build_epilogue(&ctx);
1885 #endif
1886         /* Now we can get the actual image size of the JITed arm code.
1887          * Currently, we are not considering the THUMB-2 instructions
1888          * for jit, although it can decrease the size of the image.
1889          *
1890          * As each arm instruction is of length 32bit, we are translating
1891          * number of JITed intructions into the size required to store these
1892          * JITed code.
1893          */
1894         image_size = sizeof(u32) * ctx.idx;
1895 
1896         /* Now we know the size of the structure to make */
1897         header = bpf_jit_binary_alloc(image_size, &image_ptr,
1898                                       sizeof(u32), jit_fill_hole);
1899         /* Not able to allocate memory for the structure then
1900          * we must fall back to the interpretation
1901          */
1902         if (header == NULL) {
1903                 prog = orig_prog;
1904                 goto out_imms;
1905         }
1906 
1907         /* 2.) Actual pass to generate final JIT code */
1908         ctx.target = (u32 *) image_ptr;
1909         ctx.idx = 0;
1910 
1911         build_prologue(&ctx);
1912 
1913         /* If building the body of the JITed code fails somehow,
1914          * we fall back to the interpretation.
1915          */
1916         if (build_body(&ctx) < 0) {
1917                 image_ptr = NULL;
1918                 bpf_jit_binary_free(header);
1919                 prog = orig_prog;
1920                 goto out_imms;
1921         }
1922         build_epilogue(&ctx);
1923 
1924         /* 3.) Extra pass to validate JITed Code */
1925         if (validate_code(&ctx)) {
1926                 image_ptr = NULL;
1927                 bpf_jit_binary_free(header);
1928                 prog = orig_prog;
1929                 goto out_imms;
1930         }
1931         flush_icache_range((u32)header, (u32)(ctx.target + ctx.idx));
1932 
1933         if (bpf_jit_enable > 1)
1934                 /* there are 2 passes here */
1935                 bpf_jit_dump(prog->len, image_size, 2, ctx.target);
1936 
1937         bpf_jit_binary_lock_ro(header);
1938         prog->bpf_func = (void *)ctx.target;
1939         prog->jited = 1;
1940         prog->jited_len = image_size;
1941 
1942 out_imms:
1943 #if __LINUX_ARM_ARCH__ < 7
1944         if (ctx.imm_count)
1945                 kfree(ctx.imms);
1946 #endif
1947 out_off:
1948         kfree(ctx.offsets);
1949 out:
1950         if (tmp_blinded)
1951                 bpf_jit_prog_release_other(prog, prog == orig_prog ?
1952                                            tmp : orig_prog);
1953         return prog;
1954 }
1955 
1956 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp