~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/net/bpf_jit_comp.c

Version: ~ [ linux-5.2-rc1 ] ~ [ linux-5.1.2 ] ~ [ linux-5.0.16 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.43 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.119 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.176 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.179 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.139 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.67 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * bpf_jit_comp.c: BPF JIT compiler
  3  *
  4  * Copyright (C) 2011-2013 Eric Dumazet (eric.dumazet@gmail.com)
  5  * Internal BPF Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
  6  *
  7  * This program is free software; you can redistribute it and/or
  8  * modify it under the terms of the GNU General Public License
  9  * as published by the Free Software Foundation; version 2
 10  * of the License.
 11  */
 12 #include <linux/netdevice.h>
 13 #include <linux/filter.h>
 14 #include <linux/if_vlan.h>
 15 #include <linux/bpf.h>
 16 
 17 #include <asm/set_memory.h>
 18 #include <asm/nospec-branch.h>
 19 
 20 static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len)
 21 {
 22         if (len == 1)
 23                 *ptr = bytes;
 24         else if (len == 2)
 25                 *(u16 *)ptr = bytes;
 26         else {
 27                 *(u32 *)ptr = bytes;
 28                 barrier();
 29         }
 30         return ptr + len;
 31 }
 32 
 33 #define EMIT(bytes, len) \
 34         do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)
 35 
 36 #define EMIT1(b1)               EMIT(b1, 1)
 37 #define EMIT2(b1, b2)           EMIT((b1) + ((b2) << 8), 2)
 38 #define EMIT3(b1, b2, b3)       EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3)
 39 #define EMIT4(b1, b2, b3, b4)   EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4)
 40 
 41 #define EMIT1_off32(b1, off) \
 42         do { EMIT1(b1); EMIT(off, 4); } while (0)
 43 #define EMIT2_off32(b1, b2, off) \
 44         do { EMIT2(b1, b2); EMIT(off, 4); } while (0)
 45 #define EMIT3_off32(b1, b2, b3, off) \
 46         do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)
 47 #define EMIT4_off32(b1, b2, b3, b4, off) \
 48         do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)
 49 
 50 static bool is_imm8(int value)
 51 {
 52         return value <= 127 && value >= -128;
 53 }
 54 
 55 static bool is_simm32(s64 value)
 56 {
 57         return value == (s64)(s32)value;
 58 }
 59 
 60 static bool is_uimm32(u64 value)
 61 {
 62         return value == (u64)(u32)value;
 63 }
 64 
 65 /* mov dst, src */
 66 #define EMIT_mov(DST, SRC)                                                               \
 67         do {                                                                             \
 68                 if (DST != SRC)                                                          \
 69                         EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \
 70         } while (0)
 71 
 72 static int bpf_size_to_x86_bytes(int bpf_size)
 73 {
 74         if (bpf_size == BPF_W)
 75                 return 4;
 76         else if (bpf_size == BPF_H)
 77                 return 2;
 78         else if (bpf_size == BPF_B)
 79                 return 1;
 80         else if (bpf_size == BPF_DW)
 81                 return 4; /* imm32 */
 82         else
 83                 return 0;
 84 }
 85 
 86 /*
 87  * List of x86 cond jumps opcodes (. + s8)
 88  * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32)
 89  */
 90 #define X86_JB  0x72
 91 #define X86_JAE 0x73
 92 #define X86_JE  0x74
 93 #define X86_JNE 0x75
 94 #define X86_JBE 0x76
 95 #define X86_JA  0x77
 96 #define X86_JL  0x7C
 97 #define X86_JGE 0x7D
 98 #define X86_JLE 0x7E
 99 #define X86_JG  0x7F
100 
101 /* Pick a register outside of BPF range for JIT internal work */
102 #define AUX_REG (MAX_BPF_JIT_REG + 1)
103 
104 /*
105  * The following table maps BPF registers to x86-64 registers.
106  *
107  * x86-64 register R12 is unused, since if used as base address
108  * register in load/store instructions, it always needs an
109  * extra byte of encoding and is callee saved.
110  *
111  * Also x86-64 register R9 is unused. x86-64 register R10 is
112  * used for blinding (if enabled).
113  */
114 static const int reg2hex[] = {
115         [BPF_REG_0] = 0,  /* RAX */
116         [BPF_REG_1] = 7,  /* RDI */
117         [BPF_REG_2] = 6,  /* RSI */
118         [BPF_REG_3] = 2,  /* RDX */
119         [BPF_REG_4] = 1,  /* RCX */
120         [BPF_REG_5] = 0,  /* R8  */
121         [BPF_REG_6] = 3,  /* RBX callee saved */
122         [BPF_REG_7] = 5,  /* R13 callee saved */
123         [BPF_REG_8] = 6,  /* R14 callee saved */
124         [BPF_REG_9] = 7,  /* R15 callee saved */
125         [BPF_REG_FP] = 5, /* RBP readonly */
126         [BPF_REG_AX] = 2, /* R10 temp register */
127         [AUX_REG] = 3,    /* R11 temp register */
128 };
129 
130 /*
131  * is_ereg() == true if BPF register 'reg' maps to x86-64 r8..r15
132  * which need extra byte of encoding.
133  * rax,rcx,...,rbp have simpler encoding
134  */
135 static bool is_ereg(u32 reg)
136 {
137         return (1 << reg) & (BIT(BPF_REG_5) |
138                              BIT(AUX_REG) |
139                              BIT(BPF_REG_7) |
140                              BIT(BPF_REG_8) |
141                              BIT(BPF_REG_9) |
142                              BIT(BPF_REG_AX));
143 }
144 
145 static bool is_axreg(u32 reg)
146 {
147         return reg == BPF_REG_0;
148 }
149 
150 /* Add modifiers if 'reg' maps to x86-64 registers R8..R15 */
151 static u8 add_1mod(u8 byte, u32 reg)
152 {
153         if (is_ereg(reg))
154                 byte |= 1;
155         return byte;
156 }
157 
158 static u8 add_2mod(u8 byte, u32 r1, u32 r2)
159 {
160         if (is_ereg(r1))
161                 byte |= 1;
162         if (is_ereg(r2))
163                 byte |= 4;
164         return byte;
165 }
166 
167 /* Encode 'dst_reg' register into x86-64 opcode 'byte' */
168 static u8 add_1reg(u8 byte, u32 dst_reg)
169 {
170         return byte + reg2hex[dst_reg];
171 }
172 
173 /* Encode 'dst_reg' and 'src_reg' registers into x86-64 opcode 'byte' */
174 static u8 add_2reg(u8 byte, u32 dst_reg, u32 src_reg)
175 {
176         return byte + reg2hex[dst_reg] + (reg2hex[src_reg] << 3);
177 }
178 
179 static void jit_fill_hole(void *area, unsigned int size)
180 {
181         /* Fill whole space with INT3 instructions */
182         memset(area, 0xcc, size);
183 }
184 
185 struct jit_context {
186         int cleanup_addr; /* Epilogue code offset */
187 };
188 
189 /* Maximum number of bytes emitted while JITing one eBPF insn */
190 #define BPF_MAX_INSN_SIZE       128
191 #define BPF_INSN_SAFETY         64
192 
193 #define AUX_STACK_SPACE         40 /* Space for RBX, R13, R14, R15, tailcnt */
194 
195 #define PROLOGUE_SIZE           37
196 
197 /*
198  * Emit x86-64 prologue code for BPF program and check its size.
199  * bpf_tail_call helper will skip it while jumping into another program
200  */
201 static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf)
202 {
203         u8 *prog = *pprog;
204         int cnt = 0;
205 
206         /* push rbp */
207         EMIT1(0x55);
208 
209         /* mov rbp,rsp */
210         EMIT3(0x48, 0x89, 0xE5);
211 
212         /* sub rsp, rounded_stack_depth + AUX_STACK_SPACE */
213         EMIT3_off32(0x48, 0x81, 0xEC,
214                     round_up(stack_depth, 8) + AUX_STACK_SPACE);
215 
216         /* sub rbp, AUX_STACK_SPACE */
217         EMIT4(0x48, 0x83, 0xED, AUX_STACK_SPACE);
218 
219         /* mov qword ptr [rbp+0],rbx */
220         EMIT4(0x48, 0x89, 0x5D, 0);
221         /* mov qword ptr [rbp+8],r13 */
222         EMIT4(0x4C, 0x89, 0x6D, 8);
223         /* mov qword ptr [rbp+16],r14 */
224         EMIT4(0x4C, 0x89, 0x75, 16);
225         /* mov qword ptr [rbp+24],r15 */
226         EMIT4(0x4C, 0x89, 0x7D, 24);
227 
228         if (!ebpf_from_cbpf) {
229                 /*
230                  * Clear the tail call counter (tail_call_cnt): for eBPF tail
231                  * calls we need to reset the counter to 0. It's done in two
232                  * instructions, resetting RAX register to 0, and moving it
233                  * to the counter location.
234                  */
235 
236                 /* xor eax, eax */
237                 EMIT2(0x31, 0xc0);
238                 /* mov qword ptr [rbp+32], rax */
239                 EMIT4(0x48, 0x89, 0x45, 32);
240 
241                 BUILD_BUG_ON(cnt != PROLOGUE_SIZE);
242         }
243 
244         *pprog = prog;
245 }
246 
247 /*
248  * Generate the following code:
249  *
250  * ... bpf_tail_call(void *ctx, struct bpf_array *array, u64 index) ...
251  *   if (index >= array->map.max_entries)
252  *     goto out;
253  *   if (++tail_call_cnt > MAX_TAIL_CALL_CNT)
254  *     goto out;
255  *   prog = array->ptrs[index];
256  *   if (prog == NULL)
257  *     goto out;
258  *   goto *(prog->bpf_func + prologue_size);
259  * out:
260  */
261 static void emit_bpf_tail_call(u8 **pprog)
262 {
263         u8 *prog = *pprog;
264         int label1, label2, label3;
265         int cnt = 0;
266 
267         /*
268          * rdi - pointer to ctx
269          * rsi - pointer to bpf_array
270          * rdx - index in bpf_array
271          */
272 
273         /*
274          * if (index >= array->map.max_entries)
275          *      goto out;
276          */
277         EMIT2(0x89, 0xD2);                        /* mov edx, edx */
278         EMIT3(0x39, 0x56,                         /* cmp dword ptr [rsi + 16], edx */
279               offsetof(struct bpf_array, map.max_entries));
280 #define OFFSET1 (41 + RETPOLINE_RAX_BPF_JIT_SIZE) /* Number of bytes to jump */
281         EMIT2(X86_JBE, OFFSET1);                  /* jbe out */
282         label1 = cnt;
283 
284         /*
285          * if (tail_call_cnt > MAX_TAIL_CALL_CNT)
286          *      goto out;
287          */
288         EMIT2_off32(0x8B, 0x85, 36);              /* mov eax, dword ptr [rbp + 36] */
289         EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT);     /* cmp eax, MAX_TAIL_CALL_CNT */
290 #define OFFSET2 (30 + RETPOLINE_RAX_BPF_JIT_SIZE)
291         EMIT2(X86_JA, OFFSET2);                   /* ja out */
292         label2 = cnt;
293         EMIT3(0x83, 0xC0, 0x01);                  /* add eax, 1 */
294         EMIT2_off32(0x89, 0x85, 36);              /* mov dword ptr [rbp + 36], eax */
295 
296         /* prog = array->ptrs[index]; */
297         EMIT4_off32(0x48, 0x8B, 0x84, 0xD6,       /* mov rax, [rsi + rdx * 8 + offsetof(...)] */
298                     offsetof(struct bpf_array, ptrs));
299 
300         /*
301          * if (prog == NULL)
302          *      goto out;
303          */
304         EMIT3(0x48, 0x85, 0xC0);                  /* test rax,rax */
305 #define OFFSET3 (8 + RETPOLINE_RAX_BPF_JIT_SIZE)
306         EMIT2(X86_JE, OFFSET3);                   /* je out */
307         label3 = cnt;
308 
309         /* goto *(prog->bpf_func + prologue_size); */
310         EMIT4(0x48, 0x8B, 0x40,                   /* mov rax, qword ptr [rax + 32] */
311               offsetof(struct bpf_prog, bpf_func));
312         EMIT4(0x48, 0x83, 0xC0, PROLOGUE_SIZE);   /* add rax, prologue_size */
313 
314         /*
315          * Wow we're ready to jump into next BPF program
316          * rdi == ctx (1st arg)
317          * rax == prog->bpf_func + prologue_size
318          */
319         RETPOLINE_RAX_BPF_JIT();
320 
321         /* out: */
322         BUILD_BUG_ON(cnt - label1 != OFFSET1);
323         BUILD_BUG_ON(cnt - label2 != OFFSET2);
324         BUILD_BUG_ON(cnt - label3 != OFFSET3);
325         *pprog = prog;
326 }
327 
328 static void emit_mov_imm32(u8 **pprog, bool sign_propagate,
329                            u32 dst_reg, const u32 imm32)
330 {
331         u8 *prog = *pprog;
332         u8 b1, b2, b3;
333         int cnt = 0;
334 
335         /*
336          * Optimization: if imm32 is positive, use 'mov %eax, imm32'
337          * (which zero-extends imm32) to save 2 bytes.
338          */
339         if (sign_propagate && (s32)imm32 < 0) {
340                 /* 'mov %rax, imm32' sign extends imm32 */
341                 b1 = add_1mod(0x48, dst_reg);
342                 b2 = 0xC7;
343                 b3 = 0xC0;
344                 EMIT3_off32(b1, b2, add_1reg(b3, dst_reg), imm32);
345                 goto done;
346         }
347 
348         /*
349          * Optimization: if imm32 is zero, use 'xor %eax, %eax'
350          * to save 3 bytes.
351          */
352         if (imm32 == 0) {
353                 if (is_ereg(dst_reg))
354                         EMIT1(add_2mod(0x40, dst_reg, dst_reg));
355                 b2 = 0x31; /* xor */
356                 b3 = 0xC0;
357                 EMIT2(b2, add_2reg(b3, dst_reg, dst_reg));
358                 goto done;
359         }
360 
361         /* mov %eax, imm32 */
362         if (is_ereg(dst_reg))
363                 EMIT1(add_1mod(0x40, dst_reg));
364         EMIT1_off32(add_1reg(0xB8, dst_reg), imm32);
365 done:
366         *pprog = prog;
367 }
368 
369 static void emit_mov_imm64(u8 **pprog, u32 dst_reg,
370                            const u32 imm32_hi, const u32 imm32_lo)
371 {
372         u8 *prog = *pprog;
373         int cnt = 0;
374 
375         if (is_uimm32(((u64)imm32_hi << 32) | (u32)imm32_lo)) {
376                 /*
377                  * For emitting plain u32, where sign bit must not be
378                  * propagated LLVM tends to load imm64 over mov32
379                  * directly, so save couple of bytes by just doing
380                  * 'mov %eax, imm32' instead.
381                  */
382                 emit_mov_imm32(&prog, false, dst_reg, imm32_lo);
383         } else {
384                 /* movabsq %rax, imm64 */
385                 EMIT2(add_1mod(0x48, dst_reg), add_1reg(0xB8, dst_reg));
386                 EMIT(imm32_lo, 4);
387                 EMIT(imm32_hi, 4);
388         }
389 
390         *pprog = prog;
391 }
392 
393 static void emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg)
394 {
395         u8 *prog = *pprog;
396         int cnt = 0;
397 
398         if (is64) {
399                 /* mov dst, src */
400                 EMIT_mov(dst_reg, src_reg);
401         } else {
402                 /* mov32 dst, src */
403                 if (is_ereg(dst_reg) || is_ereg(src_reg))
404                         EMIT1(add_2mod(0x40, dst_reg, src_reg));
405                 EMIT2(0x89, add_2reg(0xC0, dst_reg, src_reg));
406         }
407 
408         *pprog = prog;
409 }
410 
411 static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
412                   int oldproglen, struct jit_context *ctx)
413 {
414         struct bpf_insn *insn = bpf_prog->insnsi;
415         int insn_cnt = bpf_prog->len;
416         bool seen_exit = false;
417         u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];
418         int i, cnt = 0;
419         int proglen = 0;
420         u8 *prog = temp;
421 
422         emit_prologue(&prog, bpf_prog->aux->stack_depth,
423                       bpf_prog_was_classic(bpf_prog));
424 
425         for (i = 0; i < insn_cnt; i++, insn++) {
426                 const s32 imm32 = insn->imm;
427                 u32 dst_reg = insn->dst_reg;
428                 u32 src_reg = insn->src_reg;
429                 u8 b2 = 0, b3 = 0;
430                 s64 jmp_offset;
431                 u8 jmp_cond;
432                 int ilen;
433                 u8 *func;
434 
435                 switch (insn->code) {
436                         /* ALU */
437                 case BPF_ALU | BPF_ADD | BPF_X:
438                 case BPF_ALU | BPF_SUB | BPF_X:
439                 case BPF_ALU | BPF_AND | BPF_X:
440                 case BPF_ALU | BPF_OR | BPF_X:
441                 case BPF_ALU | BPF_XOR | BPF_X:
442                 case BPF_ALU64 | BPF_ADD | BPF_X:
443                 case BPF_ALU64 | BPF_SUB | BPF_X:
444                 case BPF_ALU64 | BPF_AND | BPF_X:
445                 case BPF_ALU64 | BPF_OR | BPF_X:
446                 case BPF_ALU64 | BPF_XOR | BPF_X:
447                         switch (BPF_OP(insn->code)) {
448                         case BPF_ADD: b2 = 0x01; break;
449                         case BPF_SUB: b2 = 0x29; break;
450                         case BPF_AND: b2 = 0x21; break;
451                         case BPF_OR: b2 = 0x09; break;
452                         case BPF_XOR: b2 = 0x31; break;
453                         }
454                         if (BPF_CLASS(insn->code) == BPF_ALU64)
455                                 EMIT1(add_2mod(0x48, dst_reg, src_reg));
456                         else if (is_ereg(dst_reg) || is_ereg(src_reg))
457                                 EMIT1(add_2mod(0x40, dst_reg, src_reg));
458                         EMIT2(b2, add_2reg(0xC0, dst_reg, src_reg));
459                         break;
460 
461                 case BPF_ALU64 | BPF_MOV | BPF_X:
462                 case BPF_ALU | BPF_MOV | BPF_X:
463                         emit_mov_reg(&prog,
464                                      BPF_CLASS(insn->code) == BPF_ALU64,
465                                      dst_reg, src_reg);
466                         break;
467 
468                         /* neg dst */
469                 case BPF_ALU | BPF_NEG:
470                 case BPF_ALU64 | BPF_NEG:
471                         if (BPF_CLASS(insn->code) == BPF_ALU64)
472                                 EMIT1(add_1mod(0x48, dst_reg));
473                         else if (is_ereg(dst_reg))
474                                 EMIT1(add_1mod(0x40, dst_reg));
475                         EMIT2(0xF7, add_1reg(0xD8, dst_reg));
476                         break;
477 
478                 case BPF_ALU | BPF_ADD | BPF_K:
479                 case BPF_ALU | BPF_SUB | BPF_K:
480                 case BPF_ALU | BPF_AND | BPF_K:
481                 case BPF_ALU | BPF_OR | BPF_K:
482                 case BPF_ALU | BPF_XOR | BPF_K:
483                 case BPF_ALU64 | BPF_ADD | BPF_K:
484                 case BPF_ALU64 | BPF_SUB | BPF_K:
485                 case BPF_ALU64 | BPF_AND | BPF_K:
486                 case BPF_ALU64 | BPF_OR | BPF_K:
487                 case BPF_ALU64 | BPF_XOR | BPF_K:
488                         if (BPF_CLASS(insn->code) == BPF_ALU64)
489                                 EMIT1(add_1mod(0x48, dst_reg));
490                         else if (is_ereg(dst_reg))
491                                 EMIT1(add_1mod(0x40, dst_reg));
492 
493                         /*
494                          * b3 holds 'normal' opcode, b2 short form only valid
495                          * in case dst is eax/rax.
496                          */
497                         switch (BPF_OP(insn->code)) {
498                         case BPF_ADD:
499                                 b3 = 0xC0;
500                                 b2 = 0x05;
501                                 break;
502                         case BPF_SUB:
503                                 b3 = 0xE8;
504                                 b2 = 0x2D;
505                                 break;
506                         case BPF_AND:
507                                 b3 = 0xE0;
508                                 b2 = 0x25;
509                                 break;
510                         case BPF_OR:
511                                 b3 = 0xC8;
512                                 b2 = 0x0D;
513                                 break;
514                         case BPF_XOR:
515                                 b3 = 0xF0;
516                                 b2 = 0x35;
517                                 break;
518                         }
519 
520                         if (is_imm8(imm32))
521                                 EMIT3(0x83, add_1reg(b3, dst_reg), imm32);
522                         else if (is_axreg(dst_reg))
523                                 EMIT1_off32(b2, imm32);
524                         else
525                                 EMIT2_off32(0x81, add_1reg(b3, dst_reg), imm32);
526                         break;
527 
528                 case BPF_ALU64 | BPF_MOV | BPF_K:
529                 case BPF_ALU | BPF_MOV | BPF_K:
530                         emit_mov_imm32(&prog, BPF_CLASS(insn->code) == BPF_ALU64,
531                                        dst_reg, imm32);
532                         break;
533 
534                 case BPF_LD | BPF_IMM | BPF_DW:
535                         emit_mov_imm64(&prog, dst_reg, insn[1].imm, insn[0].imm);
536                         insn++;
537                         i++;
538                         break;
539 
540                         /* dst %= src, dst /= src, dst %= imm32, dst /= imm32 */
541                 case BPF_ALU | BPF_MOD | BPF_X:
542                 case BPF_ALU | BPF_DIV | BPF_X:
543                 case BPF_ALU | BPF_MOD | BPF_K:
544                 case BPF_ALU | BPF_DIV | BPF_K:
545                 case BPF_ALU64 | BPF_MOD | BPF_X:
546                 case BPF_ALU64 | BPF_DIV | BPF_X:
547                 case BPF_ALU64 | BPF_MOD | BPF_K:
548                 case BPF_ALU64 | BPF_DIV | BPF_K:
549                         EMIT1(0x50); /* push rax */
550                         EMIT1(0x52); /* push rdx */
551 
552                         if (BPF_SRC(insn->code) == BPF_X)
553                                 /* mov r11, src_reg */
554                                 EMIT_mov(AUX_REG, src_reg);
555                         else
556                                 /* mov r11, imm32 */
557                                 EMIT3_off32(0x49, 0xC7, 0xC3, imm32);
558 
559                         /* mov rax, dst_reg */
560                         EMIT_mov(BPF_REG_0, dst_reg);
561 
562                         /*
563                          * xor edx, edx
564                          * equivalent to 'xor rdx, rdx', but one byte less
565                          */
566                         EMIT2(0x31, 0xd2);
567 
568                         if (BPF_CLASS(insn->code) == BPF_ALU64)
569                                 /* div r11 */
570                                 EMIT3(0x49, 0xF7, 0xF3);
571                         else
572                                 /* div r11d */
573                                 EMIT3(0x41, 0xF7, 0xF3);
574 
575                         if (BPF_OP(insn->code) == BPF_MOD)
576                                 /* mov r11, rdx */
577                                 EMIT3(0x49, 0x89, 0xD3);
578                         else
579                                 /* mov r11, rax */
580                                 EMIT3(0x49, 0x89, 0xC3);
581 
582                         EMIT1(0x5A); /* pop rdx */
583                         EMIT1(0x58); /* pop rax */
584 
585                         /* mov dst_reg, r11 */
586                         EMIT_mov(dst_reg, AUX_REG);
587                         break;
588 
589                 case BPF_ALU | BPF_MUL | BPF_K:
590                 case BPF_ALU | BPF_MUL | BPF_X:
591                 case BPF_ALU64 | BPF_MUL | BPF_K:
592                 case BPF_ALU64 | BPF_MUL | BPF_X:
593                 {
594                         bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
595 
596                         if (dst_reg != BPF_REG_0)
597                                 EMIT1(0x50); /* push rax */
598                         if (dst_reg != BPF_REG_3)
599                                 EMIT1(0x52); /* push rdx */
600 
601                         /* mov r11, dst_reg */
602                         EMIT_mov(AUX_REG, dst_reg);
603 
604                         if (BPF_SRC(insn->code) == BPF_X)
605                                 emit_mov_reg(&prog, is64, BPF_REG_0, src_reg);
606                         else
607                                 emit_mov_imm32(&prog, is64, BPF_REG_0, imm32);
608 
609                         if (is64)
610                                 EMIT1(add_1mod(0x48, AUX_REG));
611                         else if (is_ereg(AUX_REG))
612                                 EMIT1(add_1mod(0x40, AUX_REG));
613                         /* mul(q) r11 */
614                         EMIT2(0xF7, add_1reg(0xE0, AUX_REG));
615 
616                         if (dst_reg != BPF_REG_3)
617                                 EMIT1(0x5A); /* pop rdx */
618                         if (dst_reg != BPF_REG_0) {
619                                 /* mov dst_reg, rax */
620                                 EMIT_mov(dst_reg, BPF_REG_0);
621                                 EMIT1(0x58); /* pop rax */
622                         }
623                         break;
624                 }
625                         /* Shifts */
626                 case BPF_ALU | BPF_LSH | BPF_K:
627                 case BPF_ALU | BPF_RSH | BPF_K:
628                 case BPF_ALU | BPF_ARSH | BPF_K:
629                 case BPF_ALU64 | BPF_LSH | BPF_K:
630                 case BPF_ALU64 | BPF_RSH | BPF_K:
631                 case BPF_ALU64 | BPF_ARSH | BPF_K:
632                         if (BPF_CLASS(insn->code) == BPF_ALU64)
633                                 EMIT1(add_1mod(0x48, dst_reg));
634                         else if (is_ereg(dst_reg))
635                                 EMIT1(add_1mod(0x40, dst_reg));
636 
637                         switch (BPF_OP(insn->code)) {
638                         case BPF_LSH: b3 = 0xE0; break;
639                         case BPF_RSH: b3 = 0xE8; break;
640                         case BPF_ARSH: b3 = 0xF8; break;
641                         }
642 
643                         if (imm32 == 1)
644                                 EMIT2(0xD1, add_1reg(b3, dst_reg));
645                         else
646                                 EMIT3(0xC1, add_1reg(b3, dst_reg), imm32);
647                         break;
648 
649                 case BPF_ALU | BPF_LSH | BPF_X:
650                 case BPF_ALU | BPF_RSH | BPF_X:
651                 case BPF_ALU | BPF_ARSH | BPF_X:
652                 case BPF_ALU64 | BPF_LSH | BPF_X:
653                 case BPF_ALU64 | BPF_RSH | BPF_X:
654                 case BPF_ALU64 | BPF_ARSH | BPF_X:
655 
656                         /* Check for bad case when dst_reg == rcx */
657                         if (dst_reg == BPF_REG_4) {
658                                 /* mov r11, dst_reg */
659                                 EMIT_mov(AUX_REG, dst_reg);
660                                 dst_reg = AUX_REG;
661                         }
662 
663                         if (src_reg != BPF_REG_4) { /* common case */
664                                 EMIT1(0x51); /* push rcx */
665 
666                                 /* mov rcx, src_reg */
667                                 EMIT_mov(BPF_REG_4, src_reg);
668                         }
669 
670                         /* shl %rax, %cl | shr %rax, %cl | sar %rax, %cl */
671                         if (BPF_CLASS(insn->code) == BPF_ALU64)
672                                 EMIT1(add_1mod(0x48, dst_reg));
673                         else if (is_ereg(dst_reg))
674                                 EMIT1(add_1mod(0x40, dst_reg));
675 
676                         switch (BPF_OP(insn->code)) {
677                         case BPF_LSH: b3 = 0xE0; break;
678                         case BPF_RSH: b3 = 0xE8; break;
679                         case BPF_ARSH: b3 = 0xF8; break;
680                         }
681                         EMIT2(0xD3, add_1reg(b3, dst_reg));
682 
683                         if (src_reg != BPF_REG_4)
684                                 EMIT1(0x59); /* pop rcx */
685 
686                         if (insn->dst_reg == BPF_REG_4)
687                                 /* mov dst_reg, r11 */
688                                 EMIT_mov(insn->dst_reg, AUX_REG);
689                         break;
690 
691                 case BPF_ALU | BPF_END | BPF_FROM_BE:
692                         switch (imm32) {
693                         case 16:
694                                 /* Emit 'ror %ax, 8' to swap lower 2 bytes */
695                                 EMIT1(0x66);
696                                 if (is_ereg(dst_reg))
697                                         EMIT1(0x41);
698                                 EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8);
699 
700                                 /* Emit 'movzwl eax, ax' */
701                                 if (is_ereg(dst_reg))
702                                         EMIT3(0x45, 0x0F, 0xB7);
703                                 else
704                                         EMIT2(0x0F, 0xB7);
705                                 EMIT1(add_2reg(0xC0, dst_reg, dst_reg));
706                                 break;
707                         case 32:
708                                 /* Emit 'bswap eax' to swap lower 4 bytes */
709                                 if (is_ereg(dst_reg))
710                                         EMIT2(0x41, 0x0F);
711                                 else
712                                         EMIT1(0x0F);
713                                 EMIT1(add_1reg(0xC8, dst_reg));
714                                 break;
715                         case 64:
716                                 /* Emit 'bswap rax' to swap 8 bytes */
717                                 EMIT3(add_1mod(0x48, dst_reg), 0x0F,
718                                       add_1reg(0xC8, dst_reg));
719                                 break;
720                         }
721                         break;
722 
723                 case BPF_ALU | BPF_END | BPF_FROM_LE:
724                         switch (imm32) {
725                         case 16:
726                                 /*
727                                  * Emit 'movzwl eax, ax' to zero extend 16-bit
728                                  * into 64 bit
729                                  */
730                                 if (is_ereg(dst_reg))
731                                         EMIT3(0x45, 0x0F, 0xB7);
732                                 else
733                                         EMIT2(0x0F, 0xB7);
734                                 EMIT1(add_2reg(0xC0, dst_reg, dst_reg));
735                                 break;
736                         case 32:
737                                 /* Emit 'mov eax, eax' to clear upper 32-bits */
738                                 if (is_ereg(dst_reg))
739                                         EMIT1(0x45);
740                                 EMIT2(0x89, add_2reg(0xC0, dst_reg, dst_reg));
741                                 break;
742                         case 64:
743                                 /* nop */
744                                 break;
745                         }
746                         break;
747 
748                         /* ST: *(u8*)(dst_reg + off) = imm */
749                 case BPF_ST | BPF_MEM | BPF_B:
750                         if (is_ereg(dst_reg))
751                                 EMIT2(0x41, 0xC6);
752                         else
753                                 EMIT1(0xC6);
754                         goto st;
755                 case BPF_ST | BPF_MEM | BPF_H:
756                         if (is_ereg(dst_reg))
757                                 EMIT3(0x66, 0x41, 0xC7);
758                         else
759                                 EMIT2(0x66, 0xC7);
760                         goto st;
761                 case BPF_ST | BPF_MEM | BPF_W:
762                         if (is_ereg(dst_reg))
763                                 EMIT2(0x41, 0xC7);
764                         else
765                                 EMIT1(0xC7);
766                         goto st;
767                 case BPF_ST | BPF_MEM | BPF_DW:
768                         EMIT2(add_1mod(0x48, dst_reg), 0xC7);
769 
770 st:                     if (is_imm8(insn->off))
771                                 EMIT2(add_1reg(0x40, dst_reg), insn->off);
772                         else
773                                 EMIT1_off32(add_1reg(0x80, dst_reg), insn->off);
774 
775                         EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code)));
776                         break;
777 
778                         /* STX: *(u8*)(dst_reg + off) = src_reg */
779                 case BPF_STX | BPF_MEM | BPF_B:
780                         /* Emit 'mov byte ptr [rax + off], al' */
781                         if (is_ereg(dst_reg) || is_ereg(src_reg) ||
782                             /* We have to add extra byte for x86 SIL, DIL regs */
783                             src_reg == BPF_REG_1 || src_reg == BPF_REG_2)
784                                 EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88);
785                         else
786                                 EMIT1(0x88);
787                         goto stx;
788                 case BPF_STX | BPF_MEM | BPF_H:
789                         if (is_ereg(dst_reg) || is_ereg(src_reg))
790                                 EMIT3(0x66, add_2mod(0x40, dst_reg, src_reg), 0x89);
791                         else
792                                 EMIT2(0x66, 0x89);
793                         goto stx;
794                 case BPF_STX | BPF_MEM | BPF_W:
795                         if (is_ereg(dst_reg) || is_ereg(src_reg))
796                                 EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x89);
797                         else
798                                 EMIT1(0x89);
799                         goto stx;
800                 case BPF_STX | BPF_MEM | BPF_DW:
801                         EMIT2(add_2mod(0x48, dst_reg, src_reg), 0x89);
802 stx:                    if (is_imm8(insn->off))
803                                 EMIT2(add_2reg(0x40, dst_reg, src_reg), insn->off);
804                         else
805                                 EMIT1_off32(add_2reg(0x80, dst_reg, src_reg),
806                                             insn->off);
807                         break;
808 
809                         /* LDX: dst_reg = *(u8*)(src_reg + off) */
810                 case BPF_LDX | BPF_MEM | BPF_B:
811                         /* Emit 'movzx rax, byte ptr [rax + off]' */
812                         EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6);
813                         goto ldx;
814                 case BPF_LDX | BPF_MEM | BPF_H:
815                         /* Emit 'movzx rax, word ptr [rax + off]' */
816                         EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7);
817                         goto ldx;
818                 case BPF_LDX | BPF_MEM | BPF_W:
819                         /* Emit 'mov eax, dword ptr [rax+0x14]' */
820                         if (is_ereg(dst_reg) || is_ereg(src_reg))
821                                 EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B);
822                         else
823                                 EMIT1(0x8B);
824                         goto ldx;
825                 case BPF_LDX | BPF_MEM | BPF_DW:
826                         /* Emit 'mov rax, qword ptr [rax+0x14]' */
827                         EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B);
828 ldx:                    /*
829                          * If insn->off == 0 we can save one extra byte, but
830                          * special case of x86 R13 which always needs an offset
831                          * is not worth the hassle
832                          */
833                         if (is_imm8(insn->off))
834                                 EMIT2(add_2reg(0x40, src_reg, dst_reg), insn->off);
835                         else
836                                 EMIT1_off32(add_2reg(0x80, src_reg, dst_reg),
837                                             insn->off);
838                         break;
839 
840                         /* STX XADD: lock *(u32*)(dst_reg + off) += src_reg */
841                 case BPF_STX | BPF_XADD | BPF_W:
842                         /* Emit 'lock add dword ptr [rax + off], eax' */
843                         if (is_ereg(dst_reg) || is_ereg(src_reg))
844                                 EMIT3(0xF0, add_2mod(0x40, dst_reg, src_reg), 0x01);
845                         else
846                                 EMIT2(0xF0, 0x01);
847                         goto xadd;
848                 case BPF_STX | BPF_XADD | BPF_DW:
849                         EMIT3(0xF0, add_2mod(0x48, dst_reg, src_reg), 0x01);
850 xadd:                   if (is_imm8(insn->off))
851                                 EMIT2(add_2reg(0x40, dst_reg, src_reg), insn->off);
852                         else
853                                 EMIT1_off32(add_2reg(0x80, dst_reg, src_reg),
854                                             insn->off);
855                         break;
856 
857                         /* call */
858                 case BPF_JMP | BPF_CALL:
859                         func = (u8 *) __bpf_call_base + imm32;
860                         jmp_offset = func - (image + addrs[i]);
861                         if (!imm32 || !is_simm32(jmp_offset)) {
862                                 pr_err("unsupported BPF func %d addr %p image %p\n",
863                                        imm32, func, image);
864                                 return -EINVAL;
865                         }
866                         EMIT1_off32(0xE8, jmp_offset);
867                         break;
868 
869                 case BPF_JMP | BPF_TAIL_CALL:
870                         emit_bpf_tail_call(&prog);
871                         break;
872 
873                         /* cond jump */
874                 case BPF_JMP | BPF_JEQ | BPF_X:
875                 case BPF_JMP | BPF_JNE | BPF_X:
876                 case BPF_JMP | BPF_JGT | BPF_X:
877                 case BPF_JMP | BPF_JLT | BPF_X:
878                 case BPF_JMP | BPF_JGE | BPF_X:
879                 case BPF_JMP | BPF_JLE | BPF_X:
880                 case BPF_JMP | BPF_JSGT | BPF_X:
881                 case BPF_JMP | BPF_JSLT | BPF_X:
882                 case BPF_JMP | BPF_JSGE | BPF_X:
883                 case BPF_JMP | BPF_JSLE | BPF_X:
884                 case BPF_JMP32 | BPF_JEQ | BPF_X:
885                 case BPF_JMP32 | BPF_JNE | BPF_X:
886                 case BPF_JMP32 | BPF_JGT | BPF_X:
887                 case BPF_JMP32 | BPF_JLT | BPF_X:
888                 case BPF_JMP32 | BPF_JGE | BPF_X:
889                 case BPF_JMP32 | BPF_JLE | BPF_X:
890                 case BPF_JMP32 | BPF_JSGT | BPF_X:
891                 case BPF_JMP32 | BPF_JSLT | BPF_X:
892                 case BPF_JMP32 | BPF_JSGE | BPF_X:
893                 case BPF_JMP32 | BPF_JSLE | BPF_X:
894                         /* cmp dst_reg, src_reg */
895                         if (BPF_CLASS(insn->code) == BPF_JMP)
896                                 EMIT1(add_2mod(0x48, dst_reg, src_reg));
897                         else if (is_ereg(dst_reg) || is_ereg(src_reg))
898                                 EMIT1(add_2mod(0x40, dst_reg, src_reg));
899                         EMIT2(0x39, add_2reg(0xC0, dst_reg, src_reg));
900                         goto emit_cond_jmp;
901 
902                 case BPF_JMP | BPF_JSET | BPF_X:
903                 case BPF_JMP32 | BPF_JSET | BPF_X:
904                         /* test dst_reg, src_reg */
905                         if (BPF_CLASS(insn->code) == BPF_JMP)
906                                 EMIT1(add_2mod(0x48, dst_reg, src_reg));
907                         else if (is_ereg(dst_reg) || is_ereg(src_reg))
908                                 EMIT1(add_2mod(0x40, dst_reg, src_reg));
909                         EMIT2(0x85, add_2reg(0xC0, dst_reg, src_reg));
910                         goto emit_cond_jmp;
911 
912                 case BPF_JMP | BPF_JSET | BPF_K:
913                 case BPF_JMP32 | BPF_JSET | BPF_K:
914                         /* test dst_reg, imm32 */
915                         if (BPF_CLASS(insn->code) == BPF_JMP)
916                                 EMIT1(add_1mod(0x48, dst_reg));
917                         else if (is_ereg(dst_reg))
918                                 EMIT1(add_1mod(0x40, dst_reg));
919                         EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32);
920                         goto emit_cond_jmp;
921 
922                 case BPF_JMP | BPF_JEQ | BPF_K:
923                 case BPF_JMP | BPF_JNE | BPF_K:
924                 case BPF_JMP | BPF_JGT | BPF_K:
925                 case BPF_JMP | BPF_JLT | BPF_K:
926                 case BPF_JMP | BPF_JGE | BPF_K:
927                 case BPF_JMP | BPF_JLE | BPF_K:
928                 case BPF_JMP | BPF_JSGT | BPF_K:
929                 case BPF_JMP | BPF_JSLT | BPF_K:
930                 case BPF_JMP | BPF_JSGE | BPF_K:
931                 case BPF_JMP | BPF_JSLE | BPF_K:
932                 case BPF_JMP32 | BPF_JEQ | BPF_K:
933                 case BPF_JMP32 | BPF_JNE | BPF_K:
934                 case BPF_JMP32 | BPF_JGT | BPF_K:
935                 case BPF_JMP32 | BPF_JLT | BPF_K:
936                 case BPF_JMP32 | BPF_JGE | BPF_K:
937                 case BPF_JMP32 | BPF_JLE | BPF_K:
938                 case BPF_JMP32 | BPF_JSGT | BPF_K:
939                 case BPF_JMP32 | BPF_JSLT | BPF_K:
940                 case BPF_JMP32 | BPF_JSGE | BPF_K:
941                 case BPF_JMP32 | BPF_JSLE | BPF_K:
942                         /* cmp dst_reg, imm8/32 */
943                         if (BPF_CLASS(insn->code) == BPF_JMP)
944                                 EMIT1(add_1mod(0x48, dst_reg));
945                         else if (is_ereg(dst_reg))
946                                 EMIT1(add_1mod(0x40, dst_reg));
947 
948                         if (is_imm8(imm32))
949                                 EMIT3(0x83, add_1reg(0xF8, dst_reg), imm32);
950                         else
951                                 EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32);
952 
953 emit_cond_jmp:          /* Convert BPF opcode to x86 */
954                         switch (BPF_OP(insn->code)) {
955                         case BPF_JEQ:
956                                 jmp_cond = X86_JE;
957                                 break;
958                         case BPF_JSET:
959                         case BPF_JNE:
960                                 jmp_cond = X86_JNE;
961                                 break;
962                         case BPF_JGT:
963                                 /* GT is unsigned '>', JA in x86 */
964                                 jmp_cond = X86_JA;
965                                 break;
966                         case BPF_JLT:
967                                 /* LT is unsigned '<', JB in x86 */
968                                 jmp_cond = X86_JB;
969                                 break;
970                         case BPF_JGE:
971                                 /* GE is unsigned '>=', JAE in x86 */
972                                 jmp_cond = X86_JAE;
973                                 break;
974                         case BPF_JLE:
975                                 /* LE is unsigned '<=', JBE in x86 */
976                                 jmp_cond = X86_JBE;
977                                 break;
978                         case BPF_JSGT:
979                                 /* Signed '>', GT in x86 */
980                                 jmp_cond = X86_JG;
981                                 break;
982                         case BPF_JSLT:
983                                 /* Signed '<', LT in x86 */
984                                 jmp_cond = X86_JL;
985                                 break;
986                         case BPF_JSGE:
987                                 /* Signed '>=', GE in x86 */
988                                 jmp_cond = X86_JGE;
989                                 break;
990                         case BPF_JSLE:
991                                 /* Signed '<=', LE in x86 */
992                                 jmp_cond = X86_JLE;
993                                 break;
994                         default: /* to silence GCC warning */
995                                 return -EFAULT;
996                         }
997                         jmp_offset = addrs[i + insn->off] - addrs[i];
998                         if (is_imm8(jmp_offset)) {
999                                 EMIT2(jmp_cond, jmp_offset);
1000                         } else if (is_simm32(jmp_offset)) {
1001                                 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset);
1002                         } else {
1003                                 pr_err("cond_jmp gen bug %llx\n", jmp_offset);
1004                                 return -EFAULT;
1005                         }
1006 
1007                         break;
1008 
1009                 case BPF_JMP | BPF_JA:
1010                         if (insn->off == -1)
1011                                 /* -1 jmp instructions will always jump
1012                                  * backwards two bytes. Explicitly handling
1013                                  * this case avoids wasting too many passes
1014                                  * when there are long sequences of replaced
1015                                  * dead code.
1016                                  */
1017                                 jmp_offset = -2;
1018                         else
1019                                 jmp_offset = addrs[i + insn->off] - addrs[i];
1020 
1021                         if (!jmp_offset)
1022                                 /* Optimize out nop jumps */
1023                                 break;
1024 emit_jmp:
1025                         if (is_imm8(jmp_offset)) {
1026                                 EMIT2(0xEB, jmp_offset);
1027                         } else if (is_simm32(jmp_offset)) {
1028                                 EMIT1_off32(0xE9, jmp_offset);
1029                         } else {
1030                                 pr_err("jmp gen bug %llx\n", jmp_offset);
1031                                 return -EFAULT;
1032                         }
1033                         break;
1034 
1035                 case BPF_JMP | BPF_EXIT:
1036                         if (seen_exit) {
1037                                 jmp_offset = ctx->cleanup_addr - addrs[i];
1038                                 goto emit_jmp;
1039                         }
1040                         seen_exit = true;
1041                         /* Update cleanup_addr */
1042                         ctx->cleanup_addr = proglen;
1043                         /* mov rbx, qword ptr [rbp+0] */
1044                         EMIT4(0x48, 0x8B, 0x5D, 0);
1045                         /* mov r13, qword ptr [rbp+8] */
1046                         EMIT4(0x4C, 0x8B, 0x6D, 8);
1047                         /* mov r14, qword ptr [rbp+16] */
1048                         EMIT4(0x4C, 0x8B, 0x75, 16);
1049                         /* mov r15, qword ptr [rbp+24] */
1050                         EMIT4(0x4C, 0x8B, 0x7D, 24);
1051 
1052                         /* add rbp, AUX_STACK_SPACE */
1053                         EMIT4(0x48, 0x83, 0xC5, AUX_STACK_SPACE);
1054                         EMIT1(0xC9); /* leave */
1055                         EMIT1(0xC3); /* ret */
1056                         break;
1057 
1058                 default:
1059                         /*
1060                          * By design x86-64 JIT should support all BPF instructions.
1061                          * This error will be seen if new instruction was added
1062                          * to the interpreter, but not to the JIT, or if there is
1063                          * junk in bpf_prog.
1064                          */
1065                         pr_err("bpf_jit: unknown opcode %02x\n", insn->code);
1066                         return -EINVAL;
1067                 }
1068 
1069                 ilen = prog - temp;
1070                 if (ilen > BPF_MAX_INSN_SIZE) {
1071                         pr_err("bpf_jit: fatal insn size error\n");
1072                         return -EFAULT;
1073                 }
1074 
1075                 if (image) {
1076                         if (unlikely(proglen + ilen > oldproglen)) {
1077                                 pr_err("bpf_jit: fatal error\n");
1078                                 return -EFAULT;
1079                         }
1080                         memcpy(image + proglen, temp, ilen);
1081                 }
1082                 proglen += ilen;
1083                 addrs[i] = proglen;
1084                 prog = temp;
1085         }
1086         return proglen;
1087 }
1088 
1089 struct x64_jit_data {
1090         struct bpf_binary_header *header;
1091         int *addrs;
1092         u8 *image;
1093         int proglen;
1094         struct jit_context ctx;
1095 };
1096 
1097 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1098 {
1099         struct bpf_binary_header *header = NULL;
1100         struct bpf_prog *tmp, *orig_prog = prog;
1101         struct x64_jit_data *jit_data;
1102         int proglen, oldproglen = 0;
1103         struct jit_context ctx = {};
1104         bool tmp_blinded = false;
1105         bool extra_pass = false;
1106         u8 *image = NULL;
1107         int *addrs;
1108         int pass;
1109         int i;
1110 
1111         if (!prog->jit_requested)
1112                 return orig_prog;
1113 
1114         tmp = bpf_jit_blind_constants(prog);
1115         /*
1116          * If blinding was requested and we failed during blinding,
1117          * we must fall back to the interpreter.
1118          */
1119         if (IS_ERR(tmp))
1120                 return orig_prog;
1121         if (tmp != prog) {
1122                 tmp_blinded = true;
1123                 prog = tmp;
1124         }
1125 
1126         jit_data = prog->aux->jit_data;
1127         if (!jit_data) {
1128                 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
1129                 if (!jit_data) {
1130                         prog = orig_prog;
1131                         goto out;
1132                 }
1133                 prog->aux->jit_data = jit_data;
1134         }
1135         addrs = jit_data->addrs;
1136         if (addrs) {
1137                 ctx = jit_data->ctx;
1138                 oldproglen = jit_data->proglen;
1139                 image = jit_data->image;
1140                 header = jit_data->header;
1141                 extra_pass = true;
1142                 goto skip_init_addrs;
1143         }
1144         addrs = kmalloc_array(prog->len, sizeof(*addrs), GFP_KERNEL);
1145         if (!addrs) {
1146                 prog = orig_prog;
1147                 goto out_addrs;
1148         }
1149 
1150         /*
1151          * Before first pass, make a rough estimation of addrs[]
1152          * each BPF instruction is translated to less than 64 bytes
1153          */
1154         for (proglen = 0, i = 0; i < prog->len; i++) {
1155                 proglen += 64;
1156                 addrs[i] = proglen;
1157         }
1158         ctx.cleanup_addr = proglen;
1159 skip_init_addrs:
1160 
1161         /*
1162          * JITed image shrinks with every pass and the loop iterates
1163          * until the image stops shrinking. Very large BPF programs
1164          * may converge on the last pass. In such case do one more
1165          * pass to emit the final image.
1166          */
1167         for (pass = 0; pass < 20 || image; pass++) {
1168                 proglen = do_jit(prog, addrs, image, oldproglen, &ctx);
1169                 if (proglen <= 0) {
1170 out_image:
1171                         image = NULL;
1172                         if (header)
1173                                 bpf_jit_binary_free(header);
1174                         prog = orig_prog;
1175                         goto out_addrs;
1176                 }
1177                 if (image) {
1178                         if (proglen != oldproglen) {
1179                                 pr_err("bpf_jit: proglen=%d != oldproglen=%d\n",
1180                                        proglen, oldproglen);
1181                                 goto out_image;
1182                         }
1183                         break;
1184                 }
1185                 if (proglen == oldproglen) {
1186                         header = bpf_jit_binary_alloc(proglen, &image,
1187                                                       1, jit_fill_hole);
1188                         if (!header) {
1189                                 prog = orig_prog;
1190                                 goto out_addrs;
1191                         }
1192                 }
1193                 oldproglen = proglen;
1194                 cond_resched();
1195         }
1196 
1197         if (bpf_jit_enable > 1)
1198                 bpf_jit_dump(prog->len, proglen, pass + 1, image);
1199 
1200         if (image) {
1201                 if (!prog->is_func || extra_pass) {
1202                         bpf_jit_binary_lock_ro(header);
1203                 } else {
1204                         jit_data->addrs = addrs;
1205                         jit_data->ctx = ctx;
1206                         jit_data->proglen = proglen;
1207                         jit_data->image = image;
1208                         jit_data->header = header;
1209                 }
1210                 prog->bpf_func = (void *)image;
1211                 prog->jited = 1;
1212                 prog->jited_len = proglen;
1213         } else {
1214                 prog = orig_prog;
1215         }
1216 
1217         if (!image || !prog->is_func || extra_pass) {
1218                 if (image)
1219                         bpf_prog_fill_jited_linfo(prog, addrs);
1220 out_addrs:
1221                 kfree(addrs);
1222                 kfree(jit_data);
1223                 prog->aux->jit_data = NULL;
1224         }
1225 out:
1226         if (tmp_blinded)
1227                 bpf_jit_prog_release_other(prog, prog == orig_prog ?
1228                                            tmp : orig_prog);
1229         return prog;
1230 }
1231 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp