~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/powerpc/net/bpf_jit_comp64.c

Version: ~ [ linux-5.3 ] ~ [ linux-5.2.15 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.73 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.144 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.193 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.193 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.73 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * bpf_jit_comp64.c: eBPF JIT compiler
  3  *
  4  * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
  5  *                IBM Corporation
  6  *
  7  * Based on the powerpc classic BPF JIT compiler by Matt Evans
  8  *
  9  * This program is free software; you can redistribute it and/or
 10  * modify it under the terms of the GNU General Public License
 11  * as published by the Free Software Foundation; version 2
 12  * of the License.
 13  */
 14 #include <linux/moduleloader.h>
 15 #include <asm/cacheflush.h>
 16 #include <linux/netdevice.h>
 17 #include <linux/filter.h>
 18 #include <linux/if_vlan.h>
 19 #include <asm/kprobes.h>
 20 #include <linux/bpf.h>
 21 
 22 #include "bpf_jit64.h"
 23 
 24 int bpf_jit_enable __read_mostly;
 25 
 26 static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
 27 {
 28         int *p = area;
 29 
 30         /* Fill whole space with trap instructions */
 31         while (p < (int *)((char *)area + size))
 32                 *p++ = BREAKPOINT_INSTRUCTION;
 33 }
 34 
 35 static inline void bpf_flush_icache(void *start, void *end)
 36 {
 37         smp_wmb();
 38         flush_icache_range((unsigned long)start, (unsigned long)end);
 39 }
 40 
 41 static inline bool bpf_is_seen_register(struct codegen_context *ctx, int i)
 42 {
 43         return (ctx->seen & (1 << (31 - b2p[i])));
 44 }
 45 
 46 static inline void bpf_set_seen_register(struct codegen_context *ctx, int i)
 47 {
 48         ctx->seen |= (1 << (31 - b2p[i]));
 49 }
 50 
 51 static inline bool bpf_has_stack_frame(struct codegen_context *ctx)
 52 {
 53         /*
 54          * We only need a stack frame if:
 55          * - we call other functions (kernel helpers), or
 56          * - the bpf program uses its stack area
 57          * The latter condition is deduced from the usage of BPF_REG_FP
 58          */
 59         return ctx->seen & SEEN_FUNC || bpf_is_seen_register(ctx, BPF_REG_FP);
 60 }
 61 
 62 /*
 63  * When not setting up our own stackframe, the redzone usage is:
 64  *
 65  *              [       prev sp         ] <-------------
 66  *              [         ...           ]               |
 67  * sp (r1) ---> [    stack pointer      ] --------------
 68  *              [   nv gpr save area    ] 8*8
 69  *              [    tail_call_cnt      ] 8
 70  *              [    local_tmp_var      ] 8
 71  *              [   unused red zone     ] 208 bytes protected
 72  */
 73 static int bpf_jit_stack_local(struct codegen_context *ctx)
 74 {
 75         if (bpf_has_stack_frame(ctx))
 76                 return STACK_FRAME_MIN_SIZE + MAX_BPF_STACK;
 77         else
 78                 return -(BPF_PPC_STACK_SAVE + 16);
 79 }
 80 
 81 static int bpf_jit_stack_tailcallcnt(struct codegen_context *ctx)
 82 {
 83         return bpf_jit_stack_local(ctx) + 8;
 84 }
 85 
 86 static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
 87 {
 88         if (reg >= BPF_PPC_NVR_MIN && reg < 32)
 89                 return (bpf_has_stack_frame(ctx) ? BPF_PPC_STACKFRAME : 0)
 90                                                         - (8 * (32 - reg));
 91 
 92         pr_err("BPF JIT is asking about unknown registers");
 93         BUG();
 94 }
 95 
 96 static void bpf_jit_emit_skb_loads(u32 *image, struct codegen_context *ctx)
 97 {
 98         /*
 99          * Load skb->len and skb->data_len
100          * r3 points to skb
101          */
102         PPC_LWZ(b2p[SKB_HLEN_REG], 3, offsetof(struct sk_buff, len));
103         PPC_LWZ(b2p[TMP_REG_1], 3, offsetof(struct sk_buff, data_len));
104         /* header_len = len - data_len */
105         PPC_SUB(b2p[SKB_HLEN_REG], b2p[SKB_HLEN_REG], b2p[TMP_REG_1]);
106 
107         /* skb->data pointer */
108         PPC_BPF_LL(b2p[SKB_DATA_REG], 3, offsetof(struct sk_buff, data));
109 }
110 
111 static void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
112 {
113         int i;
114 
115         /*
116          * Initialize tail_call_cnt if we do tail calls.
117          * Otherwise, put in NOPs so that it can be skipped when we are
118          * invoked through a tail call.
119          */
120         if (ctx->seen & SEEN_TAILCALL) {
121                 PPC_LI(b2p[TMP_REG_1], 0);
122                 /* this goes in the redzone */
123                 PPC_BPF_STL(b2p[TMP_REG_1], 1, -(BPF_PPC_STACK_SAVE + 8));
124         } else {
125                 PPC_NOP();
126                 PPC_NOP();
127         }
128 
129 #define BPF_TAILCALL_PROLOGUE_SIZE      8
130 
131         if (bpf_has_stack_frame(ctx)) {
132                 /*
133                  * We need a stack frame, but we don't necessarily need to
134                  * save/restore LR unless we call other functions
135                  */
136                 if (ctx->seen & SEEN_FUNC) {
137                         EMIT(PPC_INST_MFLR | __PPC_RT(R0));
138                         PPC_BPF_STL(0, 1, PPC_LR_STKOFF);
139                 }
140 
141                 PPC_BPF_STLU(1, 1, -BPF_PPC_STACKFRAME);
142         }
143 
144         /*
145          * Back up non-volatile regs -- BPF registers 6-10
146          * If we haven't created our own stack frame, we save these
147          * in the protected zone below the previous stack frame
148          */
149         for (i = BPF_REG_6; i <= BPF_REG_10; i++)
150                 if (bpf_is_seen_register(ctx, i))
151                         PPC_BPF_STL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));
152 
153         /*
154          * Save additional non-volatile regs if we cache skb
155          * Also, setup skb data
156          */
157         if (ctx->seen & SEEN_SKB) {
158                 PPC_BPF_STL(b2p[SKB_HLEN_REG], 1,
159                                 bpf_jit_stack_offsetof(ctx, b2p[SKB_HLEN_REG]));
160                 PPC_BPF_STL(b2p[SKB_DATA_REG], 1,
161                                 bpf_jit_stack_offsetof(ctx, b2p[SKB_DATA_REG]));
162                 bpf_jit_emit_skb_loads(image, ctx);
163         }
164 
165         /* Setup frame pointer to point to the bpf stack area */
166         if (bpf_is_seen_register(ctx, BPF_REG_FP))
167                 PPC_ADDI(b2p[BPF_REG_FP], 1,
168                                 STACK_FRAME_MIN_SIZE + MAX_BPF_STACK);
169 }
170 
171 static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx)
172 {
173         int i;
174 
175         /* Restore NVRs */
176         for (i = BPF_REG_6; i <= BPF_REG_10; i++)
177                 if (bpf_is_seen_register(ctx, i))
178                         PPC_BPF_LL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));
179 
180         /* Restore non-volatile registers used for skb cache */
181         if (ctx->seen & SEEN_SKB) {
182                 PPC_BPF_LL(b2p[SKB_HLEN_REG], 1,
183                                 bpf_jit_stack_offsetof(ctx, b2p[SKB_HLEN_REG]));
184                 PPC_BPF_LL(b2p[SKB_DATA_REG], 1,
185                                 bpf_jit_stack_offsetof(ctx, b2p[SKB_DATA_REG]));
186         }
187 
188         /* Tear down our stack frame */
189         if (bpf_has_stack_frame(ctx)) {
190                 PPC_ADDI(1, 1, BPF_PPC_STACKFRAME);
191                 if (ctx->seen & SEEN_FUNC) {
192                         PPC_BPF_LL(0, 1, PPC_LR_STKOFF);
193                         PPC_MTLR(0);
194                 }
195         }
196 }
197 
198 static void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
199 {
200         bpf_jit_emit_common_epilogue(image, ctx);
201 
202         /* Move result to r3 */
203         PPC_MR(3, b2p[BPF_REG_0]);
204 
205         PPC_BLR();
206 }
207 
208 static void bpf_jit_emit_func_call(u32 *image, struct codegen_context *ctx, u64 func)
209 {
210 #ifdef PPC64_ELF_ABI_v1
211         /* func points to the function descriptor */
212         PPC_LI64(b2p[TMP_REG_2], func);
213         /* Load actual entry point from function descriptor */
214         PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_2], 0);
215         /* ... and move it to LR */
216         PPC_MTLR(b2p[TMP_REG_1]);
217         /*
218          * Load TOC from function descriptor at offset 8.
219          * We can clobber r2 since we get called through a
220          * function pointer (so caller will save/restore r2)
221          * and since we don't use a TOC ourself.
222          */
223         PPC_BPF_LL(2, b2p[TMP_REG_2], 8);
224 #else
225         /* We can clobber r12 */
226         PPC_FUNC_ADDR(12, func);
227         PPC_MTLR(12);
228 #endif
229         PPC_BLRL();
230 }
231 
232 static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out)
233 {
234         /*
235          * By now, the eBPF program has already setup parameters in r3, r4 and r5
236          * r3/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program
237          * r4/BPF_REG_2 - pointer to bpf_array
238          * r5/BPF_REG_3 - index in bpf_array
239          */
240         int b2p_bpf_array = b2p[BPF_REG_2];
241         int b2p_index = b2p[BPF_REG_3];
242 
243         /*
244          * if (index >= array->map.max_entries)
245          *   goto out;
246          */
247         PPC_LWZ(b2p[TMP_REG_1], b2p_bpf_array, offsetof(struct bpf_array, map.max_entries));
248         PPC_CMPLW(b2p_index, b2p[TMP_REG_1]);
249         PPC_BCC(COND_GE, out);
250 
251         /*
252          * if (tail_call_cnt > MAX_TAIL_CALL_CNT)
253          *   goto out;
254          */
255         PPC_LD(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx));
256         PPC_CMPLWI(b2p[TMP_REG_1], MAX_TAIL_CALL_CNT);
257         PPC_BCC(COND_GT, out);
258 
259         /*
260          * tail_call_cnt++;
261          */
262         PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], 1);
263         PPC_BPF_STL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx));
264 
265         /* prog = array->ptrs[index]; */
266         PPC_MULI(b2p[TMP_REG_1], b2p_index, 8);
267         PPC_ADD(b2p[TMP_REG_1], b2p[TMP_REG_1], b2p_bpf_array);
268         PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_array, ptrs));
269 
270         /*
271          * if (prog == NULL)
272          *   goto out;
273          */
274         PPC_CMPLDI(b2p[TMP_REG_1], 0);
275         PPC_BCC(COND_EQ, out);
276 
277         /* goto *(prog->bpf_func + prologue_size); */
278         PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_prog, bpf_func));
279 #ifdef PPC64_ELF_ABI_v1
280         /* skip past the function descriptor */
281         PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1],
282                         FUNCTION_DESCR_SIZE + BPF_TAILCALL_PROLOGUE_SIZE);
283 #else
284         PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], BPF_TAILCALL_PROLOGUE_SIZE);
285 #endif
286         PPC_MTCTR(b2p[TMP_REG_1]);
287 
288         /* tear down stack, restore NVRs, ... */
289         bpf_jit_emit_common_epilogue(image, ctx);
290 
291         PPC_BCTR();
292         /* out: */
293 }
294 
295 /* Assemble the body code between the prologue & epilogue */
296 static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
297                               struct codegen_context *ctx,
298                               u32 *addrs)
299 {
300         const struct bpf_insn *insn = fp->insnsi;
301         int flen = fp->len;
302         int i;
303 
304         /* Start of epilogue code - will only be valid 2nd pass onwards */
305         u32 exit_addr = addrs[flen];
306 
307         for (i = 0; i < flen; i++) {
308                 u32 code = insn[i].code;
309                 u32 dst_reg = b2p[insn[i].dst_reg];
310                 u32 src_reg = b2p[insn[i].src_reg];
311                 s16 off = insn[i].off;
312                 s32 imm = insn[i].imm;
313                 u64 imm64;
314                 u8 *func;
315                 u32 true_cond;
316 
317                 /*
318                  * addrs[] maps a BPF bytecode address into a real offset from
319                  * the start of the body code.
320                  */
321                 addrs[i] = ctx->idx * 4;
322 
323                 /*
324                  * As an optimization, we note down which non-volatile registers
325                  * are used so that we can only save/restore those in our
326                  * prologue and epilogue. We do this here regardless of whether
327                  * the actual BPF instruction uses src/dst registers or not
328                  * (for instance, BPF_CALL does not use them). The expectation
329                  * is that those instructions will have src_reg/dst_reg set to
330                  * 0. Even otherwise, we just lose some prologue/epilogue
331                  * optimization but everything else should work without
332                  * any issues.
333                  */
334                 if (dst_reg >= BPF_PPC_NVR_MIN && dst_reg < 32)
335                         bpf_set_seen_register(ctx, insn[i].dst_reg);
336                 if (src_reg >= BPF_PPC_NVR_MIN && src_reg < 32)
337                         bpf_set_seen_register(ctx, insn[i].src_reg);
338 
339                 switch (code) {
340                 /*
341                  * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG
342                  */
343                 case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */
344                 case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */
345                         PPC_ADD(dst_reg, dst_reg, src_reg);
346                         goto bpf_alu32_trunc;
347                 case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */
348                 case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */
349                         PPC_SUB(dst_reg, dst_reg, src_reg);
350                         goto bpf_alu32_trunc;
351                 case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */
352                 case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */
353                 case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */
354                 case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */
355                         if (BPF_OP(code) == BPF_SUB)
356                                 imm = -imm;
357                         if (imm) {
358                                 if (imm >= -32768 && imm < 32768)
359                                         PPC_ADDI(dst_reg, dst_reg, IMM_L(imm));
360                                 else {
361                                         PPC_LI32(b2p[TMP_REG_1], imm);
362                                         PPC_ADD(dst_reg, dst_reg, b2p[TMP_REG_1]);
363                                 }
364                         }
365                         goto bpf_alu32_trunc;
366                 case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */
367                 case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */
368                         if (BPF_CLASS(code) == BPF_ALU)
369                                 PPC_MULW(dst_reg, dst_reg, src_reg);
370                         else
371                                 PPC_MULD(dst_reg, dst_reg, src_reg);
372                         goto bpf_alu32_trunc;
373                 case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */
374                 case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */
375                         if (imm >= -32768 && imm < 32768)
376                                 PPC_MULI(dst_reg, dst_reg, IMM_L(imm));
377                         else {
378                                 PPC_LI32(b2p[TMP_REG_1], imm);
379                                 if (BPF_CLASS(code) == BPF_ALU)
380                                         PPC_MULW(dst_reg, dst_reg,
381                                                         b2p[TMP_REG_1]);
382                                 else
383                                         PPC_MULD(dst_reg, dst_reg,
384                                                         b2p[TMP_REG_1]);
385                         }
386                         goto bpf_alu32_trunc;
387                 case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */
388                 case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */
389                         PPC_CMPWI(src_reg, 0);
390                         PPC_BCC_SHORT(COND_NE, (ctx->idx * 4) + 12);
391                         PPC_LI(b2p[BPF_REG_0], 0);
392                         PPC_JMP(exit_addr);
393                         if (BPF_OP(code) == BPF_MOD) {
394                                 PPC_DIVWU(b2p[TMP_REG_1], dst_reg, src_reg);
395                                 PPC_MULW(b2p[TMP_REG_1], src_reg,
396                                                 b2p[TMP_REG_1]);
397                                 PPC_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]);
398                         } else
399                                 PPC_DIVWU(dst_reg, dst_reg, src_reg);
400                         goto bpf_alu32_trunc;
401                 case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */
402                 case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */
403                         PPC_CMPDI(src_reg, 0);
404                         PPC_BCC_SHORT(COND_NE, (ctx->idx * 4) + 12);
405                         PPC_LI(b2p[BPF_REG_0], 0);
406                         PPC_JMP(exit_addr);
407                         if (BPF_OP(code) == BPF_MOD) {
408                                 PPC_DIVD(b2p[TMP_REG_1], dst_reg, src_reg);
409                                 PPC_MULD(b2p[TMP_REG_1], src_reg,
410                                                 b2p[TMP_REG_1]);
411                                 PPC_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]);
412                         } else
413                                 PPC_DIVD(dst_reg, dst_reg, src_reg);
414                         break;
415                 case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */
416                 case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */
417                 case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */
418                 case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */
419                         if (imm == 0)
420                                 return -EINVAL;
421                         else if (imm == 1)
422                                 goto bpf_alu32_trunc;
423 
424                         PPC_LI32(b2p[TMP_REG_1], imm);
425                         switch (BPF_CLASS(code)) {
426                         case BPF_ALU:
427                                 if (BPF_OP(code) == BPF_MOD) {
428                                         PPC_DIVWU(b2p[TMP_REG_2], dst_reg,
429                                                         b2p[TMP_REG_1]);
430                                         PPC_MULW(b2p[TMP_REG_1],
431                                                         b2p[TMP_REG_1],
432                                                         b2p[TMP_REG_2]);
433                                         PPC_SUB(dst_reg, dst_reg,
434                                                         b2p[TMP_REG_1]);
435                                 } else
436                                         PPC_DIVWU(dst_reg, dst_reg,
437                                                         b2p[TMP_REG_1]);
438                                 break;
439                         case BPF_ALU64:
440                                 if (BPF_OP(code) == BPF_MOD) {
441                                         PPC_DIVD(b2p[TMP_REG_2], dst_reg,
442                                                         b2p[TMP_REG_1]);
443                                         PPC_MULD(b2p[TMP_REG_1],
444                                                         b2p[TMP_REG_1],
445                                                         b2p[TMP_REG_2]);
446                                         PPC_SUB(dst_reg, dst_reg,
447                                                         b2p[TMP_REG_1]);
448                                 } else
449                                         PPC_DIVD(dst_reg, dst_reg,
450                                                         b2p[TMP_REG_1]);
451                                 break;
452                         }
453                         goto bpf_alu32_trunc;
454                 case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */
455                 case BPF_ALU64 | BPF_NEG: /* dst = -dst */
456                         PPC_NEG(dst_reg, dst_reg);
457                         goto bpf_alu32_trunc;
458 
459                 /*
460                  * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH
461                  */
462                 case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */
463                 case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */
464                         PPC_AND(dst_reg, dst_reg, src_reg);
465                         goto bpf_alu32_trunc;
466                 case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */
467                 case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */
468                         if (!IMM_H(imm))
469                                 PPC_ANDI(dst_reg, dst_reg, IMM_L(imm));
470                         else {
471                                 /* Sign-extended */
472                                 PPC_LI32(b2p[TMP_REG_1], imm);
473                                 PPC_AND(dst_reg, dst_reg, b2p[TMP_REG_1]);
474                         }
475                         goto bpf_alu32_trunc;
476                 case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
477                 case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */
478                         PPC_OR(dst_reg, dst_reg, src_reg);
479                         goto bpf_alu32_trunc;
480                 case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */
481                 case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */
482                         if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
483                                 /* Sign-extended */
484                                 PPC_LI32(b2p[TMP_REG_1], imm);
485                                 PPC_OR(dst_reg, dst_reg, b2p[TMP_REG_1]);
486                         } else {
487                                 if (IMM_L(imm))
488                                         PPC_ORI(dst_reg, dst_reg, IMM_L(imm));
489                                 if (IMM_H(imm))
490                                         PPC_ORIS(dst_reg, dst_reg, IMM_H(imm));
491                         }
492                         goto bpf_alu32_trunc;
493                 case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */
494                 case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */
495                         PPC_XOR(dst_reg, dst_reg, src_reg);
496                         goto bpf_alu32_trunc;
497                 case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */
498                 case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */
499                         if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
500                                 /* Sign-extended */
501                                 PPC_LI32(b2p[TMP_REG_1], imm);
502                                 PPC_XOR(dst_reg, dst_reg, b2p[TMP_REG_1]);
503                         } else {
504                                 if (IMM_L(imm))
505                                         PPC_XORI(dst_reg, dst_reg, IMM_L(imm));
506                                 if (IMM_H(imm))
507                                         PPC_XORIS(dst_reg, dst_reg, IMM_H(imm));
508                         }
509                         goto bpf_alu32_trunc;
510                 case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */
511                         /* slw clears top 32 bits */
512                         PPC_SLW(dst_reg, dst_reg, src_reg);
513                         break;
514                 case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */
515                         PPC_SLD(dst_reg, dst_reg, src_reg);
516                         break;
517                 case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<== (u32) imm */
518                         /* with imm 0, we still need to clear top 32 bits */
519                         PPC_SLWI(dst_reg, dst_reg, imm);
520                         break;
521                 case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<== imm */
522                         if (imm != 0)
523                                 PPC_SLDI(dst_reg, dst_reg, imm);
524                         break;
525                 case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */
526                         PPC_SRW(dst_reg, dst_reg, src_reg);
527                         break;
528                 case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */
529                         PPC_SRD(dst_reg, dst_reg, src_reg);
530                         break;
531                 case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */
532                         PPC_SRWI(dst_reg, dst_reg, imm);
533                         break;
534                 case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */
535                         if (imm != 0)
536                                 PPC_SRDI(dst_reg, dst_reg, imm);
537                         break;
538                 case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */
539                         PPC_SRAD(dst_reg, dst_reg, src_reg);
540                         break;
541                 case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */
542                         if (imm != 0)
543                                 PPC_SRADI(dst_reg, dst_reg, imm);
544                         break;
545 
546                 /*
547                  * MOV
548                  */
549                 case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */
550                 case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
551                         PPC_MR(dst_reg, src_reg);
552                         goto bpf_alu32_trunc;
553                 case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */
554                 case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */
555                         PPC_LI32(dst_reg, imm);
556                         if (imm < 0)
557                                 goto bpf_alu32_trunc;
558                         break;
559 
560 bpf_alu32_trunc:
561                 /* Truncate to 32-bits */
562                 if (BPF_CLASS(code) == BPF_ALU)
563                         PPC_RLWINM(dst_reg, dst_reg, 0, 0, 31);
564                 break;
565 
566                 /*
567                  * BPF_FROM_BE/LE
568                  */
569                 case BPF_ALU | BPF_END | BPF_FROM_LE:
570                 case BPF_ALU | BPF_END | BPF_FROM_BE:
571 #ifdef __BIG_ENDIAN__
572                         if (BPF_SRC(code) == BPF_FROM_BE)
573                                 goto emit_clear;
574 #else /* !__BIG_ENDIAN__ */
575                         if (BPF_SRC(code) == BPF_FROM_LE)
576                                 goto emit_clear;
577 #endif
578                         switch (imm) {
579                         case 16:
580                                 /* Rotate 8 bits left & mask with 0x0000ff00 */
581                                 PPC_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 16, 23);
582                                 /* Rotate 8 bits right & insert LSB to reg */
583                                 PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 24, 31);
584                                 /* Move result back to dst_reg */
585                                 PPC_MR(dst_reg, b2p[TMP_REG_1]);
586                                 break;
587                         case 32:
588                                 /*
589                                  * Rotate word left by 8 bits:
590                                  * 2 bytes are already in their final position
591                                  * -- byte 2 and 4 (of bytes 1, 2, 3 and 4)
592                                  */
593                                 PPC_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 0, 31);
594                                 /* Rotate 24 bits and insert byte 1 */
595                                 PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 0, 7);
596                                 /* Rotate 24 bits and insert byte 3 */
597                                 PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 16, 23);
598                                 PPC_MR(dst_reg, b2p[TMP_REG_1]);
599                                 break;
600                         case 64:
601                                 /*
602                                  * Way easier and faster(?) to store the value
603                                  * into stack and then use ldbrx
604                                  *
605                                  * ctx->seen will be reliable in pass2, but
606                                  * the instructions generated will remain the
607                                  * same across all passes
608                                  */
609                                 PPC_STD(dst_reg, 1, bpf_jit_stack_local(ctx));
610                                 PPC_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx));
611                                 PPC_LDBRX(dst_reg, 0, b2p[TMP_REG_1]);
612                                 break;
613                         }
614                         break;
615 
616 emit_clear:
617                         switch (imm) {
618                         case 16:
619                                 /* zero-extend 16 bits into 64 bits */
620                                 PPC_RLDICL(dst_reg, dst_reg, 0, 48);
621                                 break;
622                         case 32:
623                                 /* zero-extend 32 bits into 64 bits */
624                                 PPC_RLDICL(dst_reg, dst_reg, 0, 32);
625                                 break;
626                         case 64:
627                                 /* nop */
628                                 break;
629                         }
630                         break;
631 
632                 /*
633                  * BPF_ST(X)
634                  */
635                 case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */
636                 case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
637                         if (BPF_CLASS(code) == BPF_ST) {
638                                 PPC_LI(b2p[TMP_REG_1], imm);
639                                 src_reg = b2p[TMP_REG_1];
640                         }
641                         PPC_STB(src_reg, dst_reg, off);
642                         break;
643                 case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
644                 case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
645                         if (BPF_CLASS(code) == BPF_ST) {
646                                 PPC_LI(b2p[TMP_REG_1], imm);
647                                 src_reg = b2p[TMP_REG_1];
648                         }
649                         PPC_STH(src_reg, dst_reg, off);
650                         break;
651                 case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
652                 case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
653                         if (BPF_CLASS(code) == BPF_ST) {
654                                 PPC_LI32(b2p[TMP_REG_1], imm);
655                                 src_reg = b2p[TMP_REG_1];
656                         }
657                         PPC_STW(src_reg, dst_reg, off);
658                         break;
659                 case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
660                 case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
661                         if (BPF_CLASS(code) == BPF_ST) {
662                                 PPC_LI32(b2p[TMP_REG_1], imm);
663                                 src_reg = b2p[TMP_REG_1];
664                         }
665                         PPC_STD(src_reg, dst_reg, off);
666                         break;
667 
668                 /*
669                  * BPF_STX XADD (atomic_add)
670                  */
671                 /* *(u32 *)(dst + off) += src */
672                 case BPF_STX | BPF_XADD | BPF_W:
673                         /* Get EA into TMP_REG_1 */
674                         PPC_ADDI(b2p[TMP_REG_1], dst_reg, off);
675                         /* error if EA is not word-aligned */
676                         PPC_ANDI(b2p[TMP_REG_2], b2p[TMP_REG_1], 0x03);
677                         PPC_BCC_SHORT(COND_EQ, (ctx->idx * 4) + 12);
678                         PPC_LI(b2p[BPF_REG_0], 0);
679                         PPC_JMP(exit_addr);
680                         /* load value from memory into TMP_REG_2 */
681                         PPC_BPF_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0);
682                         /* add value from src_reg into this */
683                         PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg);
684                         /* store result back */
685                         PPC_BPF_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]);
686                         /* we're done if this succeeded */
687                         PPC_BCC_SHORT(COND_EQ, (ctx->idx * 4) + (7*4));
688                         /* otherwise, let's try once more */
689                         PPC_BPF_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0);
690                         PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg);
691                         PPC_BPF_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]);
692                         /* exit if the store was not successful */
693                         PPC_LI(b2p[BPF_REG_0], 0);
694                         PPC_BCC(COND_NE, exit_addr);
695                         break;
696                 /* *(u64 *)(dst + off) += src */
697                 case BPF_STX | BPF_XADD | BPF_DW:
698                         PPC_ADDI(b2p[TMP_REG_1], dst_reg, off);
699                         /* error if EA is not doubleword-aligned */
700                         PPC_ANDI(b2p[TMP_REG_2], b2p[TMP_REG_1], 0x07);
701                         PPC_BCC_SHORT(COND_EQ, (ctx->idx * 4) + (3*4));
702                         PPC_LI(b2p[BPF_REG_0], 0);
703                         PPC_JMP(exit_addr);
704                         PPC_BPF_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0);
705                         PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg);
706                         PPC_BPF_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]);
707                         PPC_BCC_SHORT(COND_EQ, (ctx->idx * 4) + (7*4));
708                         PPC_BPF_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0);
709                         PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg);
710                         PPC_BPF_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]);
711                         PPC_LI(b2p[BPF_REG_0], 0);
712                         PPC_BCC(COND_NE, exit_addr);
713                         break;
714 
715                 /*
716                  * BPF_LDX
717                  */
718                 /* dst = *(u8 *)(ul) (src + off) */
719                 case BPF_LDX | BPF_MEM | BPF_B:
720                         PPC_LBZ(dst_reg, src_reg, off);
721                         break;
722                 /* dst = *(u16 *)(ul) (src + off) */
723                 case BPF_LDX | BPF_MEM | BPF_H:
724                         PPC_LHZ(dst_reg, src_reg, off);
725                         break;
726                 /* dst = *(u32 *)(ul) (src + off) */
727                 case BPF_LDX | BPF_MEM | BPF_W:
728                         PPC_LWZ(dst_reg, src_reg, off);
729                         break;
730                 /* dst = *(u64 *)(ul) (src + off) */
731                 case BPF_LDX | BPF_MEM | BPF_DW:
732                         PPC_LD(dst_reg, src_reg, off);
733                         break;
734 
735                 /*
736                  * Doubleword load
737                  * 16 byte instruction that uses two 'struct bpf_insn'
738                  */
739                 case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */
740                         imm64 = ((u64)(u32) insn[i].imm) |
741                                     (((u64)(u32) insn[i+1].imm) << 32);
742                         /* Adjust for two bpf instructions */
743                         addrs[++i] = ctx->idx * 4;
744                         PPC_LI64(dst_reg, imm64);
745                         break;
746 
747                 /*
748                  * Return/Exit
749                  */
750                 case BPF_JMP | BPF_EXIT:
751                         /*
752                          * If this isn't the very last instruction, branch to
753                          * the epilogue. If we _are_ the last instruction,
754                          * we'll just fall through to the epilogue.
755                          */
756                         if (i != flen - 1)
757                                 PPC_JMP(exit_addr);
758                         /* else fall through to the epilogue */
759                         break;
760 
761                 /*
762                  * Call kernel helper
763                  */
764                 case BPF_JMP | BPF_CALL:
765                         ctx->seen |= SEEN_FUNC;
766                         func = (u8 *) __bpf_call_base + imm;
767 
768                         /* Save skb pointer if we need to re-cache skb data */
769                         if (bpf_helper_changes_pkt_data(func))
770                                 PPC_BPF_STL(3, 1, bpf_jit_stack_local(ctx));
771 
772                         bpf_jit_emit_func_call(image, ctx, (u64)func);
773 
774                         /* move return value from r3 to BPF_REG_0 */
775                         PPC_MR(b2p[BPF_REG_0], 3);
776 
777                         /* refresh skb cache */
778                         if (bpf_helper_changes_pkt_data(func)) {
779                                 /* reload skb pointer to r3 */
780                                 PPC_BPF_LL(3, 1, bpf_jit_stack_local(ctx));
781                                 bpf_jit_emit_skb_loads(image, ctx);
782                         }
783                         break;
784 
785                 /*
786                  * Jumps and branches
787                  */
788                 case BPF_JMP | BPF_JA:
789                         PPC_JMP(addrs[i + 1 + off]);
790                         break;
791 
792                 case BPF_JMP | BPF_JGT | BPF_K:
793                 case BPF_JMP | BPF_JGT | BPF_X:
794                 case BPF_JMP | BPF_JSGT | BPF_K:
795                 case BPF_JMP | BPF_JSGT | BPF_X:
796                         true_cond = COND_GT;
797                         goto cond_branch;
798                 case BPF_JMP | BPF_JGE | BPF_K:
799                 case BPF_JMP | BPF_JGE | BPF_X:
800                 case BPF_JMP | BPF_JSGE | BPF_K:
801                 case BPF_JMP | BPF_JSGE | BPF_X:
802                         true_cond = COND_GE;
803                         goto cond_branch;
804                 case BPF_JMP | BPF_JEQ | BPF_K:
805                 case BPF_JMP | BPF_JEQ | BPF_X:
806                         true_cond = COND_EQ;
807                         goto cond_branch;
808                 case BPF_JMP | BPF_JNE | BPF_K:
809                 case BPF_JMP | BPF_JNE | BPF_X:
810                         true_cond = COND_NE;
811                         goto cond_branch;
812                 case BPF_JMP | BPF_JSET | BPF_K:
813                 case BPF_JMP | BPF_JSET | BPF_X:
814                         true_cond = COND_NE;
815                         /* Fall through */
816 
817 cond_branch:
818                         switch (code) {
819                         case BPF_JMP | BPF_JGT | BPF_X:
820                         case BPF_JMP | BPF_JGE | BPF_X:
821                         case BPF_JMP | BPF_JEQ | BPF_X:
822                         case BPF_JMP | BPF_JNE | BPF_X:
823                                 /* unsigned comparison */
824                                 PPC_CMPLD(dst_reg, src_reg);
825                                 break;
826                         case BPF_JMP | BPF_JSGT | BPF_X:
827                         case BPF_JMP | BPF_JSGE | BPF_X:
828                                 /* signed comparison */
829                                 PPC_CMPD(dst_reg, src_reg);
830                                 break;
831                         case BPF_JMP | BPF_JSET | BPF_X:
832                                 PPC_AND_DOT(b2p[TMP_REG_1], dst_reg, src_reg);
833                                 break;
834                         case BPF_JMP | BPF_JNE | BPF_K:
835                         case BPF_JMP | BPF_JEQ | BPF_K:
836                         case BPF_JMP | BPF_JGT | BPF_K:
837                         case BPF_JMP | BPF_JGE | BPF_K:
838                                 /*
839                                  * Need sign-extended load, so only positive
840                                  * values can be used as imm in cmpldi
841                                  */
842                                 if (imm >= 0 && imm < 32768)
843                                         PPC_CMPLDI(dst_reg, imm);
844                                 else {
845                                         /* sign-extending load */
846                                         PPC_LI32(b2p[TMP_REG_1], imm);
847                                         /* ... but unsigned comparison */
848                                         PPC_CMPLD(dst_reg, b2p[TMP_REG_1]);
849                                 }
850                                 break;
851                         case BPF_JMP | BPF_JSGT | BPF_K:
852                         case BPF_JMP | BPF_JSGE | BPF_K:
853                                 /*
854                                  * signed comparison, so any 16-bit value
855                                  * can be used in cmpdi
856                                  */
857                                 if (imm >= -32768 && imm < 32768)
858                                         PPC_CMPDI(dst_reg, imm);
859                                 else {
860                                         PPC_LI32(b2p[TMP_REG_1], imm);
861                                         PPC_CMPD(dst_reg, b2p[TMP_REG_1]);
862                                 }
863                                 break;
864                         case BPF_JMP | BPF_JSET | BPF_K:
865                                 /* andi does not sign-extend the immediate */
866                                 if (imm >= 0 && imm < 32768)
867                                         /* PPC_ANDI is _only/always_ dot-form */
868                                         PPC_ANDI(b2p[TMP_REG_1], dst_reg, imm);
869                                 else {
870                                         PPC_LI32(b2p[TMP_REG_1], imm);
871                                         PPC_AND_DOT(b2p[TMP_REG_1], dst_reg,
872                                                     b2p[TMP_REG_1]);
873                                 }
874                                 break;
875                         }
876                         PPC_BCC(true_cond, addrs[i + 1 + off]);
877                         break;
878 
879                 /*
880                  * Loads from packet header/data
881                  * Assume 32-bit input value in imm and X (src_reg)
882                  */
883 
884                 /* Absolute loads */
885                 case BPF_LD | BPF_W | BPF_ABS:
886                         func = (u8 *)CHOOSE_LOAD_FUNC(imm, sk_load_word);
887                         goto common_load_abs;
888                 case BPF_LD | BPF_H | BPF_ABS:
889                         func = (u8 *)CHOOSE_LOAD_FUNC(imm, sk_load_half);
890                         goto common_load_abs;
891                 case BPF_LD | BPF_B | BPF_ABS:
892                         func = (u8 *)CHOOSE_LOAD_FUNC(imm, sk_load_byte);
893 common_load_abs:
894                         /*
895                          * Load from [imm]
896                          * Load into r4, which can just be passed onto
897                          *  skb load helpers as the second parameter
898                          */
899                         PPC_LI32(4, imm);
900                         goto common_load;
901 
902                 /* Indirect loads */
903                 case BPF_LD | BPF_W | BPF_IND:
904                         func = (u8 *)sk_load_word;
905                         goto common_load_ind;
906                 case BPF_LD | BPF_H | BPF_IND:
907                         func = (u8 *)sk_load_half;
908                         goto common_load_ind;
909                 case BPF_LD | BPF_B | BPF_IND:
910                         func = (u8 *)sk_load_byte;
911 common_load_ind:
912                         /*
913                          * Load from [src_reg + imm]
914                          * Treat src_reg as a 32-bit value
915                          */
916                         PPC_EXTSW(4, src_reg);
917                         if (imm) {
918                                 if (imm >= -32768 && imm < 32768)
919                                         PPC_ADDI(4, 4, IMM_L(imm));
920                                 else {
921                                         PPC_LI32(b2p[TMP_REG_1], imm);
922                                         PPC_ADD(4, 4, b2p[TMP_REG_1]);
923                                 }
924                         }
925 
926 common_load:
927                         ctx->seen |= SEEN_SKB;
928                         ctx->seen |= SEEN_FUNC;
929                         bpf_jit_emit_func_call(image, ctx, (u64)func);
930 
931                         /*
932                          * Helper returns 'lt' condition on error, and an
933                          * appropriate return value in BPF_REG_0
934                          */
935                         PPC_BCC(COND_LT, exit_addr);
936                         break;
937 
938                 /*
939                  * Tail call
940                  */
941                 case BPF_JMP | BPF_CALL | BPF_X:
942                         ctx->seen |= SEEN_TAILCALL;
943                         bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]);
944                         break;
945 
946                 default:
947                         /*
948                          * The filter contains something cruel & unusual.
949                          * We don't handle it, but also there shouldn't be
950                          * anything missing from our list.
951                          */
952                         pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n",
953                                         code, i);
954                         return -ENOTSUPP;
955                 }
956         }
957 
958         /* Set end-of-body-code address for exit. */
959         addrs[i] = ctx->idx * 4;
960 
961         return 0;
962 }
963 
964 void bpf_jit_compile(struct bpf_prog *fp) { }
965 
966 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
967 {
968         u32 proglen;
969         u32 alloclen;
970         u8 *image = NULL;
971         u32 *code_base;
972         u32 *addrs;
973         struct codegen_context cgctx;
974         int pass;
975         int flen;
976         struct bpf_binary_header *bpf_hdr;
977         struct bpf_prog *org_fp = fp;
978         struct bpf_prog *tmp_fp;
979         bool bpf_blinded = false;
980 
981         if (!bpf_jit_enable)
982                 return org_fp;
983 
984         tmp_fp = bpf_jit_blind_constants(org_fp);
985         if (IS_ERR(tmp_fp))
986                 return org_fp;
987 
988         if (tmp_fp != org_fp) {
989                 bpf_blinded = true;
990                 fp = tmp_fp;
991         }
992 
993         flen = fp->len;
994         addrs = kzalloc((flen+1) * sizeof(*addrs), GFP_KERNEL);
995         if (addrs == NULL) {
996                 fp = org_fp;
997                 goto out;
998         }
999 
1000         memset(&cgctx, 0, sizeof(struct codegen_context));
1001 
1002         /* Scouting faux-generate pass 0 */
1003         if (bpf_jit_build_body(fp, 0, &cgctx, addrs)) {
1004                 /* We hit something illegal or unsupported. */
1005                 fp = org_fp;
1006                 goto out;
1007         }
1008 
1009         /*
1010          * Pretend to build prologue, given the features we've seen.  This will
1011          * update ctgtx.idx as it pretends to output instructions, then we can
1012          * calculate total size from idx.
1013          */
1014         bpf_jit_build_prologue(0, &cgctx);
1015         bpf_jit_build_epilogue(0, &cgctx);
1016 
1017         proglen = cgctx.idx * 4;
1018         alloclen = proglen + FUNCTION_DESCR_SIZE;
1019 
1020         bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4,
1021                         bpf_jit_fill_ill_insns);
1022         if (!bpf_hdr) {
1023                 fp = org_fp;
1024                 goto out;
1025         }
1026 
1027         code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
1028 
1029         /* Code generation passes 1-2 */
1030         for (pass = 1; pass < 3; pass++) {
1031                 /* Now build the prologue, body code & epilogue for real. */
1032                 cgctx.idx = 0;
1033                 bpf_jit_build_prologue(code_base, &cgctx);
1034                 bpf_jit_build_body(fp, code_base, &cgctx, addrs);
1035                 bpf_jit_build_epilogue(code_base, &cgctx);
1036 
1037                 if (bpf_jit_enable > 1)
1038                         pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
1039                                 proglen - (cgctx.idx * 4), cgctx.seen);
1040         }
1041 
1042         if (bpf_jit_enable > 1)
1043                 /*
1044                  * Note that we output the base address of the code_base
1045                  * rather than image, since opcodes are in code_base.
1046                  */
1047                 bpf_jit_dump(flen, proglen, pass, code_base);
1048 
1049         if (image) {
1050                 bpf_flush_icache(bpf_hdr, image + alloclen);
1051 #ifdef PPC64_ELF_ABI_v1
1052                 /* Function descriptor nastiness: Address + TOC */
1053                 ((u64 *)image)[0] = (u64)code_base;
1054                 ((u64 *)image)[1] = local_paca->kernel_toc;
1055 #endif
1056                 fp->bpf_func = (void *)image;
1057                 fp->jited = 1;
1058         }
1059 
1060 out:
1061         kfree(addrs);
1062 
1063         if (bpf_blinded)
1064                 bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
1065 
1066         return fp;
1067 }
1068 
1069 void bpf_jit_free(struct bpf_prog *fp)
1070 {
1071         unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK;
1072         struct bpf_binary_header *bpf_hdr = (void *)addr;
1073 
1074         if (fp->jited)
1075                 bpf_jit_binary_free(bpf_hdr);
1076 
1077         bpf_prog_unlock_free(fp);
1078 }
1079 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp