~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/powerpc/net/bpf_jit_comp64.c

Version: ~ [ linux-4.18-rc1 ] ~ [ linux-4.17.2 ] ~ [ linux-4.16.16 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.50 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.109 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.138 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.113 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.57 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.27.62 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * bpf_jit_comp64.c: eBPF JIT compiler
  3  *
  4  * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
  5  *                IBM Corporation
  6  *
  7  * Based on the powerpc classic BPF JIT compiler by Matt Evans
  8  *
  9  * This program is free software; you can redistribute it and/or
 10  * modify it under the terms of the GNU General Public License
 11  * as published by the Free Software Foundation; version 2
 12  * of the License.
 13  */
 14 #include <linux/moduleloader.h>
 15 #include <asm/cacheflush.h>
 16 #include <linux/netdevice.h>
 17 #include <linux/filter.h>
 18 #include <linux/if_vlan.h>
 19 #include <asm/kprobes.h>
 20 #include <linux/bpf.h>
 21 
 22 #include "bpf_jit64.h"
 23 
 24 static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
 25 {
 26         memset32(area, BREAKPOINT_INSTRUCTION, size/4);
 27 }
 28 
 29 static inline void bpf_flush_icache(void *start, void *end)
 30 {
 31         smp_wmb();
 32         flush_icache_range((unsigned long)start, (unsigned long)end);
 33 }
 34 
 35 static inline bool bpf_is_seen_register(struct codegen_context *ctx, int i)
 36 {
 37         return (ctx->seen & (1 << (31 - b2p[i])));
 38 }
 39 
 40 static inline void bpf_set_seen_register(struct codegen_context *ctx, int i)
 41 {
 42         ctx->seen |= (1 << (31 - b2p[i]));
 43 }
 44 
 45 static inline bool bpf_has_stack_frame(struct codegen_context *ctx)
 46 {
 47         /*
 48          * We only need a stack frame if:
 49          * - we call other functions (kernel helpers), or
 50          * - the bpf program uses its stack area
 51          * The latter condition is deduced from the usage of BPF_REG_FP
 52          */
 53         return ctx->seen & SEEN_FUNC || bpf_is_seen_register(ctx, BPF_REG_FP);
 54 }
 55 
 56 /*
 57  * When not setting up our own stackframe, the redzone usage is:
 58  *
 59  *              [       prev sp         ] <-------------
 60  *              [         ...           ]               |
 61  * sp (r1) ---> [    stack pointer      ] --------------
 62  *              [   nv gpr save area    ] 6*8
 63  *              [    tail_call_cnt      ] 8
 64  *              [    local_tmp_var      ] 8
 65  *              [   unused red zone     ] 208 bytes protected
 66  */
 67 static int bpf_jit_stack_local(struct codegen_context *ctx)
 68 {
 69         if (bpf_has_stack_frame(ctx))
 70                 return STACK_FRAME_MIN_SIZE + ctx->stack_size;
 71         else
 72                 return -(BPF_PPC_STACK_SAVE + 16);
 73 }
 74 
 75 static int bpf_jit_stack_tailcallcnt(struct codegen_context *ctx)
 76 {
 77         return bpf_jit_stack_local(ctx) + 8;
 78 }
 79 
 80 static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
 81 {
 82         if (reg >= BPF_PPC_NVR_MIN && reg < 32)
 83                 return (bpf_has_stack_frame(ctx) ?
 84                         (BPF_PPC_STACKFRAME + ctx->stack_size) : 0)
 85                                 - (8 * (32 - reg));
 86 
 87         pr_err("BPF JIT is asking about unknown registers");
 88         BUG();
 89 }
 90 
 91 static void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
 92 {
 93         int i;
 94 
 95         /*
 96          * Initialize tail_call_cnt if we do tail calls.
 97          * Otherwise, put in NOPs so that it can be skipped when we are
 98          * invoked through a tail call.
 99          */
100         if (ctx->seen & SEEN_TAILCALL) {
101                 PPC_LI(b2p[TMP_REG_1], 0);
102                 /* this goes in the redzone */
103                 PPC_BPF_STL(b2p[TMP_REG_1], 1, -(BPF_PPC_STACK_SAVE + 8));
104         } else {
105                 PPC_NOP();
106                 PPC_NOP();
107         }
108 
109 #define BPF_TAILCALL_PROLOGUE_SIZE      8
110 
111         if (bpf_has_stack_frame(ctx)) {
112                 /*
113                  * We need a stack frame, but we don't necessarily need to
114                  * save/restore LR unless we call other functions
115                  */
116                 if (ctx->seen & SEEN_FUNC) {
117                         EMIT(PPC_INST_MFLR | __PPC_RT(R0));
118                         PPC_BPF_STL(0, 1, PPC_LR_STKOFF);
119                 }
120 
121                 PPC_BPF_STLU(1, 1, -(BPF_PPC_STACKFRAME + ctx->stack_size));
122         }
123 
124         /*
125          * Back up non-volatile regs -- BPF registers 6-10
126          * If we haven't created our own stack frame, we save these
127          * in the protected zone below the previous stack frame
128          */
129         for (i = BPF_REG_6; i <= BPF_REG_10; i++)
130                 if (bpf_is_seen_register(ctx, i))
131                         PPC_BPF_STL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));
132 
133         /* Setup frame pointer to point to the bpf stack area */
134         if (bpf_is_seen_register(ctx, BPF_REG_FP))
135                 PPC_ADDI(b2p[BPF_REG_FP], 1,
136                                 STACK_FRAME_MIN_SIZE + ctx->stack_size);
137 }
138 
139 static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx)
140 {
141         int i;
142 
143         /* Restore NVRs */
144         for (i = BPF_REG_6; i <= BPF_REG_10; i++)
145                 if (bpf_is_seen_register(ctx, i))
146                         PPC_BPF_LL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));
147 
148         /* Tear down our stack frame */
149         if (bpf_has_stack_frame(ctx)) {
150                 PPC_ADDI(1, 1, BPF_PPC_STACKFRAME + ctx->stack_size);
151                 if (ctx->seen & SEEN_FUNC) {
152                         PPC_BPF_LL(0, 1, PPC_LR_STKOFF);
153                         PPC_MTLR(0);
154                 }
155         }
156 }
157 
158 static void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
159 {
160         bpf_jit_emit_common_epilogue(image, ctx);
161 
162         /* Move result to r3 */
163         PPC_MR(3, b2p[BPF_REG_0]);
164 
165         PPC_BLR();
166 }
167 
168 static void bpf_jit_emit_func_call(u32 *image, struct codegen_context *ctx, u64 func)
169 {
170         unsigned int i, ctx_idx = ctx->idx;
171 
172         /* Load function address into r12 */
173         PPC_LI64(12, func);
174 
175         /* For bpf-to-bpf function calls, the callee's address is unknown
176          * until the last extra pass. As seen above, we use PPC_LI64() to
177          * load the callee's address, but this may optimize the number of
178          * instructions required based on the nature of the address.
179          *
180          * Since we don't want the number of instructions emitted to change,
181          * we pad the optimized PPC_LI64() call with NOPs to guarantee that
182          * we always have a five-instruction sequence, which is the maximum
183          * that PPC_LI64() can emit.
184          */
185         for (i = ctx->idx - ctx_idx; i < 5; i++)
186                 PPC_NOP();
187 
188 #ifdef PPC64_ELF_ABI_v1
189         /*
190          * Load TOC from function descriptor at offset 8.
191          * We can clobber r2 since we get called through a
192          * function pointer (so caller will save/restore r2)
193          * and since we don't use a TOC ourself.
194          */
195         PPC_BPF_LL(2, 12, 8);
196         /* Load actual entry point from function descriptor */
197         PPC_BPF_LL(12, 12, 0);
198 #endif
199 
200         PPC_MTLR(12);
201         PPC_BLRL();
202 }
203 
204 static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out)
205 {
206         /*
207          * By now, the eBPF program has already setup parameters in r3, r4 and r5
208          * r3/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program
209          * r4/BPF_REG_2 - pointer to bpf_array
210          * r5/BPF_REG_3 - index in bpf_array
211          */
212         int b2p_bpf_array = b2p[BPF_REG_2];
213         int b2p_index = b2p[BPF_REG_3];
214 
215         /*
216          * if (index >= array->map.max_entries)
217          *   goto out;
218          */
219         PPC_LWZ(b2p[TMP_REG_1], b2p_bpf_array, offsetof(struct bpf_array, map.max_entries));
220         PPC_RLWINM(b2p_index, b2p_index, 0, 0, 31);
221         PPC_CMPLW(b2p_index, b2p[TMP_REG_1]);
222         PPC_BCC(COND_GE, out);
223 
224         /*
225          * if (tail_call_cnt > MAX_TAIL_CALL_CNT)
226          *   goto out;
227          */
228         PPC_LD(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx));
229         PPC_CMPLWI(b2p[TMP_REG_1], MAX_TAIL_CALL_CNT);
230         PPC_BCC(COND_GT, out);
231 
232         /*
233          * tail_call_cnt++;
234          */
235         PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], 1);
236         PPC_BPF_STL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx));
237 
238         /* prog = array->ptrs[index]; */
239         PPC_MULI(b2p[TMP_REG_1], b2p_index, 8);
240         PPC_ADD(b2p[TMP_REG_1], b2p[TMP_REG_1], b2p_bpf_array);
241         PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_array, ptrs));
242 
243         /*
244          * if (prog == NULL)
245          *   goto out;
246          */
247         PPC_CMPLDI(b2p[TMP_REG_1], 0);
248         PPC_BCC(COND_EQ, out);
249 
250         /* goto *(prog->bpf_func + prologue_size); */
251         PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_prog, bpf_func));
252 #ifdef PPC64_ELF_ABI_v1
253         /* skip past the function descriptor */
254         PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1],
255                         FUNCTION_DESCR_SIZE + BPF_TAILCALL_PROLOGUE_SIZE);
256 #else
257         PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], BPF_TAILCALL_PROLOGUE_SIZE);
258 #endif
259         PPC_MTCTR(b2p[TMP_REG_1]);
260 
261         /* tear down stack, restore NVRs, ... */
262         bpf_jit_emit_common_epilogue(image, ctx);
263 
264         PPC_BCTR();
265         /* out: */
266 }
267 
268 /* Assemble the body code between the prologue & epilogue */
269 static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
270                               struct codegen_context *ctx,
271                               u32 *addrs, bool extra_pass)
272 {
273         const struct bpf_insn *insn = fp->insnsi;
274         int flen = fp->len;
275         int i;
276 
277         /* Start of epilogue code - will only be valid 2nd pass onwards */
278         u32 exit_addr = addrs[flen];
279 
280         for (i = 0; i < flen; i++) {
281                 u32 code = insn[i].code;
282                 u32 dst_reg = b2p[insn[i].dst_reg];
283                 u32 src_reg = b2p[insn[i].src_reg];
284                 s16 off = insn[i].off;
285                 s32 imm = insn[i].imm;
286                 u64 imm64;
287                 u8 *func;
288                 u32 true_cond;
289 
290                 /*
291                  * addrs[] maps a BPF bytecode address into a real offset from
292                  * the start of the body code.
293                  */
294                 addrs[i] = ctx->idx * 4;
295 
296                 /*
297                  * As an optimization, we note down which non-volatile registers
298                  * are used so that we can only save/restore those in our
299                  * prologue and epilogue. We do this here regardless of whether
300                  * the actual BPF instruction uses src/dst registers or not
301                  * (for instance, BPF_CALL does not use them). The expectation
302                  * is that those instructions will have src_reg/dst_reg set to
303                  * 0. Even otherwise, we just lose some prologue/epilogue
304                  * optimization but everything else should work without
305                  * any issues.
306                  */
307                 if (dst_reg >= BPF_PPC_NVR_MIN && dst_reg < 32)
308                         bpf_set_seen_register(ctx, insn[i].dst_reg);
309                 if (src_reg >= BPF_PPC_NVR_MIN && src_reg < 32)
310                         bpf_set_seen_register(ctx, insn[i].src_reg);
311 
312                 switch (code) {
313                 /*
314                  * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG
315                  */
316                 case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */
317                 case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */
318                         PPC_ADD(dst_reg, dst_reg, src_reg);
319                         goto bpf_alu32_trunc;
320                 case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */
321                 case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */
322                         PPC_SUB(dst_reg, dst_reg, src_reg);
323                         goto bpf_alu32_trunc;
324                 case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */
325                 case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */
326                 case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */
327                 case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */
328                         if (BPF_OP(code) == BPF_SUB)
329                                 imm = -imm;
330                         if (imm) {
331                                 if (imm >= -32768 && imm < 32768)
332                                         PPC_ADDI(dst_reg, dst_reg, IMM_L(imm));
333                                 else {
334                                         PPC_LI32(b2p[TMP_REG_1], imm);
335                                         PPC_ADD(dst_reg, dst_reg, b2p[TMP_REG_1]);
336                                 }
337                         }
338                         goto bpf_alu32_trunc;
339                 case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */
340                 case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */
341                         if (BPF_CLASS(code) == BPF_ALU)
342                                 PPC_MULW(dst_reg, dst_reg, src_reg);
343                         else
344                                 PPC_MULD(dst_reg, dst_reg, src_reg);
345                         goto bpf_alu32_trunc;
346                 case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */
347                 case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */
348                         if (imm >= -32768 && imm < 32768)
349                                 PPC_MULI(dst_reg, dst_reg, IMM_L(imm));
350                         else {
351                                 PPC_LI32(b2p[TMP_REG_1], imm);
352                                 if (BPF_CLASS(code) == BPF_ALU)
353                                         PPC_MULW(dst_reg, dst_reg,
354                                                         b2p[TMP_REG_1]);
355                                 else
356                                         PPC_MULD(dst_reg, dst_reg,
357                                                         b2p[TMP_REG_1]);
358                         }
359                         goto bpf_alu32_trunc;
360                 case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */
361                 case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */
362                         if (BPF_OP(code) == BPF_MOD) {
363                                 PPC_DIVWU(b2p[TMP_REG_1], dst_reg, src_reg);
364                                 PPC_MULW(b2p[TMP_REG_1], src_reg,
365                                                 b2p[TMP_REG_1]);
366                                 PPC_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]);
367                         } else
368                                 PPC_DIVWU(dst_reg, dst_reg, src_reg);
369                         goto bpf_alu32_trunc;
370                 case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */
371                 case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */
372                         if (BPF_OP(code) == BPF_MOD) {
373                                 PPC_DIVD(b2p[TMP_REG_1], dst_reg, src_reg);
374                                 PPC_MULD(b2p[TMP_REG_1], src_reg,
375                                                 b2p[TMP_REG_1]);
376                                 PPC_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]);
377                         } else
378                                 PPC_DIVD(dst_reg, dst_reg, src_reg);
379                         break;
380                 case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */
381                 case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */
382                 case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */
383                 case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */
384                         if (imm == 0)
385                                 return -EINVAL;
386                         else if (imm == 1)
387                                 goto bpf_alu32_trunc;
388 
389                         PPC_LI32(b2p[TMP_REG_1], imm);
390                         switch (BPF_CLASS(code)) {
391                         case BPF_ALU:
392                                 if (BPF_OP(code) == BPF_MOD) {
393                                         PPC_DIVWU(b2p[TMP_REG_2], dst_reg,
394                                                         b2p[TMP_REG_1]);
395                                         PPC_MULW(b2p[TMP_REG_1],
396                                                         b2p[TMP_REG_1],
397                                                         b2p[TMP_REG_2]);
398                                         PPC_SUB(dst_reg, dst_reg,
399                                                         b2p[TMP_REG_1]);
400                                 } else
401                                         PPC_DIVWU(dst_reg, dst_reg,
402                                                         b2p[TMP_REG_1]);
403                                 break;
404                         case BPF_ALU64:
405                                 if (BPF_OP(code) == BPF_MOD) {
406                                         PPC_DIVD(b2p[TMP_REG_2], dst_reg,
407                                                         b2p[TMP_REG_1]);
408                                         PPC_MULD(b2p[TMP_REG_1],
409                                                         b2p[TMP_REG_1],
410                                                         b2p[TMP_REG_2]);
411                                         PPC_SUB(dst_reg, dst_reg,
412                                                         b2p[TMP_REG_1]);
413                                 } else
414                                         PPC_DIVD(dst_reg, dst_reg,
415                                                         b2p[TMP_REG_1]);
416                                 break;
417                         }
418                         goto bpf_alu32_trunc;
419                 case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */
420                 case BPF_ALU64 | BPF_NEG: /* dst = -dst */
421                         PPC_NEG(dst_reg, dst_reg);
422                         goto bpf_alu32_trunc;
423 
424                 /*
425                  * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH
426                  */
427                 case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */
428                 case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */
429                         PPC_AND(dst_reg, dst_reg, src_reg);
430                         goto bpf_alu32_trunc;
431                 case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */
432                 case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */
433                         if (!IMM_H(imm))
434                                 PPC_ANDI(dst_reg, dst_reg, IMM_L(imm));
435                         else {
436                                 /* Sign-extended */
437                                 PPC_LI32(b2p[TMP_REG_1], imm);
438                                 PPC_AND(dst_reg, dst_reg, b2p[TMP_REG_1]);
439                         }
440                         goto bpf_alu32_trunc;
441                 case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
442                 case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */
443                         PPC_OR(dst_reg, dst_reg, src_reg);
444                         goto bpf_alu32_trunc;
445                 case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */
446                 case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */
447                         if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
448                                 /* Sign-extended */
449                                 PPC_LI32(b2p[TMP_REG_1], imm);
450                                 PPC_OR(dst_reg, dst_reg, b2p[TMP_REG_1]);
451                         } else {
452                                 if (IMM_L(imm))
453                                         PPC_ORI(dst_reg, dst_reg, IMM_L(imm));
454                                 if (IMM_H(imm))
455                                         PPC_ORIS(dst_reg, dst_reg, IMM_H(imm));
456                         }
457                         goto bpf_alu32_trunc;
458                 case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */
459                 case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */
460                         PPC_XOR(dst_reg, dst_reg, src_reg);
461                         goto bpf_alu32_trunc;
462                 case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */
463                 case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */
464                         if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
465                                 /* Sign-extended */
466                                 PPC_LI32(b2p[TMP_REG_1], imm);
467                                 PPC_XOR(dst_reg, dst_reg, b2p[TMP_REG_1]);
468                         } else {
469                                 if (IMM_L(imm))
470                                         PPC_XORI(dst_reg, dst_reg, IMM_L(imm));
471                                 if (IMM_H(imm))
472                                         PPC_XORIS(dst_reg, dst_reg, IMM_H(imm));
473                         }
474                         goto bpf_alu32_trunc;
475                 case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */
476                         /* slw clears top 32 bits */
477                         PPC_SLW(dst_reg, dst_reg, src_reg);
478                         break;
479                 case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */
480                         PPC_SLD(dst_reg, dst_reg, src_reg);
481                         break;
482                 case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<== (u32) imm */
483                         /* with imm 0, we still need to clear top 32 bits */
484                         PPC_SLWI(dst_reg, dst_reg, imm);
485                         break;
486                 case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<== imm */
487                         if (imm != 0)
488                                 PPC_SLDI(dst_reg, dst_reg, imm);
489                         break;
490                 case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */
491                         PPC_SRW(dst_reg, dst_reg, src_reg);
492                         break;
493                 case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */
494                         PPC_SRD(dst_reg, dst_reg, src_reg);
495                         break;
496                 case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */
497                         PPC_SRWI(dst_reg, dst_reg, imm);
498                         break;
499                 case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */
500                         if (imm != 0)
501                                 PPC_SRDI(dst_reg, dst_reg, imm);
502                         break;
503                 case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */
504                         PPC_SRAD(dst_reg, dst_reg, src_reg);
505                         break;
506                 case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */
507                         if (imm != 0)
508                                 PPC_SRADI(dst_reg, dst_reg, imm);
509                         break;
510 
511                 /*
512                  * MOV
513                  */
514                 case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */
515                 case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
516                         PPC_MR(dst_reg, src_reg);
517                         goto bpf_alu32_trunc;
518                 case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */
519                 case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */
520                         PPC_LI32(dst_reg, imm);
521                         if (imm < 0)
522                                 goto bpf_alu32_trunc;
523                         break;
524 
525 bpf_alu32_trunc:
526                 /* Truncate to 32-bits */
527                 if (BPF_CLASS(code) == BPF_ALU)
528                         PPC_RLWINM(dst_reg, dst_reg, 0, 0, 31);
529                 break;
530 
531                 /*
532                  * BPF_FROM_BE/LE
533                  */
534                 case BPF_ALU | BPF_END | BPF_FROM_LE:
535                 case BPF_ALU | BPF_END | BPF_FROM_BE:
536 #ifdef __BIG_ENDIAN__
537                         if (BPF_SRC(code) == BPF_FROM_BE)
538                                 goto emit_clear;
539 #else /* !__BIG_ENDIAN__ */
540                         if (BPF_SRC(code) == BPF_FROM_LE)
541                                 goto emit_clear;
542 #endif
543                         switch (imm) {
544                         case 16:
545                                 /* Rotate 8 bits left & mask with 0x0000ff00 */
546                                 PPC_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 16, 23);
547                                 /* Rotate 8 bits right & insert LSB to reg */
548                                 PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 24, 31);
549                                 /* Move result back to dst_reg */
550                                 PPC_MR(dst_reg, b2p[TMP_REG_1]);
551                                 break;
552                         case 32:
553                                 /*
554                                  * Rotate word left by 8 bits:
555                                  * 2 bytes are already in their final position
556                                  * -- byte 2 and 4 (of bytes 1, 2, 3 and 4)
557                                  */
558                                 PPC_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 0, 31);
559                                 /* Rotate 24 bits and insert byte 1 */
560                                 PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 0, 7);
561                                 /* Rotate 24 bits and insert byte 3 */
562                                 PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 16, 23);
563                                 PPC_MR(dst_reg, b2p[TMP_REG_1]);
564                                 break;
565                         case 64:
566                                 /*
567                                  * Way easier and faster(?) to store the value
568                                  * into stack and then use ldbrx
569                                  *
570                                  * ctx->seen will be reliable in pass2, but
571                                  * the instructions generated will remain the
572                                  * same across all passes
573                                  */
574                                 PPC_STD(dst_reg, 1, bpf_jit_stack_local(ctx));
575                                 PPC_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx));
576                                 PPC_LDBRX(dst_reg, 0, b2p[TMP_REG_1]);
577                                 break;
578                         }
579                         break;
580 
581 emit_clear:
582                         switch (imm) {
583                         case 16:
584                                 /* zero-extend 16 bits into 64 bits */
585                                 PPC_RLDICL(dst_reg, dst_reg, 0, 48);
586                                 break;
587                         case 32:
588                                 /* zero-extend 32 bits into 64 bits */
589                                 PPC_RLDICL(dst_reg, dst_reg, 0, 32);
590                                 break;
591                         case 64:
592                                 /* nop */
593                                 break;
594                         }
595                         break;
596 
597                 /*
598                  * BPF_ST(X)
599                  */
600                 case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */
601                 case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
602                         if (BPF_CLASS(code) == BPF_ST) {
603                                 PPC_LI(b2p[TMP_REG_1], imm);
604                                 src_reg = b2p[TMP_REG_1];
605                         }
606                         PPC_STB(src_reg, dst_reg, off);
607                         break;
608                 case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
609                 case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
610                         if (BPF_CLASS(code) == BPF_ST) {
611                                 PPC_LI(b2p[TMP_REG_1], imm);
612                                 src_reg = b2p[TMP_REG_1];
613                         }
614                         PPC_STH(src_reg, dst_reg, off);
615                         break;
616                 case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
617                 case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
618                         if (BPF_CLASS(code) == BPF_ST) {
619                                 PPC_LI32(b2p[TMP_REG_1], imm);
620                                 src_reg = b2p[TMP_REG_1];
621                         }
622                         PPC_STW(src_reg, dst_reg, off);
623                         break;
624                 case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
625                 case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
626                         if (BPF_CLASS(code) == BPF_ST) {
627                                 PPC_LI32(b2p[TMP_REG_1], imm);
628                                 src_reg = b2p[TMP_REG_1];
629                         }
630                         PPC_STD(src_reg, dst_reg, off);
631                         break;
632 
633                 /*
634                  * BPF_STX XADD (atomic_add)
635                  */
636                 /* *(u32 *)(dst + off) += src */
637                 case BPF_STX | BPF_XADD | BPF_W:
638                         /* Get EA into TMP_REG_1 */
639                         PPC_ADDI(b2p[TMP_REG_1], dst_reg, off);
640                         /* error if EA is not word-aligned */
641                         PPC_ANDI(b2p[TMP_REG_2], b2p[TMP_REG_1], 0x03);
642                         PPC_BCC_SHORT(COND_EQ, (ctx->idx * 4) + 12);
643                         PPC_LI(b2p[BPF_REG_0], 0);
644                         PPC_JMP(exit_addr);
645                         /* load value from memory into TMP_REG_2 */
646                         PPC_BPF_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0);
647                         /* add value from src_reg into this */
648                         PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg);
649                         /* store result back */
650                         PPC_BPF_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]);
651                         /* we're done if this succeeded */
652                         PPC_BCC_SHORT(COND_EQ, (ctx->idx * 4) + (7*4));
653                         /* otherwise, let's try once more */
654                         PPC_BPF_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0);
655                         PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg);
656                         PPC_BPF_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]);
657                         /* exit if the store was not successful */
658                         PPC_LI(b2p[BPF_REG_0], 0);
659                         PPC_BCC(COND_NE, exit_addr);
660                         break;
661                 /* *(u64 *)(dst + off) += src */
662                 case BPF_STX | BPF_XADD | BPF_DW:
663                         PPC_ADDI(b2p[TMP_REG_1], dst_reg, off);
664                         /* error if EA is not doubleword-aligned */
665                         PPC_ANDI(b2p[TMP_REG_2], b2p[TMP_REG_1], 0x07);
666                         PPC_BCC_SHORT(COND_EQ, (ctx->idx * 4) + (3*4));
667                         PPC_LI(b2p[BPF_REG_0], 0);
668                         PPC_JMP(exit_addr);
669                         PPC_BPF_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0);
670                         PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg);
671                         PPC_BPF_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]);
672                         PPC_BCC_SHORT(COND_EQ, (ctx->idx * 4) + (7*4));
673                         PPC_BPF_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0);
674                         PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg);
675                         PPC_BPF_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]);
676                         PPC_LI(b2p[BPF_REG_0], 0);
677                         PPC_BCC(COND_NE, exit_addr);
678                         break;
679 
680                 /*
681                  * BPF_LDX
682                  */
683                 /* dst = *(u8 *)(ul) (src + off) */
684                 case BPF_LDX | BPF_MEM | BPF_B:
685                         PPC_LBZ(dst_reg, src_reg, off);
686                         break;
687                 /* dst = *(u16 *)(ul) (src + off) */
688                 case BPF_LDX | BPF_MEM | BPF_H:
689                         PPC_LHZ(dst_reg, src_reg, off);
690                         break;
691                 /* dst = *(u32 *)(ul) (src + off) */
692                 case BPF_LDX | BPF_MEM | BPF_W:
693                         PPC_LWZ(dst_reg, src_reg, off);
694                         break;
695                 /* dst = *(u64 *)(ul) (src + off) */
696                 case BPF_LDX | BPF_MEM | BPF_DW:
697                         PPC_LD(dst_reg, src_reg, off);
698                         break;
699 
700                 /*
701                  * Doubleword load
702                  * 16 byte instruction that uses two 'struct bpf_insn'
703                  */
704                 case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */
705                         imm64 = ((u64)(u32) insn[i].imm) |
706                                     (((u64)(u32) insn[i+1].imm) << 32);
707                         /* Adjust for two bpf instructions */
708                         addrs[++i] = ctx->idx * 4;
709                         PPC_LI64(dst_reg, imm64);
710                         break;
711 
712                 /*
713                  * Return/Exit
714                  */
715                 case BPF_JMP | BPF_EXIT:
716                         /*
717                          * If this isn't the very last instruction, branch to
718                          * the epilogue. If we _are_ the last instruction,
719                          * we'll just fall through to the epilogue.
720                          */
721                         if (i != flen - 1)
722                                 PPC_JMP(exit_addr);
723                         /* else fall through to the epilogue */
724                         break;
725 
726                 /*
727                  * Call kernel helper or bpf function
728                  */
729                 case BPF_JMP | BPF_CALL:
730                         ctx->seen |= SEEN_FUNC;
731 
732                         /* bpf function call */
733                         if (insn[i].src_reg == BPF_PSEUDO_CALL)
734                                 if (!extra_pass)
735                                         func = NULL;
736                                 else if (fp->aux->func && off < fp->aux->func_cnt)
737                                         /* use the subprog id from the off
738                                          * field to lookup the callee address
739                                          */
740                                         func = (u8 *) fp->aux->func[off]->bpf_func;
741                                 else
742                                         return -EINVAL;
743                         /* kernel helper call */
744                         else
745                                 func = (u8 *) __bpf_call_base + imm;
746 
747                         bpf_jit_emit_func_call(image, ctx, (u64)func);
748 
749                         /* move return value from r3 to BPF_REG_0 */
750                         PPC_MR(b2p[BPF_REG_0], 3);
751                         break;
752 
753                 /*
754                  * Jumps and branches
755                  */
756                 case BPF_JMP | BPF_JA:
757                         PPC_JMP(addrs[i + 1 + off]);
758                         break;
759 
760                 case BPF_JMP | BPF_JGT | BPF_K:
761                 case BPF_JMP | BPF_JGT | BPF_X:
762                 case BPF_JMP | BPF_JSGT | BPF_K:
763                 case BPF_JMP | BPF_JSGT | BPF_X:
764                         true_cond = COND_GT;
765                         goto cond_branch;
766                 case BPF_JMP | BPF_JLT | BPF_K:
767                 case BPF_JMP | BPF_JLT | BPF_X:
768                 case BPF_JMP | BPF_JSLT | BPF_K:
769                 case BPF_JMP | BPF_JSLT | BPF_X:
770                         true_cond = COND_LT;
771                         goto cond_branch;
772                 case BPF_JMP | BPF_JGE | BPF_K:
773                 case BPF_JMP | BPF_JGE | BPF_X:
774                 case BPF_JMP | BPF_JSGE | BPF_K:
775                 case BPF_JMP | BPF_JSGE | BPF_X:
776                         true_cond = COND_GE;
777                         goto cond_branch;
778                 case BPF_JMP | BPF_JLE | BPF_K:
779                 case BPF_JMP | BPF_JLE | BPF_X:
780                 case BPF_JMP | BPF_JSLE | BPF_K:
781                 case BPF_JMP | BPF_JSLE | BPF_X:
782                         true_cond = COND_LE;
783                         goto cond_branch;
784                 case BPF_JMP | BPF_JEQ | BPF_K:
785                 case BPF_JMP | BPF_JEQ | BPF_X:
786                         true_cond = COND_EQ;
787                         goto cond_branch;
788                 case BPF_JMP | BPF_JNE | BPF_K:
789                 case BPF_JMP | BPF_JNE | BPF_X:
790                         true_cond = COND_NE;
791                         goto cond_branch;
792                 case BPF_JMP | BPF_JSET | BPF_K:
793                 case BPF_JMP | BPF_JSET | BPF_X:
794                         true_cond = COND_NE;
795                         /* Fall through */
796 
797 cond_branch:
798                         switch (code) {
799                         case BPF_JMP | BPF_JGT | BPF_X:
800                         case BPF_JMP | BPF_JLT | BPF_X:
801                         case BPF_JMP | BPF_JGE | BPF_X:
802                         case BPF_JMP | BPF_JLE | BPF_X:
803                         case BPF_JMP | BPF_JEQ | BPF_X:
804                         case BPF_JMP | BPF_JNE | BPF_X:
805                                 /* unsigned comparison */
806                                 PPC_CMPLD(dst_reg, src_reg);
807                                 break;
808                         case BPF_JMP | BPF_JSGT | BPF_X:
809                         case BPF_JMP | BPF_JSLT | BPF_X:
810                         case BPF_JMP | BPF_JSGE | BPF_X:
811                         case BPF_JMP | BPF_JSLE | BPF_X:
812                                 /* signed comparison */
813                                 PPC_CMPD(dst_reg, src_reg);
814                                 break;
815                         case BPF_JMP | BPF_JSET | BPF_X:
816                                 PPC_AND_DOT(b2p[TMP_REG_1], dst_reg, src_reg);
817                                 break;
818                         case BPF_JMP | BPF_JNE | BPF_K:
819                         case BPF_JMP | BPF_JEQ | BPF_K:
820                         case BPF_JMP | BPF_JGT | BPF_K:
821                         case BPF_JMP | BPF_JLT | BPF_K:
822                         case BPF_JMP | BPF_JGE | BPF_K:
823                         case BPF_JMP | BPF_JLE | BPF_K:
824                                 /*
825                                  * Need sign-extended load, so only positive
826                                  * values can be used as imm in cmpldi
827                                  */
828                                 if (imm >= 0 && imm < 32768)
829                                         PPC_CMPLDI(dst_reg, imm);
830                                 else {
831                                         /* sign-extending load */
832                                         PPC_LI32(b2p[TMP_REG_1], imm);
833                                         /* ... but unsigned comparison */
834                                         PPC_CMPLD(dst_reg, b2p[TMP_REG_1]);
835                                 }
836                                 break;
837                         case BPF_JMP | BPF_JSGT | BPF_K:
838                         case BPF_JMP | BPF_JSLT | BPF_K:
839                         case BPF_JMP | BPF_JSGE | BPF_K:
840                         case BPF_JMP | BPF_JSLE | BPF_K:
841                                 /*
842                                  * signed comparison, so any 16-bit value
843                                  * can be used in cmpdi
844                                  */
845                                 if (imm >= -32768 && imm < 32768)
846                                         PPC_CMPDI(dst_reg, imm);
847                                 else {
848                                         PPC_LI32(b2p[TMP_REG_1], imm);
849                                         PPC_CMPD(dst_reg, b2p[TMP_REG_1]);
850                                 }
851                                 break;
852                         case BPF_JMP | BPF_JSET | BPF_K:
853                                 /* andi does not sign-extend the immediate */
854                                 if (imm >= 0 && imm < 32768)
855                                         /* PPC_ANDI is _only/always_ dot-form */
856                                         PPC_ANDI(b2p[TMP_REG_1], dst_reg, imm);
857                                 else {
858                                         PPC_LI32(b2p[TMP_REG_1], imm);
859                                         PPC_AND_DOT(b2p[TMP_REG_1], dst_reg,
860                                                     b2p[TMP_REG_1]);
861                                 }
862                                 break;
863                         }
864                         PPC_BCC(true_cond, addrs[i + 1 + off]);
865                         break;
866 
867                 /*
868                  * Tail call
869                  */
870                 case BPF_JMP | BPF_TAIL_CALL:
871                         ctx->seen |= SEEN_TAILCALL;
872                         bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]);
873                         break;
874 
875                 default:
876                         /*
877                          * The filter contains something cruel & unusual.
878                          * We don't handle it, but also there shouldn't be
879                          * anything missing from our list.
880                          */
881                         pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n",
882                                         code, i);
883                         return -ENOTSUPP;
884                 }
885         }
886 
887         /* Set end-of-body-code address for exit. */
888         addrs[i] = ctx->idx * 4;
889 
890         return 0;
891 }
892 
893 struct powerpc64_jit_data {
894         struct bpf_binary_header *header;
895         u32 *addrs;
896         u8 *image;
897         u32 proglen;
898         struct codegen_context ctx;
899 };
900 
901 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
902 {
903         u32 proglen;
904         u32 alloclen;
905         u8 *image = NULL;
906         u32 *code_base;
907         u32 *addrs;
908         struct powerpc64_jit_data *jit_data;
909         struct codegen_context cgctx;
910         int pass;
911         int flen;
912         struct bpf_binary_header *bpf_hdr;
913         struct bpf_prog *org_fp = fp;
914         struct bpf_prog *tmp_fp;
915         bool bpf_blinded = false;
916         bool extra_pass = false;
917 
918         if (!fp->jit_requested)
919                 return org_fp;
920 
921         tmp_fp = bpf_jit_blind_constants(org_fp);
922         if (IS_ERR(tmp_fp))
923                 return org_fp;
924 
925         if (tmp_fp != org_fp) {
926                 bpf_blinded = true;
927                 fp = tmp_fp;
928         }
929 
930         jit_data = fp->aux->jit_data;
931         if (!jit_data) {
932                 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
933                 if (!jit_data) {
934                         fp = org_fp;
935                         goto out;
936                 }
937                 fp->aux->jit_data = jit_data;
938         }
939 
940         flen = fp->len;
941         addrs = jit_data->addrs;
942         if (addrs) {
943                 cgctx = jit_data->ctx;
944                 image = jit_data->image;
945                 bpf_hdr = jit_data->header;
946                 proglen = jit_data->proglen;
947                 alloclen = proglen + FUNCTION_DESCR_SIZE;
948                 extra_pass = true;
949                 goto skip_init_ctx;
950         }
951 
952         addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
953         if (addrs == NULL) {
954                 fp = org_fp;
955                 goto out_addrs;
956         }
957 
958         memset(&cgctx, 0, sizeof(struct codegen_context));
959 
960         /* Make sure that the stack is quadword aligned. */
961         cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
962 
963         /* Scouting faux-generate pass 0 */
964         if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) {
965                 /* We hit something illegal or unsupported. */
966                 fp = org_fp;
967                 goto out_addrs;
968         }
969 
970         /*
971          * Pretend to build prologue, given the features we've seen.  This will
972          * update ctgtx.idx as it pretends to output instructions, then we can
973          * calculate total size from idx.
974          */
975         bpf_jit_build_prologue(0, &cgctx);
976         bpf_jit_build_epilogue(0, &cgctx);
977 
978         proglen = cgctx.idx * 4;
979         alloclen = proglen + FUNCTION_DESCR_SIZE;
980 
981         bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4,
982                         bpf_jit_fill_ill_insns);
983         if (!bpf_hdr) {
984                 fp = org_fp;
985                 goto out_addrs;
986         }
987 
988 skip_init_ctx:
989         code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
990 
991         /* Code generation passes 1-2 */
992         for (pass = 1; pass < 3; pass++) {
993                 /* Now build the prologue, body code & epilogue for real. */
994                 cgctx.idx = 0;
995                 bpf_jit_build_prologue(code_base, &cgctx);
996                 bpf_jit_build_body(fp, code_base, &cgctx, addrs, extra_pass);
997                 bpf_jit_build_epilogue(code_base, &cgctx);
998 
999                 if (bpf_jit_enable > 1)
1000                         pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
1001                                 proglen - (cgctx.idx * 4), cgctx.seen);
1002         }
1003 
1004         if (bpf_jit_enable > 1)
1005                 /*
1006                  * Note that we output the base address of the code_base
1007                  * rather than image, since opcodes are in code_base.
1008                  */
1009                 bpf_jit_dump(flen, proglen, pass, code_base);
1010 
1011 #ifdef PPC64_ELF_ABI_v1
1012         /* Function descriptor nastiness: Address + TOC */
1013         ((u64 *)image)[0] = (u64)code_base;
1014         ((u64 *)image)[1] = local_paca->kernel_toc;
1015 #endif
1016 
1017         fp->bpf_func = (void *)image;
1018         fp->jited = 1;
1019         fp->jited_len = alloclen;
1020 
1021         bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + (bpf_hdr->pages * PAGE_SIZE));
1022         if (!fp->is_func || extra_pass) {
1023 out_addrs:
1024                 kfree(addrs);
1025                 kfree(jit_data);
1026                 fp->aux->jit_data = NULL;
1027         } else {
1028                 jit_data->addrs = addrs;
1029                 jit_data->ctx = cgctx;
1030                 jit_data->proglen = proglen;
1031                 jit_data->image = image;
1032                 jit_data->header = bpf_hdr;
1033         }
1034 
1035 out:
1036         if (bpf_blinded)
1037                 bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
1038 
1039         return fp;
1040 }
1041 
1042 /* Overriding bpf_jit_free() as we don't set images read-only. */
1043 void bpf_jit_free(struct bpf_prog *fp)
1044 {
1045         unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK;
1046         struct bpf_binary_header *bpf_hdr = (void *)addr;
1047 
1048         if (fp->jited)
1049                 bpf_jit_binary_free(bpf_hdr);
1050 
1051         bpf_prog_unlock_free(fp);
1052 }
1053 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp