~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/powerpc/net/bpf_jit_comp64.c

Version: ~ [ linux-4.20-rc2 ] ~ [ linux-4.19.1 ] ~ [ linux-4.18.18 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.80 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.136 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.163 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.125 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.60 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.31.14 ] ~ [ linux-2.6.30.10 ] ~ [ linux-2.6.29.6 ] ~ [ linux-2.6.28.10 ] ~ [ linux-2.6.27.62 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * bpf_jit_comp64.c: eBPF JIT compiler
  3  *
  4  * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
  5  *                IBM Corporation
  6  *
  7  * Based on the powerpc classic BPF JIT compiler by Matt Evans
  8  *
  9  * This program is free software; you can redistribute it and/or
 10  * modify it under the terms of the GNU General Public License
 11  * as published by the Free Software Foundation; version 2
 12  * of the License.
 13  */
 14 #include <linux/moduleloader.h>
 15 #include <asm/cacheflush.h>
 16 #include <asm/asm-compat.h>
 17 #include <linux/netdevice.h>
 18 #include <linux/filter.h>
 19 #include <linux/if_vlan.h>
 20 #include <asm/kprobes.h>
 21 #include <linux/bpf.h>
 22 
 23 #include "bpf_jit64.h"
 24 
 25 static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
 26 {
 27         memset32(area, BREAKPOINT_INSTRUCTION, size/4);
 28 }
 29 
 30 static inline void bpf_flush_icache(void *start, void *end)
 31 {
 32         smp_wmb();
 33         flush_icache_range((unsigned long)start, (unsigned long)end);
 34 }
 35 
 36 static inline bool bpf_is_seen_register(struct codegen_context *ctx, int i)
 37 {
 38         return (ctx->seen & (1 << (31 - b2p[i])));
 39 }
 40 
 41 static inline void bpf_set_seen_register(struct codegen_context *ctx, int i)
 42 {
 43         ctx->seen |= (1 << (31 - b2p[i]));
 44 }
 45 
 46 static inline bool bpf_has_stack_frame(struct codegen_context *ctx)
 47 {
 48         /*
 49          * We only need a stack frame if:
 50          * - we call other functions (kernel helpers), or
 51          * - the bpf program uses its stack area
 52          * The latter condition is deduced from the usage of BPF_REG_FP
 53          */
 54         return ctx->seen & SEEN_FUNC || bpf_is_seen_register(ctx, BPF_REG_FP);
 55 }
 56 
 57 /*
 58  * When not setting up our own stackframe, the redzone usage is:
 59  *
 60  *              [       prev sp         ] <-------------
 61  *              [         ...           ]               |
 62  * sp (r1) ---> [    stack pointer      ] --------------
 63  *              [   nv gpr save area    ] 6*8
 64  *              [    tail_call_cnt      ] 8
 65  *              [    local_tmp_var      ] 8
 66  *              [   unused red zone     ] 208 bytes protected
 67  */
 68 static int bpf_jit_stack_local(struct codegen_context *ctx)
 69 {
 70         if (bpf_has_stack_frame(ctx))
 71                 return STACK_FRAME_MIN_SIZE + ctx->stack_size;
 72         else
 73                 return -(BPF_PPC_STACK_SAVE + 16);
 74 }
 75 
 76 static int bpf_jit_stack_tailcallcnt(struct codegen_context *ctx)
 77 {
 78         return bpf_jit_stack_local(ctx) + 8;
 79 }
 80 
 81 static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
 82 {
 83         if (reg >= BPF_PPC_NVR_MIN && reg < 32)
 84                 return (bpf_has_stack_frame(ctx) ?
 85                         (BPF_PPC_STACKFRAME + ctx->stack_size) : 0)
 86                                 - (8 * (32 - reg));
 87 
 88         pr_err("BPF JIT is asking about unknown registers");
 89         BUG();
 90 }
 91 
 92 static void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
 93 {
 94         int i;
 95 
 96         /*
 97          * Initialize tail_call_cnt if we do tail calls.
 98          * Otherwise, put in NOPs so that it can be skipped when we are
 99          * invoked through a tail call.
100          */
101         if (ctx->seen & SEEN_TAILCALL) {
102                 PPC_LI(b2p[TMP_REG_1], 0);
103                 /* this goes in the redzone */
104                 PPC_BPF_STL(b2p[TMP_REG_1], 1, -(BPF_PPC_STACK_SAVE + 8));
105         } else {
106                 PPC_NOP();
107                 PPC_NOP();
108         }
109 
110 #define BPF_TAILCALL_PROLOGUE_SIZE      8
111 
112         if (bpf_has_stack_frame(ctx)) {
113                 /*
114                  * We need a stack frame, but we don't necessarily need to
115                  * save/restore LR unless we call other functions
116                  */
117                 if (ctx->seen & SEEN_FUNC) {
118                         EMIT(PPC_INST_MFLR | __PPC_RT(R0));
119                         PPC_BPF_STL(0, 1, PPC_LR_STKOFF);
120                 }
121 
122                 PPC_BPF_STLU(1, 1, -(BPF_PPC_STACKFRAME + ctx->stack_size));
123         }
124 
125         /*
126          * Back up non-volatile regs -- BPF registers 6-10
127          * If we haven't created our own stack frame, we save these
128          * in the protected zone below the previous stack frame
129          */
130         for (i = BPF_REG_6; i <= BPF_REG_10; i++)
131                 if (bpf_is_seen_register(ctx, i))
132                         PPC_BPF_STL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));
133 
134         /* Setup frame pointer to point to the bpf stack area */
135         if (bpf_is_seen_register(ctx, BPF_REG_FP))
136                 PPC_ADDI(b2p[BPF_REG_FP], 1,
137                                 STACK_FRAME_MIN_SIZE + ctx->stack_size);
138 }
139 
140 static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx)
141 {
142         int i;
143 
144         /* Restore NVRs */
145         for (i = BPF_REG_6; i <= BPF_REG_10; i++)
146                 if (bpf_is_seen_register(ctx, i))
147                         PPC_BPF_LL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));
148 
149         /* Tear down our stack frame */
150         if (bpf_has_stack_frame(ctx)) {
151                 PPC_ADDI(1, 1, BPF_PPC_STACKFRAME + ctx->stack_size);
152                 if (ctx->seen & SEEN_FUNC) {
153                         PPC_BPF_LL(0, 1, PPC_LR_STKOFF);
154                         PPC_MTLR(0);
155                 }
156         }
157 }
158 
159 static void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
160 {
161         bpf_jit_emit_common_epilogue(image, ctx);
162 
163         /* Move result to r3 */
164         PPC_MR(3, b2p[BPF_REG_0]);
165 
166         PPC_BLR();
167 }
168 
169 static void bpf_jit_emit_func_call(u32 *image, struct codegen_context *ctx, u64 func)
170 {
171         unsigned int i, ctx_idx = ctx->idx;
172 
173         /* Load function address into r12 */
174         PPC_LI64(12, func);
175 
176         /* For bpf-to-bpf function calls, the callee's address is unknown
177          * until the last extra pass. As seen above, we use PPC_LI64() to
178          * load the callee's address, but this may optimize the number of
179          * instructions required based on the nature of the address.
180          *
181          * Since we don't want the number of instructions emitted to change,
182          * we pad the optimized PPC_LI64() call with NOPs to guarantee that
183          * we always have a five-instruction sequence, which is the maximum
184          * that PPC_LI64() can emit.
185          */
186         for (i = ctx->idx - ctx_idx; i < 5; i++)
187                 PPC_NOP();
188 
189 #ifdef PPC64_ELF_ABI_v1
190         /*
191          * Load TOC from function descriptor at offset 8.
192          * We can clobber r2 since we get called through a
193          * function pointer (so caller will save/restore r2)
194          * and since we don't use a TOC ourself.
195          */
196         PPC_BPF_LL(2, 12, 8);
197         /* Load actual entry point from function descriptor */
198         PPC_BPF_LL(12, 12, 0);
199 #endif
200 
201         PPC_MTLR(12);
202         PPC_BLRL();
203 }
204 
205 static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out)
206 {
207         /*
208          * By now, the eBPF program has already setup parameters in r3, r4 and r5
209          * r3/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program
210          * r4/BPF_REG_2 - pointer to bpf_array
211          * r5/BPF_REG_3 - index in bpf_array
212          */
213         int b2p_bpf_array = b2p[BPF_REG_2];
214         int b2p_index = b2p[BPF_REG_3];
215 
216         /*
217          * if (index >= array->map.max_entries)
218          *   goto out;
219          */
220         PPC_LWZ(b2p[TMP_REG_1], b2p_bpf_array, offsetof(struct bpf_array, map.max_entries));
221         PPC_RLWINM(b2p_index, b2p_index, 0, 0, 31);
222         PPC_CMPLW(b2p_index, b2p[TMP_REG_1]);
223         PPC_BCC(COND_GE, out);
224 
225         /*
226          * if (tail_call_cnt > MAX_TAIL_CALL_CNT)
227          *   goto out;
228          */
229         PPC_LD(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx));
230         PPC_CMPLWI(b2p[TMP_REG_1], MAX_TAIL_CALL_CNT);
231         PPC_BCC(COND_GT, out);
232 
233         /*
234          * tail_call_cnt++;
235          */
236         PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], 1);
237         PPC_BPF_STL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx));
238 
239         /* prog = array->ptrs[index]; */
240         PPC_MULI(b2p[TMP_REG_1], b2p_index, 8);
241         PPC_ADD(b2p[TMP_REG_1], b2p[TMP_REG_1], b2p_bpf_array);
242         PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_array, ptrs));
243 
244         /*
245          * if (prog == NULL)
246          *   goto out;
247          */
248         PPC_CMPLDI(b2p[TMP_REG_1], 0);
249         PPC_BCC(COND_EQ, out);
250 
251         /* goto *(prog->bpf_func + prologue_size); */
252         PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_prog, bpf_func));
253 #ifdef PPC64_ELF_ABI_v1
254         /* skip past the function descriptor */
255         PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1],
256                         FUNCTION_DESCR_SIZE + BPF_TAILCALL_PROLOGUE_SIZE);
257 #else
258         PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], BPF_TAILCALL_PROLOGUE_SIZE);
259 #endif
260         PPC_MTCTR(b2p[TMP_REG_1]);
261 
262         /* tear down stack, restore NVRs, ... */
263         bpf_jit_emit_common_epilogue(image, ctx);
264 
265         PPC_BCTR();
266         /* out: */
267 }
268 
269 /* Assemble the body code between the prologue & epilogue */
270 static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
271                               struct codegen_context *ctx,
272                               u32 *addrs, bool extra_pass)
273 {
274         const struct bpf_insn *insn = fp->insnsi;
275         int flen = fp->len;
276         int i;
277 
278         /* Start of epilogue code - will only be valid 2nd pass onwards */
279         u32 exit_addr = addrs[flen];
280 
281         for (i = 0; i < flen; i++) {
282                 u32 code = insn[i].code;
283                 u32 dst_reg = b2p[insn[i].dst_reg];
284                 u32 src_reg = b2p[insn[i].src_reg];
285                 s16 off = insn[i].off;
286                 s32 imm = insn[i].imm;
287                 u64 imm64;
288                 u8 *func;
289                 u32 true_cond;
290                 u32 tmp_idx;
291 
292                 /*
293                  * addrs[] maps a BPF bytecode address into a real offset from
294                  * the start of the body code.
295                  */
296                 addrs[i] = ctx->idx * 4;
297 
298                 /*
299                  * As an optimization, we note down which non-volatile registers
300                  * are used so that we can only save/restore those in our
301                  * prologue and epilogue. We do this here regardless of whether
302                  * the actual BPF instruction uses src/dst registers or not
303                  * (for instance, BPF_CALL does not use them). The expectation
304                  * is that those instructions will have src_reg/dst_reg set to
305                  * 0. Even otherwise, we just lose some prologue/epilogue
306                  * optimization but everything else should work without
307                  * any issues.
308                  */
309                 if (dst_reg >= BPF_PPC_NVR_MIN && dst_reg < 32)
310                         bpf_set_seen_register(ctx, insn[i].dst_reg);
311                 if (src_reg >= BPF_PPC_NVR_MIN && src_reg < 32)
312                         bpf_set_seen_register(ctx, insn[i].src_reg);
313 
314                 switch (code) {
315                 /*
316                  * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG
317                  */
318                 case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */
319                 case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */
320                         PPC_ADD(dst_reg, dst_reg, src_reg);
321                         goto bpf_alu32_trunc;
322                 case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */
323                 case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */
324                         PPC_SUB(dst_reg, dst_reg, src_reg);
325                         goto bpf_alu32_trunc;
326                 case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */
327                 case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */
328                 case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */
329                 case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */
330                         if (BPF_OP(code) == BPF_SUB)
331                                 imm = -imm;
332                         if (imm) {
333                                 if (imm >= -32768 && imm < 32768)
334                                         PPC_ADDI(dst_reg, dst_reg, IMM_L(imm));
335                                 else {
336                                         PPC_LI32(b2p[TMP_REG_1], imm);
337                                         PPC_ADD(dst_reg, dst_reg, b2p[TMP_REG_1]);
338                                 }
339                         }
340                         goto bpf_alu32_trunc;
341                 case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */
342                 case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */
343                         if (BPF_CLASS(code) == BPF_ALU)
344                                 PPC_MULW(dst_reg, dst_reg, src_reg);
345                         else
346                                 PPC_MULD(dst_reg, dst_reg, src_reg);
347                         goto bpf_alu32_trunc;
348                 case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */
349                 case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */
350                         if (imm >= -32768 && imm < 32768)
351                                 PPC_MULI(dst_reg, dst_reg, IMM_L(imm));
352                         else {
353                                 PPC_LI32(b2p[TMP_REG_1], imm);
354                                 if (BPF_CLASS(code) == BPF_ALU)
355                                         PPC_MULW(dst_reg, dst_reg,
356                                                         b2p[TMP_REG_1]);
357                                 else
358                                         PPC_MULD(dst_reg, dst_reg,
359                                                         b2p[TMP_REG_1]);
360                         }
361                         goto bpf_alu32_trunc;
362                 case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */
363                 case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */
364                         if (BPF_OP(code) == BPF_MOD) {
365                                 PPC_DIVWU(b2p[TMP_REG_1], dst_reg, src_reg);
366                                 PPC_MULW(b2p[TMP_REG_1], src_reg,
367                                                 b2p[TMP_REG_1]);
368                                 PPC_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]);
369                         } else
370                                 PPC_DIVWU(dst_reg, dst_reg, src_reg);
371                         goto bpf_alu32_trunc;
372                 case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */
373                 case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */
374                         if (BPF_OP(code) == BPF_MOD) {
375                                 PPC_DIVD(b2p[TMP_REG_1], dst_reg, src_reg);
376                                 PPC_MULD(b2p[TMP_REG_1], src_reg,
377                                                 b2p[TMP_REG_1]);
378                                 PPC_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]);
379                         } else
380                                 PPC_DIVD(dst_reg, dst_reg, src_reg);
381                         break;
382                 case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */
383                 case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */
384                 case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */
385                 case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */
386                         if (imm == 0)
387                                 return -EINVAL;
388                         else if (imm == 1)
389                                 goto bpf_alu32_trunc;
390 
391                         PPC_LI32(b2p[TMP_REG_1], imm);
392                         switch (BPF_CLASS(code)) {
393                         case BPF_ALU:
394                                 if (BPF_OP(code) == BPF_MOD) {
395                                         PPC_DIVWU(b2p[TMP_REG_2], dst_reg,
396                                                         b2p[TMP_REG_1]);
397                                         PPC_MULW(b2p[TMP_REG_1],
398                                                         b2p[TMP_REG_1],
399                                                         b2p[TMP_REG_2]);
400                                         PPC_SUB(dst_reg, dst_reg,
401                                                         b2p[TMP_REG_1]);
402                                 } else
403                                         PPC_DIVWU(dst_reg, dst_reg,
404                                                         b2p[TMP_REG_1]);
405                                 break;
406                         case BPF_ALU64:
407                                 if (BPF_OP(code) == BPF_MOD) {
408                                         PPC_DIVD(b2p[TMP_REG_2], dst_reg,
409                                                         b2p[TMP_REG_1]);
410                                         PPC_MULD(b2p[TMP_REG_1],
411                                                         b2p[TMP_REG_1],
412                                                         b2p[TMP_REG_2]);
413                                         PPC_SUB(dst_reg, dst_reg,
414                                                         b2p[TMP_REG_1]);
415                                 } else
416                                         PPC_DIVD(dst_reg, dst_reg,
417                                                         b2p[TMP_REG_1]);
418                                 break;
419                         }
420                         goto bpf_alu32_trunc;
421                 case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */
422                 case BPF_ALU64 | BPF_NEG: /* dst = -dst */
423                         PPC_NEG(dst_reg, dst_reg);
424                         goto bpf_alu32_trunc;
425 
426                 /*
427                  * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH
428                  */
429                 case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */
430                 case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */
431                         PPC_AND(dst_reg, dst_reg, src_reg);
432                         goto bpf_alu32_trunc;
433                 case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */
434                 case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */
435                         if (!IMM_H(imm))
436                                 PPC_ANDI(dst_reg, dst_reg, IMM_L(imm));
437                         else {
438                                 /* Sign-extended */
439                                 PPC_LI32(b2p[TMP_REG_1], imm);
440                                 PPC_AND(dst_reg, dst_reg, b2p[TMP_REG_1]);
441                         }
442                         goto bpf_alu32_trunc;
443                 case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
444                 case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */
445                         PPC_OR(dst_reg, dst_reg, src_reg);
446                         goto bpf_alu32_trunc;
447                 case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */
448                 case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */
449                         if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
450                                 /* Sign-extended */
451                                 PPC_LI32(b2p[TMP_REG_1], imm);
452                                 PPC_OR(dst_reg, dst_reg, b2p[TMP_REG_1]);
453                         } else {
454                                 if (IMM_L(imm))
455                                         PPC_ORI(dst_reg, dst_reg, IMM_L(imm));
456                                 if (IMM_H(imm))
457                                         PPC_ORIS(dst_reg, dst_reg, IMM_H(imm));
458                         }
459                         goto bpf_alu32_trunc;
460                 case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */
461                 case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */
462                         PPC_XOR(dst_reg, dst_reg, src_reg);
463                         goto bpf_alu32_trunc;
464                 case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */
465                 case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */
466                         if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
467                                 /* Sign-extended */
468                                 PPC_LI32(b2p[TMP_REG_1], imm);
469                                 PPC_XOR(dst_reg, dst_reg, b2p[TMP_REG_1]);
470                         } else {
471                                 if (IMM_L(imm))
472                                         PPC_XORI(dst_reg, dst_reg, IMM_L(imm));
473                                 if (IMM_H(imm))
474                                         PPC_XORIS(dst_reg, dst_reg, IMM_H(imm));
475                         }
476                         goto bpf_alu32_trunc;
477                 case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */
478                         /* slw clears top 32 bits */
479                         PPC_SLW(dst_reg, dst_reg, src_reg);
480                         break;
481                 case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */
482                         PPC_SLD(dst_reg, dst_reg, src_reg);
483                         break;
484                 case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<== (u32) imm */
485                         /* with imm 0, we still need to clear top 32 bits */
486                         PPC_SLWI(dst_reg, dst_reg, imm);
487                         break;
488                 case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<== imm */
489                         if (imm != 0)
490                                 PPC_SLDI(dst_reg, dst_reg, imm);
491                         break;
492                 case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */
493                         PPC_SRW(dst_reg, dst_reg, src_reg);
494                         break;
495                 case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */
496                         PPC_SRD(dst_reg, dst_reg, src_reg);
497                         break;
498                 case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */
499                         PPC_SRWI(dst_reg, dst_reg, imm);
500                         break;
501                 case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */
502                         if (imm != 0)
503                                 PPC_SRDI(dst_reg, dst_reg, imm);
504                         break;
505                 case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */
506                         PPC_SRAD(dst_reg, dst_reg, src_reg);
507                         break;
508                 case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */
509                         if (imm != 0)
510                                 PPC_SRADI(dst_reg, dst_reg, imm);
511                         break;
512 
513                 /*
514                  * MOV
515                  */
516                 case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */
517                 case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
518                         PPC_MR(dst_reg, src_reg);
519                         goto bpf_alu32_trunc;
520                 case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */
521                 case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */
522                         PPC_LI32(dst_reg, imm);
523                         if (imm < 0)
524                                 goto bpf_alu32_trunc;
525                         break;
526 
527 bpf_alu32_trunc:
528                 /* Truncate to 32-bits */
529                 if (BPF_CLASS(code) == BPF_ALU)
530                         PPC_RLWINM(dst_reg, dst_reg, 0, 0, 31);
531                 break;
532 
533                 /*
534                  * BPF_FROM_BE/LE
535                  */
536                 case BPF_ALU | BPF_END | BPF_FROM_LE:
537                 case BPF_ALU | BPF_END | BPF_FROM_BE:
538 #ifdef __BIG_ENDIAN__
539                         if (BPF_SRC(code) == BPF_FROM_BE)
540                                 goto emit_clear;
541 #else /* !__BIG_ENDIAN__ */
542                         if (BPF_SRC(code) == BPF_FROM_LE)
543                                 goto emit_clear;
544 #endif
545                         switch (imm) {
546                         case 16:
547                                 /* Rotate 8 bits left & mask with 0x0000ff00 */
548                                 PPC_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 16, 23);
549                                 /* Rotate 8 bits right & insert LSB to reg */
550                                 PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 24, 31);
551                                 /* Move result back to dst_reg */
552                                 PPC_MR(dst_reg, b2p[TMP_REG_1]);
553                                 break;
554                         case 32:
555                                 /*
556                                  * Rotate word left by 8 bits:
557                                  * 2 bytes are already in their final position
558                                  * -- byte 2 and 4 (of bytes 1, 2, 3 and 4)
559                                  */
560                                 PPC_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 0, 31);
561                                 /* Rotate 24 bits and insert byte 1 */
562                                 PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 0, 7);
563                                 /* Rotate 24 bits and insert byte 3 */
564                                 PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 16, 23);
565                                 PPC_MR(dst_reg, b2p[TMP_REG_1]);
566                                 break;
567                         case 64:
568                                 /*
569                                  * Way easier and faster(?) to store the value
570                                  * into stack and then use ldbrx
571                                  *
572                                  * ctx->seen will be reliable in pass2, but
573                                  * the instructions generated will remain the
574                                  * same across all passes
575                                  */
576                                 PPC_STD(dst_reg, 1, bpf_jit_stack_local(ctx));
577                                 PPC_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx));
578                                 PPC_LDBRX(dst_reg, 0, b2p[TMP_REG_1]);
579                                 break;
580                         }
581                         break;
582 
583 emit_clear:
584                         switch (imm) {
585                         case 16:
586                                 /* zero-extend 16 bits into 64 bits */
587                                 PPC_RLDICL(dst_reg, dst_reg, 0, 48);
588                                 break;
589                         case 32:
590                                 /* zero-extend 32 bits into 64 bits */
591                                 PPC_RLDICL(dst_reg, dst_reg, 0, 32);
592                                 break;
593                         case 64:
594                                 /* nop */
595                                 break;
596                         }
597                         break;
598 
599                 /*
600                  * BPF_ST(X)
601                  */
602                 case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */
603                 case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
604                         if (BPF_CLASS(code) == BPF_ST) {
605                                 PPC_LI(b2p[TMP_REG_1], imm);
606                                 src_reg = b2p[TMP_REG_1];
607                         }
608                         PPC_STB(src_reg, dst_reg, off);
609                         break;
610                 case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
611                 case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
612                         if (BPF_CLASS(code) == BPF_ST) {
613                                 PPC_LI(b2p[TMP_REG_1], imm);
614                                 src_reg = b2p[TMP_REG_1];
615                         }
616                         PPC_STH(src_reg, dst_reg, off);
617                         break;
618                 case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
619                 case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
620                         if (BPF_CLASS(code) == BPF_ST) {
621                                 PPC_LI32(b2p[TMP_REG_1], imm);
622                                 src_reg = b2p[TMP_REG_1];
623                         }
624                         PPC_STW(src_reg, dst_reg, off);
625                         break;
626                 case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
627                 case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
628                         if (BPF_CLASS(code) == BPF_ST) {
629                                 PPC_LI32(b2p[TMP_REG_1], imm);
630                                 src_reg = b2p[TMP_REG_1];
631                         }
632                         PPC_STD(src_reg, dst_reg, off);
633                         break;
634 
635                 /*
636                  * BPF_STX XADD (atomic_add)
637                  */
638                 /* *(u32 *)(dst + off) += src */
639                 case BPF_STX | BPF_XADD | BPF_W:
640                         /* Get EA into TMP_REG_1 */
641                         PPC_ADDI(b2p[TMP_REG_1], dst_reg, off);
642                         tmp_idx = ctx->idx * 4;
643                         /* load value from memory into TMP_REG_2 */
644                         PPC_BPF_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0);
645                         /* add value from src_reg into this */
646                         PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg);
647                         /* store result back */
648                         PPC_BPF_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]);
649                         /* we're done if this succeeded */
650                         PPC_BCC_SHORT(COND_NE, tmp_idx);
651                         break;
652                 /* *(u64 *)(dst + off) += src */
653                 case BPF_STX | BPF_XADD | BPF_DW:
654                         PPC_ADDI(b2p[TMP_REG_1], dst_reg, off);
655                         tmp_idx = ctx->idx * 4;
656                         PPC_BPF_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0);
657                         PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg);
658                         PPC_BPF_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]);
659                         PPC_BCC_SHORT(COND_NE, tmp_idx);
660                         break;
661 
662                 /*
663                  * BPF_LDX
664                  */
665                 /* dst = *(u8 *)(ul) (src + off) */
666                 case BPF_LDX | BPF_MEM | BPF_B:
667                         PPC_LBZ(dst_reg, src_reg, off);
668                         break;
669                 /* dst = *(u16 *)(ul) (src + off) */
670                 case BPF_LDX | BPF_MEM | BPF_H:
671                         PPC_LHZ(dst_reg, src_reg, off);
672                         break;
673                 /* dst = *(u32 *)(ul) (src + off) */
674                 case BPF_LDX | BPF_MEM | BPF_W:
675                         PPC_LWZ(dst_reg, src_reg, off);
676                         break;
677                 /* dst = *(u64 *)(ul) (src + off) */
678                 case BPF_LDX | BPF_MEM | BPF_DW:
679                         PPC_LD(dst_reg, src_reg, off);
680                         break;
681 
682                 /*
683                  * Doubleword load
684                  * 16 byte instruction that uses two 'struct bpf_insn'
685                  */
686                 case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */
687                         imm64 = ((u64)(u32) insn[i].imm) |
688                                     (((u64)(u32) insn[i+1].imm) << 32);
689                         /* Adjust for two bpf instructions */
690                         addrs[++i] = ctx->idx * 4;
691                         PPC_LI64(dst_reg, imm64);
692                         break;
693 
694                 /*
695                  * Return/Exit
696                  */
697                 case BPF_JMP | BPF_EXIT:
698                         /*
699                          * If this isn't the very last instruction, branch to
700                          * the epilogue. If we _are_ the last instruction,
701                          * we'll just fall through to the epilogue.
702                          */
703                         if (i != flen - 1)
704                                 PPC_JMP(exit_addr);
705                         /* else fall through to the epilogue */
706                         break;
707 
708                 /*
709                  * Call kernel helper or bpf function
710                  */
711                 case BPF_JMP | BPF_CALL:
712                         ctx->seen |= SEEN_FUNC;
713 
714                         /* bpf function call */
715                         if (insn[i].src_reg == BPF_PSEUDO_CALL)
716                                 if (!extra_pass)
717                                         func = NULL;
718                                 else if (fp->aux->func && off < fp->aux->func_cnt)
719                                         /* use the subprog id from the off
720                                          * field to lookup the callee address
721                                          */
722                                         func = (u8 *) fp->aux->func[off]->bpf_func;
723                                 else
724                                         return -EINVAL;
725                         /* kernel helper call */
726                         else
727                                 func = (u8 *) __bpf_call_base + imm;
728 
729                         bpf_jit_emit_func_call(image, ctx, (u64)func);
730 
731                         /* move return value from r3 to BPF_REG_0 */
732                         PPC_MR(b2p[BPF_REG_0], 3);
733                         break;
734 
735                 /*
736                  * Jumps and branches
737                  */
738                 case BPF_JMP | BPF_JA:
739                         PPC_JMP(addrs[i + 1 + off]);
740                         break;
741 
742                 case BPF_JMP | BPF_JGT | BPF_K:
743                 case BPF_JMP | BPF_JGT | BPF_X:
744                 case BPF_JMP | BPF_JSGT | BPF_K:
745                 case BPF_JMP | BPF_JSGT | BPF_X:
746                         true_cond = COND_GT;
747                         goto cond_branch;
748                 case BPF_JMP | BPF_JLT | BPF_K:
749                 case BPF_JMP | BPF_JLT | BPF_X:
750                 case BPF_JMP | BPF_JSLT | BPF_K:
751                 case BPF_JMP | BPF_JSLT | BPF_X:
752                         true_cond = COND_LT;
753                         goto cond_branch;
754                 case BPF_JMP | BPF_JGE | BPF_K:
755                 case BPF_JMP | BPF_JGE | BPF_X:
756                 case BPF_JMP | BPF_JSGE | BPF_K:
757                 case BPF_JMP | BPF_JSGE | BPF_X:
758                         true_cond = COND_GE;
759                         goto cond_branch;
760                 case BPF_JMP | BPF_JLE | BPF_K:
761                 case BPF_JMP | BPF_JLE | BPF_X:
762                 case BPF_JMP | BPF_JSLE | BPF_K:
763                 case BPF_JMP | BPF_JSLE | BPF_X:
764                         true_cond = COND_LE;
765                         goto cond_branch;
766                 case BPF_JMP | BPF_JEQ | BPF_K:
767                 case BPF_JMP | BPF_JEQ | BPF_X:
768                         true_cond = COND_EQ;
769                         goto cond_branch;
770                 case BPF_JMP | BPF_JNE | BPF_K:
771                 case BPF_JMP | BPF_JNE | BPF_X:
772                         true_cond = COND_NE;
773                         goto cond_branch;
774                 case BPF_JMP | BPF_JSET | BPF_K:
775                 case BPF_JMP | BPF_JSET | BPF_X:
776                         true_cond = COND_NE;
777                         /* Fall through */
778 
779 cond_branch:
780                         switch (code) {
781                         case BPF_JMP | BPF_JGT | BPF_X:
782                         case BPF_JMP | BPF_JLT | BPF_X:
783                         case BPF_JMP | BPF_JGE | BPF_X:
784                         case BPF_JMP | BPF_JLE | BPF_X:
785                         case BPF_JMP | BPF_JEQ | BPF_X:
786                         case BPF_JMP | BPF_JNE | BPF_X:
787                                 /* unsigned comparison */
788                                 PPC_CMPLD(dst_reg, src_reg);
789                                 break;
790                         case BPF_JMP | BPF_JSGT | BPF_X:
791                         case BPF_JMP | BPF_JSLT | BPF_X:
792                         case BPF_JMP | BPF_JSGE | BPF_X:
793                         case BPF_JMP | BPF_JSLE | BPF_X:
794                                 /* signed comparison */
795                                 PPC_CMPD(dst_reg, src_reg);
796                                 break;
797                         case BPF_JMP | BPF_JSET | BPF_X:
798                                 PPC_AND_DOT(b2p[TMP_REG_1], dst_reg, src_reg);
799                                 break;
800                         case BPF_JMP | BPF_JNE | BPF_K:
801                         case BPF_JMP | BPF_JEQ | BPF_K:
802                         case BPF_JMP | BPF_JGT | BPF_K:
803                         case BPF_JMP | BPF_JLT | BPF_K:
804                         case BPF_JMP | BPF_JGE | BPF_K:
805                         case BPF_JMP | BPF_JLE | BPF_K:
806                                 /*
807                                  * Need sign-extended load, so only positive
808                                  * values can be used as imm in cmpldi
809                                  */
810                                 if (imm >= 0 && imm < 32768)
811                                         PPC_CMPLDI(dst_reg, imm);
812                                 else {
813                                         /* sign-extending load */
814                                         PPC_LI32(b2p[TMP_REG_1], imm);
815                                         /* ... but unsigned comparison */
816                                         PPC_CMPLD(dst_reg, b2p[TMP_REG_1]);
817                                 }
818                                 break;
819                         case BPF_JMP | BPF_JSGT | BPF_K:
820                         case BPF_JMP | BPF_JSLT | BPF_K:
821                         case BPF_JMP | BPF_JSGE | BPF_K:
822                         case BPF_JMP | BPF_JSLE | BPF_K:
823                                 /*
824                                  * signed comparison, so any 16-bit value
825                                  * can be used in cmpdi
826                                  */
827                                 if (imm >= -32768 && imm < 32768)
828                                         PPC_CMPDI(dst_reg, imm);
829                                 else {
830                                         PPC_LI32(b2p[TMP_REG_1], imm);
831                                         PPC_CMPD(dst_reg, b2p[TMP_REG_1]);
832                                 }
833                                 break;
834                         case BPF_JMP | BPF_JSET | BPF_K:
835                                 /* andi does not sign-extend the immediate */
836                                 if (imm >= 0 && imm < 32768)
837                                         /* PPC_ANDI is _only/always_ dot-form */
838                                         PPC_ANDI(b2p[TMP_REG_1], dst_reg, imm);
839                                 else {
840                                         PPC_LI32(b2p[TMP_REG_1], imm);
841                                         PPC_AND_DOT(b2p[TMP_REG_1], dst_reg,
842                                                     b2p[TMP_REG_1]);
843                                 }
844                                 break;
845                         }
846                         PPC_BCC(true_cond, addrs[i + 1 + off]);
847                         break;
848 
849                 /*
850                  * Tail call
851                  */
852                 case BPF_JMP | BPF_TAIL_CALL:
853                         ctx->seen |= SEEN_TAILCALL;
854                         bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]);
855                         break;
856 
857                 default:
858                         /*
859                          * The filter contains something cruel & unusual.
860                          * We don't handle it, but also there shouldn't be
861                          * anything missing from our list.
862                          */
863                         pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n",
864                                         code, i);
865                         return -ENOTSUPP;
866                 }
867         }
868 
869         /* Set end-of-body-code address for exit. */
870         addrs[i] = ctx->idx * 4;
871 
872         return 0;
873 }
874 
875 struct powerpc64_jit_data {
876         struct bpf_binary_header *header;
877         u32 *addrs;
878         u8 *image;
879         u32 proglen;
880         struct codegen_context ctx;
881 };
882 
883 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
884 {
885         u32 proglen;
886         u32 alloclen;
887         u8 *image = NULL;
888         u32 *code_base;
889         u32 *addrs;
890         struct powerpc64_jit_data *jit_data;
891         struct codegen_context cgctx;
892         int pass;
893         int flen;
894         struct bpf_binary_header *bpf_hdr;
895         struct bpf_prog *org_fp = fp;
896         struct bpf_prog *tmp_fp;
897         bool bpf_blinded = false;
898         bool extra_pass = false;
899 
900         if (!fp->jit_requested)
901                 return org_fp;
902 
903         tmp_fp = bpf_jit_blind_constants(org_fp);
904         if (IS_ERR(tmp_fp))
905                 return org_fp;
906 
907         if (tmp_fp != org_fp) {
908                 bpf_blinded = true;
909                 fp = tmp_fp;
910         }
911 
912         jit_data = fp->aux->jit_data;
913         if (!jit_data) {
914                 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
915                 if (!jit_data) {
916                         fp = org_fp;
917                         goto out;
918                 }
919                 fp->aux->jit_data = jit_data;
920         }
921 
922         flen = fp->len;
923         addrs = jit_data->addrs;
924         if (addrs) {
925                 cgctx = jit_data->ctx;
926                 image = jit_data->image;
927                 bpf_hdr = jit_data->header;
928                 proglen = jit_data->proglen;
929                 alloclen = proglen + FUNCTION_DESCR_SIZE;
930                 extra_pass = true;
931                 goto skip_init_ctx;
932         }
933 
934         addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
935         if (addrs == NULL) {
936                 fp = org_fp;
937                 goto out_addrs;
938         }
939 
940         memset(&cgctx, 0, sizeof(struct codegen_context));
941 
942         /* Make sure that the stack is quadword aligned. */
943         cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
944 
945         /* Scouting faux-generate pass 0 */
946         if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) {
947                 /* We hit something illegal or unsupported. */
948                 fp = org_fp;
949                 goto out_addrs;
950         }
951 
952         /*
953          * Pretend to build prologue, given the features we've seen.  This will
954          * update ctgtx.idx as it pretends to output instructions, then we can
955          * calculate total size from idx.
956          */
957         bpf_jit_build_prologue(0, &cgctx);
958         bpf_jit_build_epilogue(0, &cgctx);
959 
960         proglen = cgctx.idx * 4;
961         alloclen = proglen + FUNCTION_DESCR_SIZE;
962 
963         bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4,
964                         bpf_jit_fill_ill_insns);
965         if (!bpf_hdr) {
966                 fp = org_fp;
967                 goto out_addrs;
968         }
969 
970 skip_init_ctx:
971         code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
972 
973         /* Code generation passes 1-2 */
974         for (pass = 1; pass < 3; pass++) {
975                 /* Now build the prologue, body code & epilogue for real. */
976                 cgctx.idx = 0;
977                 bpf_jit_build_prologue(code_base, &cgctx);
978                 bpf_jit_build_body(fp, code_base, &cgctx, addrs, extra_pass);
979                 bpf_jit_build_epilogue(code_base, &cgctx);
980 
981                 if (bpf_jit_enable > 1)
982                         pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
983                                 proglen - (cgctx.idx * 4), cgctx.seen);
984         }
985 
986         if (bpf_jit_enable > 1)
987                 /*
988                  * Note that we output the base address of the code_base
989                  * rather than image, since opcodes are in code_base.
990                  */
991                 bpf_jit_dump(flen, proglen, pass, code_base);
992 
993 #ifdef PPC64_ELF_ABI_v1
994         /* Function descriptor nastiness: Address + TOC */
995         ((u64 *)image)[0] = (u64)code_base;
996         ((u64 *)image)[1] = local_paca->kernel_toc;
997 #endif
998 
999         fp->bpf_func = (void *)image;
1000         fp->jited = 1;
1001         fp->jited_len = alloclen;
1002 
1003         bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + (bpf_hdr->pages * PAGE_SIZE));
1004         if (!fp->is_func || extra_pass) {
1005 out_addrs:
1006                 kfree(addrs);
1007                 kfree(jit_data);
1008                 fp->aux->jit_data = NULL;
1009         } else {
1010                 jit_data->addrs = addrs;
1011                 jit_data->ctx = cgctx;
1012                 jit_data->proglen = proglen;
1013                 jit_data->image = image;
1014                 jit_data->header = bpf_hdr;
1015         }
1016 
1017 out:
1018         if (bpf_blinded)
1019                 bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
1020 
1021         return fp;
1022 }
1023 
1024 /* Overriding bpf_jit_free() as we don't set images read-only. */
1025 void bpf_jit_free(struct bpf_prog *fp)
1026 {
1027         unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK;
1028         struct bpf_binary_header *bpf_hdr = (void *)addr;
1029 
1030         if (fp->jited)
1031                 bpf_jit_binary_free(bpf_hdr);
1032 
1033         bpf_prog_unlock_free(fp);
1034 }
1035 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp