~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/sparc/net/bpf_jit_comp_64.c

Version: ~ [ linux-4.15-rc3 ] ~ [ linux-4.14.5 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.68 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.105 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.47 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.87 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.51 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.96 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.27.62 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 #include <linux/moduleloader.h>
  3 #include <linux/workqueue.h>
  4 #include <linux/netdevice.h>
  5 #include <linux/filter.h>
  6 #include <linux/bpf.h>
  7 #include <linux/cache.h>
  8 #include <linux/if_vlan.h>
  9 
 10 #include <asm/cacheflush.h>
 11 #include <asm/ptrace.h>
 12 
 13 #include "bpf_jit_64.h"
 14 
 15 int bpf_jit_enable __read_mostly;
 16 
 17 static inline bool is_simm13(unsigned int value)
 18 {
 19         return value + 0x1000 < 0x2000;
 20 }
 21 
 22 static inline bool is_simm10(unsigned int value)
 23 {
 24         return value + 0x200 < 0x400;
 25 }
 26 
 27 static inline bool is_simm5(unsigned int value)
 28 {
 29         return value + 0x10 < 0x20;
 30 }
 31 
 32 static inline bool is_sethi(unsigned int value)
 33 {
 34         return (value & ~0x3fffff) == 0;
 35 }
 36 
 37 static void bpf_flush_icache(void *start_, void *end_)
 38 {
 39         /* Cheetah's I-cache is fully coherent.  */
 40         if (tlb_type == spitfire) {
 41                 unsigned long start = (unsigned long) start_;
 42                 unsigned long end = (unsigned long) end_;
 43 
 44                 start &= ~7UL;
 45                 end = (end + 7UL) & ~7UL;
 46                 while (start < end) {
 47                         flushi(start);
 48                         start += 32;
 49                 }
 50         }
 51 }
 52 
 53 #define SEEN_DATAREF 1 /* might call external helpers */
 54 #define SEEN_XREG    2 /* ebx is used */
 55 #define SEEN_MEM     4 /* use mem[] for temporary storage */
 56 
 57 #define S13(X)          ((X) & 0x1fff)
 58 #define S5(X)           ((X) & 0x1f)
 59 #define IMMED           0x00002000
 60 #define RD(X)           ((X) << 25)
 61 #define RS1(X)          ((X) << 14)
 62 #define RS2(X)          ((X))
 63 #define OP(X)           ((X) << 30)
 64 #define OP2(X)          ((X) << 22)
 65 #define OP3(X)          ((X) << 19)
 66 #define COND(X)         (((X) & 0xf) << 25)
 67 #define CBCOND(X)       (((X) & 0x1f) << 25)
 68 #define F1(X)           OP(X)
 69 #define F2(X, Y)        (OP(X) | OP2(Y))
 70 #define F3(X, Y)        (OP(X) | OP3(Y))
 71 #define ASI(X)          (((X) & 0xff) << 5)
 72 
 73 #define CONDN           COND(0x0)
 74 #define CONDE           COND(0x1)
 75 #define CONDLE          COND(0x2)
 76 #define CONDL           COND(0x3)
 77 #define CONDLEU         COND(0x4)
 78 #define CONDCS          COND(0x5)
 79 #define CONDNEG         COND(0x6)
 80 #define CONDVC          COND(0x7)
 81 #define CONDA           COND(0x8)
 82 #define CONDNE          COND(0x9)
 83 #define CONDG           COND(0xa)
 84 #define CONDGE          COND(0xb)
 85 #define CONDGU          COND(0xc)
 86 #define CONDCC          COND(0xd)
 87 #define CONDPOS         COND(0xe)
 88 #define CONDVS          COND(0xf)
 89 
 90 #define CONDGEU         CONDCC
 91 #define CONDLU          CONDCS
 92 
 93 #define WDISP22(X)      (((X) >> 2) & 0x3fffff)
 94 #define WDISP19(X)      (((X) >> 2) & 0x7ffff)
 95 
 96 /* The 10-bit branch displacement for CBCOND is split into two fields */
 97 static u32 WDISP10(u32 off)
 98 {
 99         u32 ret = ((off >> 2) & 0xff) << 5;
100 
101         ret |= ((off >> (2 + 8)) & 0x03) << 19;
102 
103         return ret;
104 }
105 
106 #define CBCONDE         CBCOND(0x09)
107 #define CBCONDLE        CBCOND(0x0a)
108 #define CBCONDL         CBCOND(0x0b)
109 #define CBCONDLEU       CBCOND(0x0c)
110 #define CBCONDCS        CBCOND(0x0d)
111 #define CBCONDN         CBCOND(0x0e)
112 #define CBCONDVS        CBCOND(0x0f)
113 #define CBCONDNE        CBCOND(0x19)
114 #define CBCONDG         CBCOND(0x1a)
115 #define CBCONDGE        CBCOND(0x1b)
116 #define CBCONDGU        CBCOND(0x1c)
117 #define CBCONDCC        CBCOND(0x1d)
118 #define CBCONDPOS       CBCOND(0x1e)
119 #define CBCONDVC        CBCOND(0x1f)
120 
121 #define CBCONDGEU       CBCONDCC
122 #define CBCONDLU        CBCONDCS
123 
124 #define ANNUL           (1 << 29)
125 #define XCC             (1 << 21)
126 
127 #define BRANCH          (F2(0, 1) | XCC)
128 #define CBCOND_OP       (F2(0, 3) | XCC)
129 
130 #define BA              (BRANCH | CONDA)
131 #define BG              (BRANCH | CONDG)
132 #define BL              (BRANCH | CONDL)
133 #define BLE             (BRANCH | CONDLE)
134 #define BGU             (BRANCH | CONDGU)
135 #define BLEU            (BRANCH | CONDLEU)
136 #define BGE             (BRANCH | CONDGE)
137 #define BGEU            (BRANCH | CONDGEU)
138 #define BLU             (BRANCH | CONDLU)
139 #define BE              (BRANCH | CONDE)
140 #define BNE             (BRANCH | CONDNE)
141 
142 #define SETHI(K, REG)   \
143         (F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff))
144 #define OR_LO(K, REG)   \
145         (F3(2, 0x02) | IMMED | RS1(REG) | ((K) & 0x3ff) | RD(REG))
146 
147 #define ADD             F3(2, 0x00)
148 #define AND             F3(2, 0x01)
149 #define ANDCC           F3(2, 0x11)
150 #define OR              F3(2, 0x02)
151 #define XOR             F3(2, 0x03)
152 #define SUB             F3(2, 0x04)
153 #define SUBCC           F3(2, 0x14)
154 #define MUL             F3(2, 0x0a)
155 #define MULX            F3(2, 0x09)
156 #define UDIVX           F3(2, 0x0d)
157 #define DIV             F3(2, 0x0e)
158 #define SLL             F3(2, 0x25)
159 #define SLLX            (F3(2, 0x25)|(1<<12))
160 #define SRA             F3(2, 0x27)
161 #define SRAX            (F3(2, 0x27)|(1<<12))
162 #define SRL             F3(2, 0x26)
163 #define SRLX            (F3(2, 0x26)|(1<<12))
164 #define JMPL            F3(2, 0x38)
165 #define SAVE            F3(2, 0x3c)
166 #define RESTORE         F3(2, 0x3d)
167 #define CALL            F1(1)
168 #define BR              F2(0, 0x01)
169 #define RD_Y            F3(2, 0x28)
170 #define WR_Y            F3(2, 0x30)
171 
172 #define LD32            F3(3, 0x00)
173 #define LD8             F3(3, 0x01)
174 #define LD16            F3(3, 0x02)
175 #define LD64            F3(3, 0x0b)
176 #define LD64A           F3(3, 0x1b)
177 #define ST8             F3(3, 0x05)
178 #define ST16            F3(3, 0x06)
179 #define ST32            F3(3, 0x04)
180 #define ST64            F3(3, 0x0e)
181 
182 #define CAS             F3(3, 0x3c)
183 #define CASX            F3(3, 0x3e)
184 
185 #define LDPTR           LD64
186 #define BASE_STACKFRAME 176
187 
188 #define LD32I           (LD32 | IMMED)
189 #define LD8I            (LD8 | IMMED)
190 #define LD16I           (LD16 | IMMED)
191 #define LD64I           (LD64 | IMMED)
192 #define LDPTRI          (LDPTR | IMMED)
193 #define ST32I           (ST32 | IMMED)
194 
195 struct jit_ctx {
196         struct bpf_prog         *prog;
197         unsigned int            *offset;
198         int                     idx;
199         int                     epilogue_offset;
200         bool                    tmp_1_used;
201         bool                    tmp_2_used;
202         bool                    tmp_3_used;
203         bool                    saw_ld_abs_ind;
204         bool                    saw_frame_pointer;
205         bool                    saw_call;
206         bool                    saw_tail_call;
207         u32                     *image;
208 };
209 
210 #define TMP_REG_1       (MAX_BPF_JIT_REG + 0)
211 #define TMP_REG_2       (MAX_BPF_JIT_REG + 1)
212 #define SKB_HLEN_REG    (MAX_BPF_JIT_REG + 2)
213 #define SKB_DATA_REG    (MAX_BPF_JIT_REG + 3)
214 #define TMP_REG_3       (MAX_BPF_JIT_REG + 4)
215 
216 /* Map BPF registers to SPARC registers */
217 static const int bpf2sparc[] = {
218         /* return value from in-kernel function, and exit value from eBPF */
219         [BPF_REG_0] = O5,
220 
221         /* arguments from eBPF program to in-kernel function */
222         [BPF_REG_1] = O0,
223         [BPF_REG_2] = O1,
224         [BPF_REG_3] = O2,
225         [BPF_REG_4] = O3,
226         [BPF_REG_5] = O4,
227 
228         /* callee saved registers that in-kernel function will preserve */
229         [BPF_REG_6] = L0,
230         [BPF_REG_7] = L1,
231         [BPF_REG_8] = L2,
232         [BPF_REG_9] = L3,
233 
234         /* read-only frame pointer to access stack */
235         [BPF_REG_FP] = L6,
236 
237         [BPF_REG_AX] = G7,
238 
239         /* temporary register for internal BPF JIT */
240         [TMP_REG_1] = G1,
241         [TMP_REG_2] = G2,
242         [TMP_REG_3] = G3,
243 
244         [SKB_HLEN_REG] = L4,
245         [SKB_DATA_REG] = L5,
246 };
247 
248 static void emit(const u32 insn, struct jit_ctx *ctx)
249 {
250         if (ctx->image != NULL)
251                 ctx->image[ctx->idx] = insn;
252 
253         ctx->idx++;
254 }
255 
256 static void emit_call(u32 *func, struct jit_ctx *ctx)
257 {
258         if (ctx->image != NULL) {
259                 void *here = &ctx->image[ctx->idx];
260                 unsigned int off;
261 
262                 off = (void *)func - here;
263                 ctx->image[ctx->idx] = CALL | ((off >> 2) & 0x3fffffff);
264         }
265         ctx->idx++;
266 }
267 
268 static void emit_nop(struct jit_ctx *ctx)
269 {
270         emit(SETHI(0, G0), ctx);
271 }
272 
273 static void emit_reg_move(u32 from, u32 to, struct jit_ctx *ctx)
274 {
275         emit(OR | RS1(G0) | RS2(from) | RD(to), ctx);
276 }
277 
278 /* Emit 32-bit constant, zero extended. */
279 static void emit_set_const(s32 K, u32 reg, struct jit_ctx *ctx)
280 {
281         emit(SETHI(K, reg), ctx);
282         emit(OR_LO(K, reg), ctx);
283 }
284 
285 /* Emit 32-bit constant, sign extended. */
286 static void emit_set_const_sext(s32 K, u32 reg, struct jit_ctx *ctx)
287 {
288         if (K >= 0) {
289                 emit(SETHI(K, reg), ctx);
290                 emit(OR_LO(K, reg), ctx);
291         } else {
292                 u32 hbits = ~(u32) K;
293                 u32 lbits = -0x400 | (u32) K;
294 
295                 emit(SETHI(hbits, reg), ctx);
296                 emit(XOR | IMMED | RS1(reg) | S13(lbits) | RD(reg), ctx);
297         }
298 }
299 
300 static void emit_alu(u32 opcode, u32 src, u32 dst, struct jit_ctx *ctx)
301 {
302         emit(opcode | RS1(dst) | RS2(src) | RD(dst), ctx);
303 }
304 
305 static void emit_alu3(u32 opcode, u32 a, u32 b, u32 c, struct jit_ctx *ctx)
306 {
307         emit(opcode | RS1(a) | RS2(b) | RD(c), ctx);
308 }
309 
310 static void emit_alu_K(unsigned int opcode, unsigned int dst, unsigned int imm,
311                        struct jit_ctx *ctx)
312 {
313         bool small_immed = is_simm13(imm);
314         unsigned int insn = opcode;
315 
316         insn |= RS1(dst) | RD(dst);
317         if (small_immed) {
318                 emit(insn | IMMED | S13(imm), ctx);
319         } else {
320                 unsigned int tmp = bpf2sparc[TMP_REG_1];
321 
322                 ctx->tmp_1_used = true;
323 
324                 emit_set_const_sext(imm, tmp, ctx);
325                 emit(insn | RS2(tmp), ctx);
326         }
327 }
328 
329 static void emit_alu3_K(unsigned int opcode, unsigned int src, unsigned int imm,
330                         unsigned int dst, struct jit_ctx *ctx)
331 {
332         bool small_immed = is_simm13(imm);
333         unsigned int insn = opcode;
334 
335         insn |= RS1(src) | RD(dst);
336         if (small_immed) {
337                 emit(insn | IMMED | S13(imm), ctx);
338         } else {
339                 unsigned int tmp = bpf2sparc[TMP_REG_1];
340 
341                 ctx->tmp_1_used = true;
342 
343                 emit_set_const_sext(imm, tmp, ctx);
344                 emit(insn | RS2(tmp), ctx);
345         }
346 }
347 
348 static void emit_loadimm32(s32 K, unsigned int dest, struct jit_ctx *ctx)
349 {
350         if (K >= 0 && is_simm13(K)) {
351                 /* or %g0, K, DEST */
352                 emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
353         } else {
354                 emit_set_const(K, dest, ctx);
355         }
356 }
357 
358 static void emit_loadimm(s32 K, unsigned int dest, struct jit_ctx *ctx)
359 {
360         if (is_simm13(K)) {
361                 /* or %g0, K, DEST */
362                 emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
363         } else {
364                 emit_set_const(K, dest, ctx);
365         }
366 }
367 
368 static void emit_loadimm_sext(s32 K, unsigned int dest, struct jit_ctx *ctx)
369 {
370         if (is_simm13(K)) {
371                 /* or %g0, K, DEST */
372                 emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
373         } else {
374                 emit_set_const_sext(K, dest, ctx);
375         }
376 }
377 
378 static void analyze_64bit_constant(u32 high_bits, u32 low_bits,
379                                    int *hbsp, int *lbsp, int *abbasp)
380 {
381         int lowest_bit_set, highest_bit_set, all_bits_between_are_set;
382         int i;
383 
384         lowest_bit_set = highest_bit_set = -1;
385         i = 0;
386         do {
387                 if ((lowest_bit_set == -1) && ((low_bits >> i) & 1))
388                         lowest_bit_set = i;
389                 if ((highest_bit_set == -1) && ((high_bits >> (32 - i - 1)) & 1))
390                         highest_bit_set = (64 - i - 1);
391         }  while (++i < 32 && (highest_bit_set == -1 ||
392                                lowest_bit_set == -1));
393         if (i == 32) {
394                 i = 0;
395                 do {
396                         if (lowest_bit_set == -1 && ((high_bits >> i) & 1))
397                                 lowest_bit_set = i + 32;
398                         if (highest_bit_set == -1 &&
399                             ((low_bits >> (32 - i - 1)) & 1))
400                                 highest_bit_set = 32 - i - 1;
401                 } while (++i < 32 && (highest_bit_set == -1 ||
402                                       lowest_bit_set == -1));
403         }
404 
405         all_bits_between_are_set = 1;
406         for (i = lowest_bit_set; i <= highest_bit_set; i++) {
407                 if (i < 32) {
408                         if ((low_bits & (1 << i)) != 0)
409                                 continue;
410                 } else {
411                         if ((high_bits & (1 << (i - 32))) != 0)
412                                 continue;
413                 }
414                 all_bits_between_are_set = 0;
415                 break;
416         }
417         *hbsp = highest_bit_set;
418         *lbsp = lowest_bit_set;
419         *abbasp = all_bits_between_are_set;
420 }
421 
422 static unsigned long create_simple_focus_bits(unsigned long high_bits,
423                                               unsigned long low_bits,
424                                               int lowest_bit_set, int shift)
425 {
426         long hi, lo;
427 
428         if (lowest_bit_set < 32) {
429                 lo = (low_bits >> lowest_bit_set) << shift;
430                 hi = ((high_bits << (32 - lowest_bit_set)) << shift);
431         } else {
432                 lo = 0;
433                 hi = ((high_bits >> (lowest_bit_set - 32)) << shift);
434         }
435         return hi | lo;
436 }
437 
438 static bool const64_is_2insns(unsigned long high_bits,
439                               unsigned long low_bits)
440 {
441         int highest_bit_set, lowest_bit_set, all_bits_between_are_set;
442 
443         if (high_bits == 0 || high_bits == 0xffffffff)
444                 return true;
445 
446         analyze_64bit_constant(high_bits, low_bits,
447                                &highest_bit_set, &lowest_bit_set,
448                                &all_bits_between_are_set);
449 
450         if ((highest_bit_set == 63 || lowest_bit_set == 0) &&
451             all_bits_between_are_set != 0)
452                 return true;
453 
454         if (highest_bit_set - lowest_bit_set < 21)
455                 return true;
456 
457         return false;
458 }
459 
460 static void sparc_emit_set_const64_quick2(unsigned long high_bits,
461                                           unsigned long low_imm,
462                                           unsigned int dest,
463                                           int shift_count, struct jit_ctx *ctx)
464 {
465         emit_loadimm32(high_bits, dest, ctx);
466 
467         /* Now shift it up into place.  */
468         emit_alu_K(SLLX, dest, shift_count, ctx);
469 
470         /* If there is a low immediate part piece, finish up by
471          * putting that in as well.
472          */
473         if (low_imm != 0)
474                 emit(OR | IMMED | RS1(dest) | S13(low_imm) | RD(dest), ctx);
475 }
476 
477 static void emit_loadimm64(u64 K, unsigned int dest, struct jit_ctx *ctx)
478 {
479         int all_bits_between_are_set, lowest_bit_set, highest_bit_set;
480         unsigned int tmp = bpf2sparc[TMP_REG_1];
481         u32 low_bits = (K & 0xffffffff);
482         u32 high_bits = (K >> 32);
483 
484         /* These two tests also take care of all of the one
485          * instruction cases.
486          */
487         if (high_bits == 0xffffffff && (low_bits & 0x80000000))
488                 return emit_loadimm_sext(K, dest, ctx);
489         if (high_bits == 0x00000000)
490                 return emit_loadimm32(K, dest, ctx);
491 
492         analyze_64bit_constant(high_bits, low_bits, &highest_bit_set,
493                                &lowest_bit_set, &all_bits_between_are_set);
494 
495         /* 1) mov       -1, %reg
496          *    sllx      %reg, shift, %reg
497          * 2) mov       -1, %reg
498          *    srlx      %reg, shift, %reg
499          * 3) mov       some_small_const, %reg
500          *    sllx      %reg, shift, %reg
501          */
502         if (((highest_bit_set == 63 || lowest_bit_set == 0) &&
503              all_bits_between_are_set != 0) ||
504             ((highest_bit_set - lowest_bit_set) < 12)) {
505                 int shift = lowest_bit_set;
506                 long the_const = -1;
507 
508                 if ((highest_bit_set != 63 && lowest_bit_set != 0) ||
509                     all_bits_between_are_set == 0) {
510                         the_const =
511                                 create_simple_focus_bits(high_bits, low_bits,
512                                                          lowest_bit_set, 0);
513                 } else if (lowest_bit_set == 0)
514                         shift = -(63 - highest_bit_set);
515 
516                 emit(OR | IMMED | RS1(G0) | S13(the_const) | RD(dest), ctx);
517                 if (shift > 0)
518                         emit_alu_K(SLLX, dest, shift, ctx);
519                 else if (shift < 0)
520                         emit_alu_K(SRLX, dest, -shift, ctx);
521 
522                 return;
523         }
524 
525         /* Now a range of 22 or less bits set somewhere.
526          * 1) sethi     %hi(focus_bits), %reg
527          *    sllx      %reg, shift, %reg
528          * 2) sethi     %hi(focus_bits), %reg
529          *    srlx      %reg, shift, %reg
530          */
531         if ((highest_bit_set - lowest_bit_set) < 21) {
532                 unsigned long focus_bits =
533                         create_simple_focus_bits(high_bits, low_bits,
534                                                  lowest_bit_set, 10);
535 
536                 emit(SETHI(focus_bits, dest), ctx);
537 
538                 /* If lowest_bit_set == 10 then a sethi alone could
539                  * have done it.
540                  */
541                 if (lowest_bit_set < 10)
542                         emit_alu_K(SRLX, dest, 10 - lowest_bit_set, ctx);
543                 else if (lowest_bit_set > 10)
544                         emit_alu_K(SLLX, dest, lowest_bit_set - 10, ctx);
545                 return;
546         }
547 
548         /* Ok, now 3 instruction sequences.  */
549         if (low_bits == 0) {
550                 emit_loadimm32(high_bits, dest, ctx);
551                 emit_alu_K(SLLX, dest, 32, ctx);
552                 return;
553         }
554 
555         /* We may be able to do something quick
556          * when the constant is negated, so try that.
557          */
558         if (const64_is_2insns((~high_bits) & 0xffffffff,
559                               (~low_bits) & 0xfffffc00)) {
560                 /* NOTE: The trailing bits get XOR'd so we need the
561                  * non-negated bits, not the negated ones.
562                  */
563                 unsigned long trailing_bits = low_bits & 0x3ff;
564 
565                 if ((((~high_bits) & 0xffffffff) == 0 &&
566                      ((~low_bits) & 0x80000000) == 0) ||
567                     (((~high_bits) & 0xffffffff) == 0xffffffff &&
568                      ((~low_bits) & 0x80000000) != 0)) {
569                         unsigned long fast_int = (~low_bits & 0xffffffff);
570 
571                         if ((is_sethi(fast_int) &&
572                              (~high_bits & 0xffffffff) == 0)) {
573                                 emit(SETHI(fast_int, dest), ctx);
574                         } else if (is_simm13(fast_int)) {
575                                 emit(OR | IMMED | RS1(G0) | S13(fast_int) | RD(dest), ctx);
576                         } else {
577                                 emit_loadimm64(fast_int, dest, ctx);
578                         }
579                 } else {
580                         u64 n = ((~low_bits) & 0xfffffc00) |
581                                 (((unsigned long)((~high_bits) & 0xffffffff))<<32);
582                         emit_loadimm64(n, dest, ctx);
583                 }
584 
585                 low_bits = -0x400 | trailing_bits;
586 
587                 emit(XOR | IMMED | RS1(dest) | S13(low_bits) | RD(dest), ctx);
588                 return;
589         }
590 
591         /* 1) sethi     %hi(xxx), %reg
592          *    or        %reg, %lo(xxx), %reg
593          *    sllx      %reg, yyy, %reg
594          */
595         if ((highest_bit_set - lowest_bit_set) < 32) {
596                 unsigned long focus_bits =
597                         create_simple_focus_bits(high_bits, low_bits,
598                                                  lowest_bit_set, 0);
599 
600                 /* So what we know is that the set bits straddle the
601                  * middle of the 64-bit word.
602                  */
603                 sparc_emit_set_const64_quick2(focus_bits, 0, dest,
604                                               lowest_bit_set, ctx);
605                 return;
606         }
607 
608         /* 1) sethi     %hi(high_bits), %reg
609          *    or        %reg, %lo(high_bits), %reg
610          *    sllx      %reg, 32, %reg
611          *    or        %reg, low_bits, %reg
612          */
613         if (is_simm13(low_bits) && ((int)low_bits > 0)) {
614                 sparc_emit_set_const64_quick2(high_bits, low_bits,
615                                               dest, 32, ctx);
616                 return;
617         }
618 
619         /* Oh well, we tried... Do a full 64-bit decomposition.  */
620         ctx->tmp_1_used = true;
621 
622         emit_loadimm32(high_bits, tmp, ctx);
623         emit_loadimm32(low_bits, dest, ctx);
624         emit_alu_K(SLLX, tmp, 32, ctx);
625         emit(OR | RS1(dest) | RS2(tmp) | RD(dest), ctx);
626 }
627 
628 static void emit_branch(unsigned int br_opc, unsigned int from_idx, unsigned int to_idx,
629                         struct jit_ctx *ctx)
630 {
631         unsigned int off = to_idx - from_idx;
632 
633         if (br_opc & XCC)
634                 emit(br_opc | WDISP19(off << 2), ctx);
635         else
636                 emit(br_opc | WDISP22(off << 2), ctx);
637 }
638 
639 static void emit_cbcond(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx,
640                         const u8 dst, const u8 src, struct jit_ctx *ctx)
641 {
642         unsigned int off = to_idx - from_idx;
643 
644         emit(cb_opc | WDISP10(off << 2) | RS1(dst) | RS2(src), ctx);
645 }
646 
647 static void emit_cbcondi(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx,
648                          const u8 dst, s32 imm, struct jit_ctx *ctx)
649 {
650         unsigned int off = to_idx - from_idx;
651 
652         emit(cb_opc | IMMED | WDISP10(off << 2) | RS1(dst) | S5(imm), ctx);
653 }
654 
655 #define emit_read_y(REG, CTX)   emit(RD_Y | RD(REG), CTX)
656 #define emit_write_y(REG, CTX)  emit(WR_Y | IMMED | RS1(REG) | S13(0), CTX)
657 
658 #define emit_cmp(R1, R2, CTX)                           \
659         emit(SUBCC | RS1(R1) | RS2(R2) | RD(G0), CTX)
660 
661 #define emit_cmpi(R1, IMM, CTX)                         \
662         emit(SUBCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX)
663 
664 #define emit_btst(R1, R2, CTX)                          \
665         emit(ANDCC | RS1(R1) | RS2(R2) | RD(G0), CTX)
666 
667 #define emit_btsti(R1, IMM, CTX)                        \
668         emit(ANDCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX)
669 
670 static int emit_compare_and_branch(const u8 code, const u8 dst, u8 src,
671                                    const s32 imm, bool is_imm, int branch_dst,
672                                    struct jit_ctx *ctx)
673 {
674         bool use_cbcond = (sparc64_elf_hwcap & AV_SPARC_CBCOND) != 0;
675         const u8 tmp = bpf2sparc[TMP_REG_1];
676 
677         branch_dst = ctx->offset[branch_dst];
678 
679         if (!is_simm10(branch_dst - ctx->idx) ||
680             BPF_OP(code) == BPF_JSET)
681                 use_cbcond = false;
682 
683         if (is_imm) {
684                 bool fits = true;
685 
686                 if (use_cbcond) {
687                         if (!is_simm5(imm))
688                                 fits = false;
689                 } else if (!is_simm13(imm)) {
690                         fits = false;
691                 }
692                 if (!fits) {
693                         ctx->tmp_1_used = true;
694                         emit_loadimm_sext(imm, tmp, ctx);
695                         src = tmp;
696                         is_imm = false;
697                 }
698         }
699 
700         if (!use_cbcond) {
701                 u32 br_opcode;
702 
703                 if (BPF_OP(code) == BPF_JSET) {
704                         if (is_imm)
705                                 emit_btsti(dst, imm, ctx);
706                         else
707                                 emit_btst(dst, src, ctx);
708                 } else {
709                         if (is_imm)
710                                 emit_cmpi(dst, imm, ctx);
711                         else
712                                 emit_cmp(dst, src, ctx);
713                 }
714                 switch (BPF_OP(code)) {
715                 case BPF_JEQ:
716                         br_opcode = BE;
717                         break;
718                 case BPF_JGT:
719                         br_opcode = BGU;
720                         break;
721                 case BPF_JLT:
722                         br_opcode = BLU;
723                         break;
724                 case BPF_JGE:
725                         br_opcode = BGEU;
726                         break;
727                 case BPF_JLE:
728                         br_opcode = BLEU;
729                         break;
730                 case BPF_JSET:
731                 case BPF_JNE:
732                         br_opcode = BNE;
733                         break;
734                 case BPF_JSGT:
735                         br_opcode = BG;
736                         break;
737                 case BPF_JSLT:
738                         br_opcode = BL;
739                         break;
740                 case BPF_JSGE:
741                         br_opcode = BGE;
742                         break;
743                 case BPF_JSLE:
744                         br_opcode = BLE;
745                         break;
746                 default:
747                         /* Make sure we dont leak kernel information to the
748                          * user.
749                          */
750                         return -EFAULT;
751                 }
752                 emit_branch(br_opcode, ctx->idx, branch_dst, ctx);
753                 emit_nop(ctx);
754         } else {
755                 u32 cbcond_opcode;
756 
757                 switch (BPF_OP(code)) {
758                 case BPF_JEQ:
759                         cbcond_opcode = CBCONDE;
760                         break;
761                 case BPF_JGT:
762                         cbcond_opcode = CBCONDGU;
763                         break;
764                 case BPF_JLT:
765                         cbcond_opcode = CBCONDLU;
766                         break;
767                 case BPF_JGE:
768                         cbcond_opcode = CBCONDGEU;
769                         break;
770                 case BPF_JLE:
771                         cbcond_opcode = CBCONDLEU;
772                         break;
773                 case BPF_JNE:
774                         cbcond_opcode = CBCONDNE;
775                         break;
776                 case BPF_JSGT:
777                         cbcond_opcode = CBCONDG;
778                         break;
779                 case BPF_JSLT:
780                         cbcond_opcode = CBCONDL;
781                         break;
782                 case BPF_JSGE:
783                         cbcond_opcode = CBCONDGE;
784                         break;
785                 case BPF_JSLE:
786                         cbcond_opcode = CBCONDLE;
787                         break;
788                 default:
789                         /* Make sure we dont leak kernel information to the
790                          * user.
791                          */
792                         return -EFAULT;
793                 }
794                 cbcond_opcode |= CBCOND_OP;
795                 if (is_imm)
796                         emit_cbcondi(cbcond_opcode, ctx->idx, branch_dst,
797                                      dst, imm, ctx);
798                 else
799                         emit_cbcond(cbcond_opcode, ctx->idx, branch_dst,
800                                     dst, src, ctx);
801         }
802         return 0;
803 }
804 
805 static void load_skb_regs(struct jit_ctx *ctx, u8 r_skb)
806 {
807         const u8 r_headlen = bpf2sparc[SKB_HLEN_REG];
808         const u8 r_data = bpf2sparc[SKB_DATA_REG];
809         const u8 r_tmp = bpf2sparc[TMP_REG_1];
810         unsigned int off;
811 
812         off = offsetof(struct sk_buff, len);
813         emit(LD32I | RS1(r_skb) | S13(off) | RD(r_headlen), ctx);
814 
815         off = offsetof(struct sk_buff, data_len);
816         emit(LD32I | RS1(r_skb) | S13(off) | RD(r_tmp), ctx);
817 
818         emit(SUB | RS1(r_headlen) | RS2(r_tmp) | RD(r_headlen), ctx);
819 
820         off = offsetof(struct sk_buff, data);
821         emit(LDPTRI | RS1(r_skb) | S13(off) | RD(r_data), ctx);
822 }
823 
824 /* Just skip the save instruction and the ctx register move.  */
825 #define BPF_TAILCALL_PROLOGUE_SKIP      16
826 #define BPF_TAILCALL_CNT_SP_OFF         (STACK_BIAS + 128)
827 
828 static void build_prologue(struct jit_ctx *ctx)
829 {
830         s32 stack_needed = BASE_STACKFRAME;
831 
832         if (ctx->saw_frame_pointer || ctx->saw_tail_call) {
833                 struct bpf_prog *prog = ctx->prog;
834                 u32 stack_depth;
835 
836                 stack_depth = prog->aux->stack_depth;
837                 stack_needed += round_up(stack_depth, 16);
838         }
839 
840         if (ctx->saw_tail_call)
841                 stack_needed += 8;
842 
843         /* save %sp, -176, %sp */
844         emit(SAVE | IMMED | RS1(SP) | S13(-stack_needed) | RD(SP), ctx);
845 
846         /* tail_call_cnt = 0 */
847         if (ctx->saw_tail_call) {
848                 u32 off = BPF_TAILCALL_CNT_SP_OFF;
849 
850                 emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(G0), ctx);
851         } else {
852                 emit_nop(ctx);
853         }
854         if (ctx->saw_frame_pointer) {
855                 const u8 vfp = bpf2sparc[BPF_REG_FP];
856 
857                 emit(ADD | IMMED | RS1(FP) | S13(STACK_BIAS) | RD(vfp), ctx);
858         }
859 
860         emit_reg_move(I0, O0, ctx);
861         /* If you add anything here, adjust BPF_TAILCALL_PROLOGUE_SKIP above. */
862 
863         if (ctx->saw_ld_abs_ind)
864                 load_skb_regs(ctx, bpf2sparc[BPF_REG_1]);
865 }
866 
867 static void build_epilogue(struct jit_ctx *ctx)
868 {
869         ctx->epilogue_offset = ctx->idx;
870 
871         /* ret (jmpl %i7 + 8, %g0) */
872         emit(JMPL | IMMED | RS1(I7) | S13(8) | RD(G0), ctx);
873 
874         /* restore %i5, %g0, %o0 */
875         emit(RESTORE | RS1(bpf2sparc[BPF_REG_0]) | RS2(G0) | RD(O0), ctx);
876 }
877 
878 static void emit_tail_call(struct jit_ctx *ctx)
879 {
880         const u8 bpf_array = bpf2sparc[BPF_REG_2];
881         const u8 bpf_index = bpf2sparc[BPF_REG_3];
882         const u8 tmp = bpf2sparc[TMP_REG_1];
883         u32 off;
884 
885         ctx->saw_tail_call = true;
886 
887         off = offsetof(struct bpf_array, map.max_entries);
888         emit(LD32 | IMMED | RS1(bpf_array) | S13(off) | RD(tmp), ctx);
889         emit_cmp(bpf_index, tmp, ctx);
890 #define OFFSET1 17
891         emit_branch(BGEU, ctx->idx, ctx->idx + OFFSET1, ctx);
892         emit_nop(ctx);
893 
894         off = BPF_TAILCALL_CNT_SP_OFF;
895         emit(LD32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx);
896         emit_cmpi(tmp, MAX_TAIL_CALL_CNT, ctx);
897 #define OFFSET2 13
898         emit_branch(BGU, ctx->idx, ctx->idx + OFFSET2, ctx);
899         emit_nop(ctx);
900 
901         emit_alu_K(ADD, tmp, 1, ctx);
902         off = BPF_TAILCALL_CNT_SP_OFF;
903         emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx);
904 
905         emit_alu3_K(SLL, bpf_index, 3, tmp, ctx);
906         emit_alu(ADD, bpf_array, tmp, ctx);
907         off = offsetof(struct bpf_array, ptrs);
908         emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx);
909 
910         emit_cmpi(tmp, 0, ctx);
911 #define OFFSET3 5
912         emit_branch(BE, ctx->idx, ctx->idx + OFFSET3, ctx);
913         emit_nop(ctx);
914 
915         off = offsetof(struct bpf_prog, bpf_func);
916         emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx);
917 
918         off = BPF_TAILCALL_PROLOGUE_SKIP;
919         emit(JMPL | IMMED | RS1(tmp) | S13(off) | RD(G0), ctx);
920         emit_nop(ctx);
921 }
922 
923 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
924 {
925         const u8 code = insn->code;
926         const u8 dst = bpf2sparc[insn->dst_reg];
927         const u8 src = bpf2sparc[insn->src_reg];
928         const int i = insn - ctx->prog->insnsi;
929         const s16 off = insn->off;
930         const s32 imm = insn->imm;
931         u32 *func;
932 
933         if (insn->src_reg == BPF_REG_FP)
934                 ctx->saw_frame_pointer = true;
935 
936         switch (code) {
937         /* dst = src */
938         case BPF_ALU | BPF_MOV | BPF_X:
939                 emit_alu3_K(SRL, src, 0, dst, ctx);
940                 break;
941         case BPF_ALU64 | BPF_MOV | BPF_X:
942                 emit_reg_move(src, dst, ctx);
943                 break;
944         /* dst = dst OP src */
945         case BPF_ALU | BPF_ADD | BPF_X:
946         case BPF_ALU64 | BPF_ADD | BPF_X:
947                 emit_alu(ADD, src, dst, ctx);
948                 goto do_alu32_trunc;
949         case BPF_ALU | BPF_SUB | BPF_X:
950         case BPF_ALU64 | BPF_SUB | BPF_X:
951                 emit_alu(SUB, src, dst, ctx);
952                 goto do_alu32_trunc;
953         case BPF_ALU | BPF_AND | BPF_X:
954         case BPF_ALU64 | BPF_AND | BPF_X:
955                 emit_alu(AND, src, dst, ctx);
956                 goto do_alu32_trunc;
957         case BPF_ALU | BPF_OR | BPF_X:
958         case BPF_ALU64 | BPF_OR | BPF_X:
959                 emit_alu(OR, src, dst, ctx);
960                 goto do_alu32_trunc;
961         case BPF_ALU | BPF_XOR | BPF_X:
962         case BPF_ALU64 | BPF_XOR | BPF_X:
963                 emit_alu(XOR, src, dst, ctx);
964                 goto do_alu32_trunc;
965         case BPF_ALU | BPF_MUL | BPF_X:
966                 emit_alu(MUL, src, dst, ctx);
967                 goto do_alu32_trunc;
968         case BPF_ALU64 | BPF_MUL | BPF_X:
969                 emit_alu(MULX, src, dst, ctx);
970                 break;
971         case BPF_ALU | BPF_DIV | BPF_X:
972                 emit_cmp(src, G0, ctx);
973                 emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx);
974                 emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx);
975 
976                 emit_write_y(G0, ctx);
977                 emit_alu(DIV, src, dst, ctx);
978                 break;
979 
980         case BPF_ALU64 | BPF_DIV | BPF_X:
981                 emit_cmp(src, G0, ctx);
982                 emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx);
983                 emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx);
984 
985                 emit_alu(UDIVX, src, dst, ctx);
986                 break;
987 
988         case BPF_ALU | BPF_MOD | BPF_X: {
989                 const u8 tmp = bpf2sparc[TMP_REG_1];
990 
991                 ctx->tmp_1_used = true;
992 
993                 emit_cmp(src, G0, ctx);
994                 emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx);
995                 emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx);
996 
997                 emit_write_y(G0, ctx);
998                 emit_alu3(DIV, dst, src, tmp, ctx);
999                 emit_alu3(MULX, tmp, src, tmp, ctx);
1000                 emit_alu3(SUB, dst, tmp, dst, ctx);
1001                 goto do_alu32_trunc;
1002         }
1003         case BPF_ALU64 | BPF_MOD | BPF_X: {
1004                 const u8 tmp = bpf2sparc[TMP_REG_1];
1005 
1006                 ctx->tmp_1_used = true;
1007 
1008                 emit_cmp(src, G0, ctx);
1009                 emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx);
1010                 emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx);
1011 
1012                 emit_alu3(UDIVX, dst, src, tmp, ctx);
1013                 emit_alu3(MULX, tmp, src, tmp, ctx);
1014                 emit_alu3(SUB, dst, tmp, dst, ctx);
1015                 break;
1016         }
1017         case BPF_ALU | BPF_LSH | BPF_X:
1018                 emit_alu(SLL, src, dst, ctx);
1019                 goto do_alu32_trunc;
1020         case BPF_ALU64 | BPF_LSH | BPF_X:
1021                 emit_alu(SLLX, src, dst, ctx);
1022                 break;
1023         case BPF_ALU | BPF_RSH | BPF_X:
1024                 emit_alu(SRL, src, dst, ctx);
1025                 break;
1026         case BPF_ALU64 | BPF_RSH | BPF_X:
1027                 emit_alu(SRLX, src, dst, ctx);
1028                 break;
1029         case BPF_ALU | BPF_ARSH | BPF_X:
1030                 emit_alu(SRA, src, dst, ctx);
1031                 goto do_alu32_trunc;
1032         case BPF_ALU64 | BPF_ARSH | BPF_X:
1033                 emit_alu(SRAX, src, dst, ctx);
1034                 break;
1035 
1036         /* dst = -dst */
1037         case BPF_ALU | BPF_NEG:
1038         case BPF_ALU64 | BPF_NEG:
1039                 emit(SUB | RS1(0) | RS2(dst) | RD(dst), ctx);
1040                 goto do_alu32_trunc;
1041 
1042         case BPF_ALU | BPF_END | BPF_FROM_BE:
1043                 switch (imm) {
1044                 case 16:
1045                         emit_alu_K(SLL, dst, 16, ctx);
1046                         emit_alu_K(SRL, dst, 16, ctx);
1047                         break;
1048                 case 32:
1049                         emit_alu_K(SRL, dst, 0, ctx);
1050                         break;
1051                 case 64:
1052                         /* nop */
1053                         break;
1054 
1055                 }
1056                 break;
1057 
1058         /* dst = BSWAP##imm(dst) */
1059         case BPF_ALU | BPF_END | BPF_FROM_LE: {
1060                 const u8 tmp = bpf2sparc[TMP_REG_1];
1061                 const u8 tmp2 = bpf2sparc[TMP_REG_2];
1062 
1063                 ctx->tmp_1_used = true;
1064                 switch (imm) {
1065                 case 16:
1066                         emit_alu3_K(AND, dst, 0xff, tmp, ctx);
1067                         emit_alu3_K(SRL, dst, 8, dst, ctx);
1068                         emit_alu3_K(AND, dst, 0xff, dst, ctx);
1069                         emit_alu3_K(SLL, tmp, 8, tmp, ctx);
1070                         emit_alu(OR, tmp, dst, ctx);
1071                         break;
1072 
1073                 case 32:
1074                         ctx->tmp_2_used = true;
1075                         emit_alu3_K(SRL, dst, 24, tmp, ctx);    /* tmp  = dst >> 24 */
1076                         emit_alu3_K(SRL, dst, 16, tmp2, ctx);   /* tmp2 = dst >> 16 */
1077                         emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */
1078                         emit_alu3_K(SLL, tmp2, 8, tmp2, ctx);   /* tmp2 = tmp2 << 8 */
1079                         emit_alu(OR, tmp2, tmp, ctx);           /* tmp  = tmp | tmp2 */
1080                         emit_alu3_K(SRL, dst, 8, tmp2, ctx);    /* tmp2 = dst >> 8 */
1081                         emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */
1082                         emit_alu3_K(SLL, tmp2, 16, tmp2, ctx);  /* tmp2 = tmp2 << 16 */
1083                         emit_alu(OR, tmp2, tmp, ctx);           /* tmp  = tmp | tmp2 */
1084                         emit_alu3_K(AND, dst, 0xff, dst, ctx);  /* dst  = dst & 0xff */
1085                         emit_alu3_K(SLL, dst, 24, dst, ctx);    /* dst  = dst << 24 */
1086                         emit_alu(OR, tmp, dst, ctx);            /* dst  = dst | tmp */
1087                         break;
1088 
1089                 case 64:
1090                         emit_alu3_K(ADD, SP, STACK_BIAS + 128, tmp, ctx);
1091                         emit(ST64 | RS1(tmp) | RS2(G0) | RD(dst), ctx);
1092                         emit(LD64A | ASI(ASI_PL) | RS1(tmp) | RS2(G0) | RD(dst), ctx);
1093                         break;
1094                 }
1095                 break;
1096         }
1097         /* dst = imm */
1098         case BPF_ALU | BPF_MOV | BPF_K:
1099                 emit_loadimm32(imm, dst, ctx);
1100                 break;
1101         case BPF_ALU64 | BPF_MOV | BPF_K:
1102                 emit_loadimm_sext(imm, dst, ctx);
1103                 break;
1104         /* dst = dst OP imm */
1105         case BPF_ALU | BPF_ADD | BPF_K:
1106         case BPF_ALU64 | BPF_ADD | BPF_K:
1107                 emit_alu_K(ADD, dst, imm, ctx);
1108                 goto do_alu32_trunc;
1109         case BPF_ALU | BPF_SUB | BPF_K:
1110         case BPF_ALU64 | BPF_SUB | BPF_K:
1111                 emit_alu_K(SUB, dst, imm, ctx);
1112                 goto do_alu32_trunc;
1113         case BPF_ALU | BPF_AND | BPF_K:
1114         case BPF_ALU64 | BPF_AND | BPF_K:
1115                 emit_alu_K(AND, dst, imm, ctx);
1116                 goto do_alu32_trunc;
1117         case BPF_ALU | BPF_OR | BPF_K:
1118         case BPF_ALU64 | BPF_OR | BPF_K:
1119                 emit_alu_K(OR, dst, imm, ctx);
1120                 goto do_alu32_trunc;
1121         case BPF_ALU | BPF_XOR | BPF_K:
1122         case BPF_ALU64 | BPF_XOR | BPF_K:
1123                 emit_alu_K(XOR, dst, imm, ctx);
1124                 goto do_alu32_trunc;
1125         case BPF_ALU | BPF_MUL | BPF_K:
1126                 emit_alu_K(MUL, dst, imm, ctx);
1127                 goto do_alu32_trunc;
1128         case BPF_ALU64 | BPF_MUL | BPF_K:
1129                 emit_alu_K(MULX, dst, imm, ctx);
1130                 break;
1131         case BPF_ALU | BPF_DIV | BPF_K:
1132                 if (imm == 0)
1133                         return -EINVAL;
1134 
1135                 emit_write_y(G0, ctx);
1136                 emit_alu_K(DIV, dst, imm, ctx);
1137                 goto do_alu32_trunc;
1138         case BPF_ALU64 | BPF_DIV | BPF_K:
1139                 if (imm == 0)
1140                         return -EINVAL;
1141 
1142                 emit_alu_K(UDIVX, dst, imm, ctx);
1143                 break;
1144         case BPF_ALU64 | BPF_MOD | BPF_K:
1145         case BPF_ALU | BPF_MOD | BPF_K: {
1146                 const u8 tmp = bpf2sparc[TMP_REG_2];
1147                 unsigned int div;
1148 
1149                 if (imm == 0)
1150                         return -EINVAL;
1151 
1152                 div = (BPF_CLASS(code) == BPF_ALU64) ? UDIVX : DIV;
1153 
1154                 ctx->tmp_2_used = true;
1155 
1156                 if (BPF_CLASS(code) != BPF_ALU64)
1157                         emit_write_y(G0, ctx);
1158                 if (is_simm13(imm)) {
1159                         emit(div | IMMED | RS1(dst) | S13(imm) | RD(tmp), ctx);
1160                         emit(MULX | IMMED | RS1(tmp) | S13(imm) | RD(tmp), ctx);
1161                         emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx);
1162                 } else {
1163                         const u8 tmp1 = bpf2sparc[TMP_REG_1];
1164 
1165                         ctx->tmp_1_used = true;
1166 
1167                         emit_set_const_sext(imm, tmp1, ctx);
1168                         emit(div | RS1(dst) | RS2(tmp1) | RD(tmp), ctx);
1169                         emit(MULX | RS1(tmp) | RS2(tmp1) | RD(tmp), ctx);
1170                         emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx);
1171                 }
1172                 goto do_alu32_trunc;
1173         }
1174         case BPF_ALU | BPF_LSH | BPF_K:
1175                 emit_alu_K(SLL, dst, imm, ctx);
1176                 goto do_alu32_trunc;
1177         case BPF_ALU64 | BPF_LSH | BPF_K:
1178                 emit_alu_K(SLLX, dst, imm, ctx);
1179                 break;
1180         case BPF_ALU | BPF_RSH | BPF_K:
1181                 emit_alu_K(SRL, dst, imm, ctx);
1182                 break;
1183         case BPF_ALU64 | BPF_RSH | BPF_K:
1184                 emit_alu_K(SRLX, dst, imm, ctx);
1185                 break;
1186         case BPF_ALU | BPF_ARSH | BPF_K:
1187                 emit_alu_K(SRA, dst, imm, ctx);
1188                 goto do_alu32_trunc;
1189         case BPF_ALU64 | BPF_ARSH | BPF_K:
1190                 emit_alu_K(SRAX, dst, imm, ctx);
1191                 break;
1192 
1193         do_alu32_trunc:
1194                 if (BPF_CLASS(code) == BPF_ALU)
1195                         emit_alu_K(SRL, dst, 0, ctx);
1196                 break;
1197 
1198         /* JUMP off */
1199         case BPF_JMP | BPF_JA:
1200                 emit_branch(BA, ctx->idx, ctx->offset[i + off], ctx);
1201                 emit_nop(ctx);
1202                 break;
1203         /* IF (dst COND src) JUMP off */
1204         case BPF_JMP | BPF_JEQ | BPF_X:
1205         case BPF_JMP | BPF_JGT | BPF_X:
1206         case BPF_JMP | BPF_JLT | BPF_X:
1207         case BPF_JMP | BPF_JGE | BPF_X:
1208         case BPF_JMP | BPF_JLE | BPF_X:
1209         case BPF_JMP | BPF_JNE | BPF_X:
1210         case BPF_JMP | BPF_JSGT | BPF_X:
1211         case BPF_JMP | BPF_JSLT | BPF_X:
1212         case BPF_JMP | BPF_JSGE | BPF_X:
1213         case BPF_JMP | BPF_JSLE | BPF_X:
1214         case BPF_JMP | BPF_JSET | BPF_X: {
1215                 int err;
1216 
1217                 err = emit_compare_and_branch(code, dst, src, 0, false, i + off, ctx);
1218                 if (err)
1219                         return err;
1220                 break;
1221         }
1222         /* IF (dst COND imm) JUMP off */
1223         case BPF_JMP | BPF_JEQ | BPF_K:
1224         case BPF_JMP | BPF_JGT | BPF_K:
1225         case BPF_JMP | BPF_JLT | BPF_K:
1226         case BPF_JMP | BPF_JGE | BPF_K:
1227         case BPF_JMP | BPF_JLE | BPF_K:
1228         case BPF_JMP | BPF_JNE | BPF_K:
1229         case BPF_JMP | BPF_JSGT | BPF_K:
1230         case BPF_JMP | BPF_JSLT | BPF_K:
1231         case BPF_JMP | BPF_JSGE | BPF_K:
1232         case BPF_JMP | BPF_JSLE | BPF_K:
1233         case BPF_JMP | BPF_JSET | BPF_K: {
1234                 int err;
1235 
1236                 err = emit_compare_and_branch(code, dst, 0, imm, true, i + off, ctx);
1237                 if (err)
1238                         return err;
1239                 break;
1240         }
1241 
1242         /* function call */
1243         case BPF_JMP | BPF_CALL:
1244         {
1245                 u8 *func = ((u8 *)__bpf_call_base) + imm;
1246 
1247                 ctx->saw_call = true;
1248 
1249                 emit_call((u32 *)func, ctx);
1250                 emit_nop(ctx);
1251 
1252                 emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx);
1253 
1254                 if (bpf_helper_changes_pkt_data(func) && ctx->saw_ld_abs_ind)
1255                         load_skb_regs(ctx, bpf2sparc[BPF_REG_6]);
1256                 break;
1257         }
1258 
1259         /* tail call */
1260         case BPF_JMP | BPF_TAIL_CALL:
1261                 emit_tail_call(ctx);
1262                 break;
1263 
1264         /* function return */
1265         case BPF_JMP | BPF_EXIT:
1266                 /* Optimization: when last instruction is EXIT,
1267                    simply fallthrough to epilogue. */
1268                 if (i == ctx->prog->len - 1)
1269                         break;
1270                 emit_branch(BA, ctx->idx, ctx->epilogue_offset, ctx);
1271                 emit_nop(ctx);
1272                 break;
1273 
1274         /* dst = imm64 */
1275         case BPF_LD | BPF_IMM | BPF_DW:
1276         {
1277                 const struct bpf_insn insn1 = insn[1];
1278                 u64 imm64;
1279 
1280                 imm64 = (u64)insn1.imm << 32 | (u32)imm;
1281                 emit_loadimm64(imm64, dst, ctx);
1282 
1283                 return 1;
1284         }
1285 
1286         /* LDX: dst = *(size *)(src + off) */
1287         case BPF_LDX | BPF_MEM | BPF_W:
1288         case BPF_LDX | BPF_MEM | BPF_H:
1289         case BPF_LDX | BPF_MEM | BPF_B:
1290         case BPF_LDX | BPF_MEM | BPF_DW: {
1291                 const u8 tmp = bpf2sparc[TMP_REG_1];
1292                 u32 opcode = 0, rs2;
1293 
1294                 ctx->tmp_1_used = true;
1295                 switch (BPF_SIZE(code)) {
1296                 case BPF_W:
1297                         opcode = LD32;
1298                         break;
1299                 case BPF_H:
1300                         opcode = LD16;
1301                         break;
1302                 case BPF_B:
1303                         opcode = LD8;
1304                         break;
1305                 case BPF_DW:
1306                         opcode = LD64;
1307                         break;
1308                 }
1309 
1310                 if (is_simm13(off)) {
1311                         opcode |= IMMED;
1312                         rs2 = S13(off);
1313                 } else {
1314                         emit_loadimm(off, tmp, ctx);
1315                         rs2 = RS2(tmp);
1316                 }
1317                 emit(opcode | RS1(src) | rs2 | RD(dst), ctx);
1318                 break;
1319         }
1320         /* ST: *(size *)(dst + off) = imm */
1321         case BPF_ST | BPF_MEM | BPF_W:
1322         case BPF_ST | BPF_MEM | BPF_H:
1323         case BPF_ST | BPF_MEM | BPF_B:
1324         case BPF_ST | BPF_MEM | BPF_DW: {
1325                 const u8 tmp = bpf2sparc[TMP_REG_1];
1326                 const u8 tmp2 = bpf2sparc[TMP_REG_2];
1327                 u32 opcode = 0, rs2;
1328 
1329                 ctx->tmp_2_used = true;
1330                 emit_loadimm(imm, tmp2, ctx);
1331 
1332                 switch (BPF_SIZE(code)) {
1333                 case BPF_W:
1334                         opcode = ST32;
1335                         break;
1336                 case BPF_H:
1337                         opcode = ST16;
1338                         break;
1339                 case BPF_B:
1340                         opcode = ST8;
1341                         break;
1342                 case BPF_DW:
1343                         opcode = ST64;
1344                         break;
1345                 }
1346 
1347                 if (is_simm13(off)) {
1348                         opcode |= IMMED;
1349                         rs2 = S13(off);
1350                 } else {
1351                         ctx->tmp_1_used = true;
1352                         emit_loadimm(off, tmp, ctx);
1353                         rs2 = RS2(tmp);
1354                 }
1355                 emit(opcode | RS1(dst) | rs2 | RD(tmp2), ctx);
1356                 break;
1357         }
1358 
1359         /* STX: *(size *)(dst + off) = src */
1360         case BPF_STX | BPF_MEM | BPF_W:
1361         case BPF_STX | BPF_MEM | BPF_H:
1362         case BPF_STX | BPF_MEM | BPF_B:
1363         case BPF_STX | BPF_MEM | BPF_DW: {
1364                 const u8 tmp = bpf2sparc[TMP_REG_1];
1365                 u32 opcode = 0, rs2;
1366 
1367                 switch (BPF_SIZE(code)) {
1368                 case BPF_W:
1369                         opcode = ST32;
1370                         break;
1371                 case BPF_H:
1372                         opcode = ST16;
1373                         break;
1374                 case BPF_B:
1375                         opcode = ST8;
1376                         break;
1377                 case BPF_DW:
1378                         opcode = ST64;
1379                         break;
1380                 }
1381                 if (is_simm13(off)) {
1382                         opcode |= IMMED;
1383                         rs2 = S13(off);
1384                 } else {
1385                         ctx->tmp_1_used = true;
1386                         emit_loadimm(off, tmp, ctx);
1387                         rs2 = RS2(tmp);
1388                 }
1389                 emit(opcode | RS1(dst) | rs2 | RD(src), ctx);
1390                 break;
1391         }
1392 
1393         /* STX XADD: lock *(u32 *)(dst + off) += src */
1394         case BPF_STX | BPF_XADD | BPF_W: {
1395                 const u8 tmp = bpf2sparc[TMP_REG_1];
1396                 const u8 tmp2 = bpf2sparc[TMP_REG_2];
1397                 const u8 tmp3 = bpf2sparc[TMP_REG_3];
1398 
1399                 ctx->tmp_1_used = true;
1400                 ctx->tmp_2_used = true;
1401                 ctx->tmp_3_used = true;
1402                 emit_loadimm(off, tmp, ctx);
1403                 emit_alu3(ADD, dst, tmp, tmp, ctx);
1404 
1405                 emit(LD32 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx);
1406                 emit_alu3(ADD, tmp2, src, tmp3, ctx);
1407                 emit(CAS | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx);
1408                 emit_cmp(tmp2, tmp3, ctx);
1409                 emit_branch(BNE, 4, 0, ctx);
1410                 emit_nop(ctx);
1411                 break;
1412         }
1413         /* STX XADD: lock *(u64 *)(dst + off) += src */
1414         case BPF_STX | BPF_XADD | BPF_DW: {
1415                 const u8 tmp = bpf2sparc[TMP_REG_1];
1416                 const u8 tmp2 = bpf2sparc[TMP_REG_2];
1417                 const u8 tmp3 = bpf2sparc[TMP_REG_3];
1418 
1419                 ctx->tmp_1_used = true;
1420                 ctx->tmp_2_used = true;
1421                 ctx->tmp_3_used = true;
1422                 emit_loadimm(off, tmp, ctx);
1423                 emit_alu3(ADD, dst, tmp, tmp, ctx);
1424 
1425                 emit(LD64 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx);
1426                 emit_alu3(ADD, tmp2, src, tmp3, ctx);
1427                 emit(CASX | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx);
1428                 emit_cmp(tmp2, tmp3, ctx);
1429                 emit_branch(BNE, 4, 0, ctx);
1430                 emit_nop(ctx);
1431                 break;
1432         }
1433 #define CHOOSE_LOAD_FUNC(K, func) \
1434                 ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset)
1435 
1436         /* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + imm)) */
1437         case BPF_LD | BPF_ABS | BPF_W:
1438                 func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_word);
1439                 goto common_load;
1440         case BPF_LD | BPF_ABS | BPF_H:
1441                 func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_half);
1442                 goto common_load;
1443         case BPF_LD | BPF_ABS | BPF_B:
1444                 func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_byte);
1445                 goto common_load;
1446         /* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + src + imm)) */
1447         case BPF_LD | BPF_IND | BPF_W:
1448                 func = bpf_jit_load_word;
1449                 goto common_load;
1450         case BPF_LD | BPF_IND | BPF_H:
1451                 func = bpf_jit_load_half;
1452                 goto common_load;
1453 
1454         case BPF_LD | BPF_IND | BPF_B:
1455                 func = bpf_jit_load_byte;
1456         common_load:
1457                 ctx->saw_ld_abs_ind = true;
1458 
1459                 emit_reg_move(bpf2sparc[BPF_REG_6], O0, ctx);
1460                 emit_loadimm(imm, O1, ctx);
1461 
1462                 if (BPF_MODE(code) == BPF_IND)
1463                         emit_alu(ADD, src, O1, ctx);
1464 
1465                 emit_call(func, ctx);
1466                 emit_alu_K(SRA, O1, 0, ctx);
1467 
1468                 emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx);
1469                 break;
1470 
1471         default:
1472                 pr_err_once("unknown opcode %02x\n", code);
1473                 return -EINVAL;
1474         }
1475 
1476         return 0;
1477 }
1478 
1479 static int build_body(struct jit_ctx *ctx)
1480 {
1481         const struct bpf_prog *prog = ctx->prog;
1482         int i;
1483 
1484         for (i = 0; i < prog->len; i++) {
1485                 const struct bpf_insn *insn = &prog->insnsi[i];
1486                 int ret;
1487 
1488                 ret = build_insn(insn, ctx);
1489 
1490                 if (ret > 0) {
1491                         i++;
1492                         ctx->offset[i] = ctx->idx;
1493                         continue;
1494                 }
1495                 ctx->offset[i] = ctx->idx;
1496                 if (ret)
1497                         return ret;
1498         }
1499         return 0;
1500 }
1501 
1502 static void jit_fill_hole(void *area, unsigned int size)
1503 {
1504         u32 *ptr;
1505         /* We are guaranteed to have aligned memory. */
1506         for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
1507                 *ptr++ = 0x91d02005; /* ta 5 */
1508 }
1509 
1510 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1511 {
1512         struct bpf_prog *tmp, *orig_prog = prog;
1513         struct bpf_binary_header *header;
1514         bool tmp_blinded = false;
1515         struct jit_ctx ctx;
1516         u32 image_size;
1517         u8 *image_ptr;
1518         int pass;
1519 
1520         if (!bpf_jit_enable)
1521                 return orig_prog;
1522 
1523         tmp = bpf_jit_blind_constants(prog);
1524         /* If blinding was requested and we failed during blinding,
1525          * we must fall back to the interpreter.
1526          */
1527         if (IS_ERR(tmp))
1528                 return orig_prog;
1529         if (tmp != prog) {
1530                 tmp_blinded = true;
1531                 prog = tmp;
1532         }
1533 
1534         memset(&ctx, 0, sizeof(ctx));
1535         ctx.prog = prog;
1536 
1537         ctx.offset = kcalloc(prog->len, sizeof(unsigned int), GFP_KERNEL);
1538         if (ctx.offset == NULL) {
1539                 prog = orig_prog;
1540                 goto out;
1541         }
1542 
1543         /* Fake pass to detect features used, and get an accurate assessment
1544          * of what the final image size will be.
1545          */
1546         if (build_body(&ctx)) {
1547                 prog = orig_prog;
1548                 goto out_off;
1549         }
1550         build_prologue(&ctx);
1551         build_epilogue(&ctx);
1552 
1553         /* Now we know the actual image size. */
1554         image_size = sizeof(u32) * ctx.idx;
1555         header = bpf_jit_binary_alloc(image_size, &image_ptr,
1556                                       sizeof(u32), jit_fill_hole);
1557         if (header == NULL) {
1558                 prog = orig_prog;
1559                 goto out_off;
1560         }
1561 
1562         ctx.image = (u32 *)image_ptr;
1563 
1564         for (pass = 1; pass < 3; pass++) {
1565                 ctx.idx = 0;
1566 
1567                 build_prologue(&ctx);
1568 
1569                 if (build_body(&ctx)) {
1570                         bpf_jit_binary_free(header);
1571                         prog = orig_prog;
1572                         goto out_off;
1573                 }
1574 
1575                 build_epilogue(&ctx);
1576 
1577                 if (bpf_jit_enable > 1)
1578                         pr_info("Pass %d: shrink = %d, seen = [%c%c%c%c%c%c%c]\n", pass,
1579                                 image_size - (ctx.idx * 4),
1580                                 ctx.tmp_1_used ? '1' : ' ',
1581                                 ctx.tmp_2_used ? '2' : ' ',
1582                                 ctx.tmp_3_used ? '3' : ' ',
1583                                 ctx.saw_ld_abs_ind ? 'L' : ' ',
1584                                 ctx.saw_frame_pointer ? 'F' : ' ',
1585                                 ctx.saw_call ? 'C' : ' ',
1586                                 ctx.saw_tail_call ? 'T' : ' ');
1587         }
1588 
1589         if (bpf_jit_enable > 1)
1590                 bpf_jit_dump(prog->len, image_size, pass, ctx.image);
1591 
1592         bpf_flush_icache(header, (u8 *)header + (header->pages * PAGE_SIZE));
1593 
1594         bpf_jit_binary_lock_ro(header);
1595 
1596         prog->bpf_func = (void *)ctx.image;
1597         prog->jited = 1;
1598         prog->jited_len = image_size;
1599 
1600 out_off:
1601         kfree(ctx.offset);
1602 out:
1603         if (tmp_blinded)
1604                 bpf_jit_prog_release_other(prog, prog == orig_prog ?
1605                                            tmp : orig_prog);
1606         return prog;
1607 }
1608 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp