~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/tools/lib/bpf/libbpf.c

Version: ~ [ linux-4.15-rc3 ] ~ [ linux-4.14.5 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.68 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.105 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.47 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.87 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.51 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.96 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.27.62 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Common eBPF ELF object loading operations.
  3  *
  4  * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
  5  * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
  6  * Copyright (C) 2015 Huawei Inc.
  7  * Copyright (C) 2017 Nicira, Inc.
  8  *
  9  * This program is free software; you can redistribute it and/or
 10  * modify it under the terms of the GNU Lesser General Public
 11  * License as published by the Free Software Foundation;
 12  * version 2.1 of the License (not later!)
 13  *
 14  * This program is distributed in the hope that it will be useful,
 15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 17  * GNU Lesser General Public License for more details.
 18  *
 19  * You should have received a copy of the GNU Lesser General Public
 20  * License along with this program; if not,  see <http://www.gnu.org/licenses>
 21  */
 22 
 23 #include <stdlib.h>
 24 #include <stdio.h>
 25 #include <stdarg.h>
 26 #include <libgen.h>
 27 #include <inttypes.h>
 28 #include <string.h>
 29 #include <unistd.h>
 30 #include <fcntl.h>
 31 #include <errno.h>
 32 #include <asm/unistd.h>
 33 #include <linux/err.h>
 34 #include <linux/kernel.h>
 35 #include <linux/bpf.h>
 36 #include <linux/list.h>
 37 #include <linux/limits.h>
 38 #include <sys/stat.h>
 39 #include <sys/types.h>
 40 #include <sys/vfs.h>
 41 #include <libelf.h>
 42 #include <gelf.h>
 43 
 44 #include "libbpf.h"
 45 #include "bpf.h"
 46 
 47 #ifndef EM_BPF
 48 #define EM_BPF 247
 49 #endif
 50 
 51 #ifndef BPF_FS_MAGIC
 52 #define BPF_FS_MAGIC            0xcafe4a11
 53 #endif
 54 
 55 #define __printf(a, b)  __attribute__((format(printf, a, b)))
 56 
 57 __printf(1, 2)
 58 static int __base_pr(const char *format, ...)
 59 {
 60         va_list args;
 61         int err;
 62 
 63         va_start(args, format);
 64         err = vfprintf(stderr, format, args);
 65         va_end(args);
 66         return err;
 67 }
 68 
 69 static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr;
 70 static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr;
 71 static __printf(1, 2) libbpf_print_fn_t __pr_debug;
 72 
 73 #define __pr(func, fmt, ...)    \
 74 do {                            \
 75         if ((func))             \
 76                 (func)("libbpf: " fmt, ##__VA_ARGS__); \
 77 } while (0)
 78 
 79 #define pr_warning(fmt, ...)    __pr(__pr_warning, fmt, ##__VA_ARGS__)
 80 #define pr_info(fmt, ...)       __pr(__pr_info, fmt, ##__VA_ARGS__)
 81 #define pr_debug(fmt, ...)      __pr(__pr_debug, fmt, ##__VA_ARGS__)
 82 
 83 void libbpf_set_print(libbpf_print_fn_t warn,
 84                       libbpf_print_fn_t info,
 85                       libbpf_print_fn_t debug)
 86 {
 87         __pr_warning = warn;
 88         __pr_info = info;
 89         __pr_debug = debug;
 90 }
 91 
 92 #define STRERR_BUFSIZE  128
 93 
 94 #define ERRNO_OFFSET(e)         ((e) - __LIBBPF_ERRNO__START)
 95 #define ERRCODE_OFFSET(c)       ERRNO_OFFSET(LIBBPF_ERRNO__##c)
 96 #define NR_ERRNO        (__LIBBPF_ERRNO__END - __LIBBPF_ERRNO__START)
 97 
 98 static const char *libbpf_strerror_table[NR_ERRNO] = {
 99         [ERRCODE_OFFSET(LIBELF)]        = "Something wrong in libelf",
100         [ERRCODE_OFFSET(FORMAT)]        = "BPF object format invalid",
101         [ERRCODE_OFFSET(KVERSION)]      = "'version' section incorrect or lost",
102         [ERRCODE_OFFSET(ENDIAN)]        = "Endian mismatch",
103         [ERRCODE_OFFSET(INTERNAL)]      = "Internal error in libbpf",
104         [ERRCODE_OFFSET(RELOC)]         = "Relocation failed",
105         [ERRCODE_OFFSET(VERIFY)]        = "Kernel verifier blocks program loading",
106         [ERRCODE_OFFSET(PROG2BIG)]      = "Program too big",
107         [ERRCODE_OFFSET(KVER)]          = "Incorrect kernel version",
108         [ERRCODE_OFFSET(PROGTYPE)]      = "Kernel doesn't support this program type",
109 };
110 
111 int libbpf_strerror(int err, char *buf, size_t size)
112 {
113         if (!buf || !size)
114                 return -1;
115 
116         err = err > 0 ? err : -err;
117 
118         if (err < __LIBBPF_ERRNO__START) {
119                 int ret;
120 
121                 ret = strerror_r(err, buf, size);
122                 buf[size - 1] = '\0';
123                 return ret;
124         }
125 
126         if (err < __LIBBPF_ERRNO__END) {
127                 const char *msg;
128 
129                 msg = libbpf_strerror_table[ERRNO_OFFSET(err)];
130                 snprintf(buf, size, "%s", msg);
131                 buf[size - 1] = '\0';
132                 return 0;
133         }
134 
135         snprintf(buf, size, "Unknown libbpf error %d", err);
136         buf[size - 1] = '\0';
137         return -1;
138 }
139 
140 #define CHECK_ERR(action, err, out) do {        \
141         err = action;                   \
142         if (err)                        \
143                 goto out;               \
144 } while(0)
145 
146 
147 /* Copied from tools/perf/util/util.h */
148 #ifndef zfree
149 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
150 #endif
151 
152 #ifndef zclose
153 # define zclose(fd) ({                  \
154         int ___err = 0;                 \
155         if ((fd) >= 0)                  \
156                 ___err = close((fd));   \
157         fd = -1;                        \
158         ___err; })
159 #endif
160 
161 #ifdef HAVE_LIBELF_MMAP_SUPPORT
162 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
163 #else
164 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
165 #endif
166 
167 /*
168  * bpf_prog should be a better name but it has been used in
169  * linux/filter.h.
170  */
171 struct bpf_program {
172         /* Index in elf obj file, for relocation use. */
173         int idx;
174         char *name;
175         char *section_name;
176         struct bpf_insn *insns;
177         size_t insns_cnt;
178         enum bpf_prog_type type;
179 
180         struct {
181                 int insn_idx;
182                 int map_idx;
183         } *reloc_desc;
184         int nr_reloc;
185 
186         struct {
187                 int nr;
188                 int *fds;
189         } instances;
190         bpf_program_prep_t preprocessor;
191 
192         struct bpf_object *obj;
193         void *priv;
194         bpf_program_clear_priv_t clear_priv;
195 };
196 
197 struct bpf_map {
198         int fd;
199         char *name;
200         size_t offset;
201         struct bpf_map_def def;
202         void *priv;
203         bpf_map_clear_priv_t clear_priv;
204 };
205 
206 static LIST_HEAD(bpf_objects_list);
207 
208 struct bpf_object {
209         char license[64];
210         u32 kern_version;
211 
212         struct bpf_program *programs;
213         size_t nr_programs;
214         struct bpf_map *maps;
215         size_t nr_maps;
216 
217         bool loaded;
218 
219         /*
220          * Information when doing elf related work. Only valid if fd
221          * is valid.
222          */
223         struct {
224                 int fd;
225                 void *obj_buf;
226                 size_t obj_buf_sz;
227                 Elf *elf;
228                 GElf_Ehdr ehdr;
229                 Elf_Data *symbols;
230                 size_t strtabidx;
231                 struct {
232                         GElf_Shdr shdr;
233                         Elf_Data *data;
234                 } *reloc;
235                 int nr_reloc;
236                 int maps_shndx;
237         } efile;
238         /*
239          * All loaded bpf_object is linked in a list, which is
240          * hidden to caller. bpf_objects__<func> handlers deal with
241          * all objects.
242          */
243         struct list_head list;
244 
245         void *priv;
246         bpf_object_clear_priv_t clear_priv;
247 
248         char path[];
249 };
250 #define obj_elf_valid(o)        ((o)->efile.elf)
251 
252 static void bpf_program__unload(struct bpf_program *prog)
253 {
254         int i;
255 
256         if (!prog)
257                 return;
258 
259         /*
260          * If the object is opened but the program was never loaded,
261          * it is possible that prog->instances.nr == -1.
262          */
263         if (prog->instances.nr > 0) {
264                 for (i = 0; i < prog->instances.nr; i++)
265                         zclose(prog->instances.fds[i]);
266         } else if (prog->instances.nr != -1) {
267                 pr_warning("Internal error: instances.nr is %d\n",
268                            prog->instances.nr);
269         }
270 
271         prog->instances.nr = -1;
272         zfree(&prog->instances.fds);
273 }
274 
275 static void bpf_program__exit(struct bpf_program *prog)
276 {
277         if (!prog)
278                 return;
279 
280         if (prog->clear_priv)
281                 prog->clear_priv(prog, prog->priv);
282 
283         prog->priv = NULL;
284         prog->clear_priv = NULL;
285 
286         bpf_program__unload(prog);
287         zfree(&prog->name);
288         zfree(&prog->section_name);
289         zfree(&prog->insns);
290         zfree(&prog->reloc_desc);
291 
292         prog->nr_reloc = 0;
293         prog->insns_cnt = 0;
294         prog->idx = -1;
295 }
296 
297 static int
298 bpf_program__init(void *data, size_t size, char *section_name, int idx,
299                   struct bpf_program *prog)
300 {
301         if (size < sizeof(struct bpf_insn)) {
302                 pr_warning("corrupted section '%s'\n", section_name);
303                 return -EINVAL;
304         }
305 
306         bzero(prog, sizeof(*prog));
307 
308         prog->section_name = strdup(section_name);
309         if (!prog->section_name) {
310                 pr_warning("failed to alloc name for prog under section %s\n",
311                            section_name);
312                 goto errout;
313         }
314 
315         prog->insns = malloc(size);
316         if (!prog->insns) {
317                 pr_warning("failed to alloc insns for prog under section %s\n",
318                            section_name);
319                 goto errout;
320         }
321         prog->insns_cnt = size / sizeof(struct bpf_insn);
322         memcpy(prog->insns, data,
323                prog->insns_cnt * sizeof(struct bpf_insn));
324         prog->idx = idx;
325         prog->instances.fds = NULL;
326         prog->instances.nr = -1;
327         prog->type = BPF_PROG_TYPE_KPROBE;
328 
329         return 0;
330 errout:
331         bpf_program__exit(prog);
332         return -ENOMEM;
333 }
334 
335 static int
336 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size,
337                         char *section_name, int idx)
338 {
339         struct bpf_program prog, *progs;
340         int nr_progs, err;
341 
342         err = bpf_program__init(data, size, section_name, idx, &prog);
343         if (err)
344                 return err;
345 
346         progs = obj->programs;
347         nr_progs = obj->nr_programs;
348 
349         progs = realloc(progs, sizeof(progs[0]) * (nr_progs + 1));
350         if (!progs) {
351                 /*
352                  * In this case the original obj->programs
353                  * is still valid, so don't need special treat for
354                  * bpf_close_object().
355                  */
356                 pr_warning("failed to alloc a new program under section '%s'\n",
357                            section_name);
358                 bpf_program__exit(&prog);
359                 return -ENOMEM;
360         }
361 
362         pr_debug("found program %s\n", prog.section_name);
363         obj->programs = progs;
364         obj->nr_programs = nr_progs + 1;
365         prog.obj = obj;
366         progs[nr_progs] = prog;
367         return 0;
368 }
369 
370 static int
371 bpf_object__init_prog_names(struct bpf_object *obj)
372 {
373         Elf_Data *symbols = obj->efile.symbols;
374         struct bpf_program *prog;
375         size_t pi, si;
376 
377         for (pi = 0; pi < obj->nr_programs; pi++) {
378                 char *name = NULL;
379 
380                 prog = &obj->programs[pi];
381 
382                 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name;
383                      si++) {
384                         GElf_Sym sym;
385 
386                         if (!gelf_getsym(symbols, si, &sym))
387                                 continue;
388                         if (sym.st_shndx != prog->idx)
389                                 continue;
390 
391                         name = elf_strptr(obj->efile.elf,
392                                           obj->efile.strtabidx,
393                                           sym.st_name);
394                         if (!name) {
395                                 pr_warning("failed to get sym name string for prog %s\n",
396                                            prog->section_name);
397                                 return -LIBBPF_ERRNO__LIBELF;
398                         }
399                 }
400 
401                 if (!name) {
402                         pr_warning("failed to find sym for prog %s\n",
403                                    prog->section_name);
404                         return -EINVAL;
405                 }
406 
407                 prog->name = strdup(name);
408                 if (!prog->name) {
409                         pr_warning("failed to allocate memory for prog sym %s\n",
410                                    name);
411                         return -ENOMEM;
412                 }
413         }
414 
415         return 0;
416 }
417 
418 static struct bpf_object *bpf_object__new(const char *path,
419                                           void *obj_buf,
420                                           size_t obj_buf_sz)
421 {
422         struct bpf_object *obj;
423 
424         obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
425         if (!obj) {
426                 pr_warning("alloc memory failed for %s\n", path);
427                 return ERR_PTR(-ENOMEM);
428         }
429 
430         strcpy(obj->path, path);
431         obj->efile.fd = -1;
432 
433         /*
434          * Caller of this function should also calls
435          * bpf_object__elf_finish() after data collection to return
436          * obj_buf to user. If not, we should duplicate the buffer to
437          * avoid user freeing them before elf finish.
438          */
439         obj->efile.obj_buf = obj_buf;
440         obj->efile.obj_buf_sz = obj_buf_sz;
441         obj->efile.maps_shndx = -1;
442 
443         obj->loaded = false;
444 
445         INIT_LIST_HEAD(&obj->list);
446         list_add(&obj->list, &bpf_objects_list);
447         return obj;
448 }
449 
450 static void bpf_object__elf_finish(struct bpf_object *obj)
451 {
452         if (!obj_elf_valid(obj))
453                 return;
454 
455         if (obj->efile.elf) {
456                 elf_end(obj->efile.elf);
457                 obj->efile.elf = NULL;
458         }
459         obj->efile.symbols = NULL;
460 
461         zfree(&obj->efile.reloc);
462         obj->efile.nr_reloc = 0;
463         zclose(obj->efile.fd);
464         obj->efile.obj_buf = NULL;
465         obj->efile.obj_buf_sz = 0;
466 }
467 
468 static int bpf_object__elf_init(struct bpf_object *obj)
469 {
470         int err = 0;
471         GElf_Ehdr *ep;
472 
473         if (obj_elf_valid(obj)) {
474                 pr_warning("elf init: internal error\n");
475                 return -LIBBPF_ERRNO__LIBELF;
476         }
477 
478         if (obj->efile.obj_buf_sz > 0) {
479                 /*
480                  * obj_buf should have been validated by
481                  * bpf_object__open_buffer().
482                  */
483                 obj->efile.elf = elf_memory(obj->efile.obj_buf,
484                                             obj->efile.obj_buf_sz);
485         } else {
486                 obj->efile.fd = open(obj->path, O_RDONLY);
487                 if (obj->efile.fd < 0) {
488                         pr_warning("failed to open %s: %s\n", obj->path,
489                                         strerror(errno));
490                         return -errno;
491                 }
492 
493                 obj->efile.elf = elf_begin(obj->efile.fd,
494                                 LIBBPF_ELF_C_READ_MMAP,
495                                 NULL);
496         }
497 
498         if (!obj->efile.elf) {
499                 pr_warning("failed to open %s as ELF file\n",
500                                 obj->path);
501                 err = -LIBBPF_ERRNO__LIBELF;
502                 goto errout;
503         }
504 
505         if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
506                 pr_warning("failed to get EHDR from %s\n",
507                                 obj->path);
508                 err = -LIBBPF_ERRNO__FORMAT;
509                 goto errout;
510         }
511         ep = &obj->efile.ehdr;
512 
513         /* Old LLVM set e_machine to EM_NONE */
514         if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) {
515                 pr_warning("%s is not an eBPF object file\n",
516                         obj->path);
517                 err = -LIBBPF_ERRNO__FORMAT;
518                 goto errout;
519         }
520 
521         return 0;
522 errout:
523         bpf_object__elf_finish(obj);
524         return err;
525 }
526 
527 static int
528 bpf_object__check_endianness(struct bpf_object *obj)
529 {
530         static unsigned int const endian = 1;
531 
532         switch (obj->efile.ehdr.e_ident[EI_DATA]) {
533         case ELFDATA2LSB:
534                 /* We are big endian, BPF obj is little endian. */
535                 if (*(unsigned char const *)&endian != 1)
536                         goto mismatch;
537                 break;
538 
539         case ELFDATA2MSB:
540                 /* We are little endian, BPF obj is big endian. */
541                 if (*(unsigned char const *)&endian != 0)
542                         goto mismatch;
543                 break;
544         default:
545                 return -LIBBPF_ERRNO__ENDIAN;
546         }
547 
548         return 0;
549 
550 mismatch:
551         pr_warning("Error: endianness mismatch.\n");
552         return -LIBBPF_ERRNO__ENDIAN;
553 }
554 
555 static int
556 bpf_object__init_license(struct bpf_object *obj,
557                          void *data, size_t size)
558 {
559         memcpy(obj->license, data,
560                min(size, sizeof(obj->license) - 1));
561         pr_debug("license of %s is %s\n", obj->path, obj->license);
562         return 0;
563 }
564 
565 static int
566 bpf_object__init_kversion(struct bpf_object *obj,
567                           void *data, size_t size)
568 {
569         u32 kver;
570 
571         if (size != sizeof(kver)) {
572                 pr_warning("invalid kver section in %s\n", obj->path);
573                 return -LIBBPF_ERRNO__FORMAT;
574         }
575         memcpy(&kver, data, sizeof(kver));
576         obj->kern_version = kver;
577         pr_debug("kernel version of %s is %x\n", obj->path,
578                  obj->kern_version);
579         return 0;
580 }
581 
582 static int compare_bpf_map(const void *_a, const void *_b)
583 {
584         const struct bpf_map *a = _a;
585         const struct bpf_map *b = _b;
586 
587         return a->offset - b->offset;
588 }
589 
590 static int
591 bpf_object__init_maps(struct bpf_object *obj)
592 {
593         int i, map_idx, map_def_sz, nr_maps = 0;
594         Elf_Scn *scn;
595         Elf_Data *data;
596         Elf_Data *symbols = obj->efile.symbols;
597 
598         if (obj->efile.maps_shndx < 0)
599                 return -EINVAL;
600         if (!symbols)
601                 return -EINVAL;
602 
603         scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx);
604         if (scn)
605                 data = elf_getdata(scn, NULL);
606         if (!scn || !data) {
607                 pr_warning("failed to get Elf_Data from map section %d\n",
608                            obj->efile.maps_shndx);
609                 return -EINVAL;
610         }
611 
612         /*
613          * Count number of maps. Each map has a name.
614          * Array of maps is not supported: only the first element is
615          * considered.
616          *
617          * TODO: Detect array of map and report error.
618          */
619         for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) {
620                 GElf_Sym sym;
621 
622                 if (!gelf_getsym(symbols, i, &sym))
623                         continue;
624                 if (sym.st_shndx != obj->efile.maps_shndx)
625                         continue;
626                 nr_maps++;
627         }
628 
629         /* Alloc obj->maps and fill nr_maps. */
630         pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path,
631                  nr_maps, data->d_size);
632 
633         if (!nr_maps)
634                 return 0;
635 
636         /* Assume equally sized map definitions */
637         map_def_sz = data->d_size / nr_maps;
638         if (!data->d_size || (data->d_size % nr_maps) != 0) {
639                 pr_warning("unable to determine map definition size "
640                            "section %s, %d maps in %zd bytes\n",
641                            obj->path, nr_maps, data->d_size);
642                 return -EINVAL;
643         }
644 
645         obj->maps = calloc(nr_maps, sizeof(obj->maps[0]));
646         if (!obj->maps) {
647                 pr_warning("alloc maps for object failed\n");
648                 return -ENOMEM;
649         }
650         obj->nr_maps = nr_maps;
651 
652         /*
653          * fill all fd with -1 so won't close incorrect
654          * fd (fd=0 is stdin) when failure (zclose won't close
655          * negative fd)).
656          */
657         for (i = 0; i < nr_maps; i++)
658                 obj->maps[i].fd = -1;
659 
660         /*
661          * Fill obj->maps using data in "maps" section.
662          */
663         for (i = 0, map_idx = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) {
664                 GElf_Sym sym;
665                 const char *map_name;
666                 struct bpf_map_def *def;
667 
668                 if (!gelf_getsym(symbols, i, &sym))
669                         continue;
670                 if (sym.st_shndx != obj->efile.maps_shndx)
671                         continue;
672 
673                 map_name = elf_strptr(obj->efile.elf,
674                                       obj->efile.strtabidx,
675                                       sym.st_name);
676                 obj->maps[map_idx].offset = sym.st_value;
677                 if (sym.st_value + map_def_sz > data->d_size) {
678                         pr_warning("corrupted maps section in %s: last map \"%s\" too small\n",
679                                    obj->path, map_name);
680                         return -EINVAL;
681                 }
682 
683                 obj->maps[map_idx].name = strdup(map_name);
684                 if (!obj->maps[map_idx].name) {
685                         pr_warning("failed to alloc map name\n");
686                         return -ENOMEM;
687                 }
688                 pr_debug("map %d is \"%s\"\n", map_idx,
689                          obj->maps[map_idx].name);
690                 def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
691                 /*
692                  * If the definition of the map in the object file fits in
693                  * bpf_map_def, copy it.  Any extra fields in our version
694                  * of bpf_map_def will default to zero as a result of the
695                  * calloc above.
696                  */
697                 if (map_def_sz <= sizeof(struct bpf_map_def)) {
698                         memcpy(&obj->maps[map_idx].def, def, map_def_sz);
699                 } else {
700                         /*
701                          * Here the map structure being read is bigger than what
702                          * we expect, truncate if the excess bits are all zero.
703                          * If they are not zero, reject this map as
704                          * incompatible.
705                          */
706                         char *b;
707                         for (b = ((char *)def) + sizeof(struct bpf_map_def);
708                              b < ((char *)def) + map_def_sz; b++) {
709                                 if (*b != 0) {
710                                         pr_warning("maps section in %s: \"%s\" "
711                                                    "has unrecognized, non-zero "
712                                                    "options\n",
713                                                    obj->path, map_name);
714                                         return -EINVAL;
715                                 }
716                         }
717                         memcpy(&obj->maps[map_idx].def, def,
718                                sizeof(struct bpf_map_def));
719                 }
720                 map_idx++;
721         }
722 
723         qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map);
724         return 0;
725 }
726 
727 static int bpf_object__elf_collect(struct bpf_object *obj)
728 {
729         Elf *elf = obj->efile.elf;
730         GElf_Ehdr *ep = &obj->efile.ehdr;
731         Elf_Scn *scn = NULL;
732         int idx = 0, err = 0;
733 
734         /* Elf is corrupted/truncated, avoid calling elf_strptr. */
735         if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) {
736                 pr_warning("failed to get e_shstrndx from %s\n",
737                            obj->path);
738                 return -LIBBPF_ERRNO__FORMAT;
739         }
740 
741         while ((scn = elf_nextscn(elf, scn)) != NULL) {
742                 char *name;
743                 GElf_Shdr sh;
744                 Elf_Data *data;
745 
746                 idx++;
747                 if (gelf_getshdr(scn, &sh) != &sh) {
748                         pr_warning("failed to get section header from %s\n",
749                                    obj->path);
750                         err = -LIBBPF_ERRNO__FORMAT;
751                         goto out;
752                 }
753 
754                 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
755                 if (!name) {
756                         pr_warning("failed to get section name from %s\n",
757                                    obj->path);
758                         err = -LIBBPF_ERRNO__FORMAT;
759                         goto out;
760                 }
761 
762                 data = elf_getdata(scn, 0);
763                 if (!data) {
764                         pr_warning("failed to get section data from %s(%s)\n",
765                                    name, obj->path);
766                         err = -LIBBPF_ERRNO__FORMAT;
767                         goto out;
768                 }
769                 pr_debug("section %s, size %ld, link %d, flags %lx, type=%d\n",
770                          name, (unsigned long)data->d_size,
771                          (int)sh.sh_link, (unsigned long)sh.sh_flags,
772                          (int)sh.sh_type);
773 
774                 if (strcmp(name, "license") == 0)
775                         err = bpf_object__init_license(obj,
776                                                        data->d_buf,
777                                                        data->d_size);
778                 else if (strcmp(name, "version") == 0)
779                         err = bpf_object__init_kversion(obj,
780                                                         data->d_buf,
781                                                         data->d_size);
782                 else if (strcmp(name, "maps") == 0)
783                         obj->efile.maps_shndx = idx;
784                 else if (sh.sh_type == SHT_SYMTAB) {
785                         if (obj->efile.symbols) {
786                                 pr_warning("bpf: multiple SYMTAB in %s\n",
787                                            obj->path);
788                                 err = -LIBBPF_ERRNO__FORMAT;
789                         } else {
790                                 obj->efile.symbols = data;
791                                 obj->efile.strtabidx = sh.sh_link;
792                         }
793                 } else if ((sh.sh_type == SHT_PROGBITS) &&
794                            (sh.sh_flags & SHF_EXECINSTR) &&
795                            (data->d_size > 0)) {
796                         err = bpf_object__add_program(obj, data->d_buf,
797                                                       data->d_size, name, idx);
798                         if (err) {
799                                 char errmsg[STRERR_BUFSIZE];
800 
801                                 strerror_r(-err, errmsg, sizeof(errmsg));
802                                 pr_warning("failed to alloc program %s (%s): %s",
803                                            name, obj->path, errmsg);
804                         }
805                 } else if (sh.sh_type == SHT_REL) {
806                         void *reloc = obj->efile.reloc;
807                         int nr_reloc = obj->efile.nr_reloc + 1;
808 
809                         reloc = realloc(reloc,
810                                         sizeof(*obj->efile.reloc) * nr_reloc);
811                         if (!reloc) {
812                                 pr_warning("realloc failed\n");
813                                 err = -ENOMEM;
814                         } else {
815                                 int n = nr_reloc - 1;
816 
817                                 obj->efile.reloc = reloc;
818                                 obj->efile.nr_reloc = nr_reloc;
819 
820                                 obj->efile.reloc[n].shdr = sh;
821                                 obj->efile.reloc[n].data = data;
822                         }
823                 }
824                 if (err)
825                         goto out;
826         }
827 
828         if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) {
829                 pr_warning("Corrupted ELF file: index of strtab invalid\n");
830                 return LIBBPF_ERRNO__FORMAT;
831         }
832         if (obj->efile.maps_shndx >= 0) {
833                 err = bpf_object__init_maps(obj);
834                 if (err)
835                         goto out;
836         }
837         err = bpf_object__init_prog_names(obj);
838 out:
839         return err;
840 }
841 
842 static struct bpf_program *
843 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx)
844 {
845         struct bpf_program *prog;
846         size_t i;
847 
848         for (i = 0; i < obj->nr_programs; i++) {
849                 prog = &obj->programs[i];
850                 if (prog->idx == idx)
851                         return prog;
852         }
853         return NULL;
854 }
855 
856 static int
857 bpf_program__collect_reloc(struct bpf_program *prog,
858                            size_t nr_maps, GElf_Shdr *shdr,
859                            Elf_Data *data, Elf_Data *symbols,
860                            int maps_shndx, struct bpf_map *maps)
861 {
862         int i, nrels;
863 
864         pr_debug("collecting relocating info for: '%s'\n",
865                  prog->section_name);
866         nrels = shdr->sh_size / shdr->sh_entsize;
867 
868         prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels);
869         if (!prog->reloc_desc) {
870                 pr_warning("failed to alloc memory in relocation\n");
871                 return -ENOMEM;
872         }
873         prog->nr_reloc = nrels;
874 
875         for (i = 0; i < nrels; i++) {
876                 GElf_Sym sym;
877                 GElf_Rel rel;
878                 unsigned int insn_idx;
879                 struct bpf_insn *insns = prog->insns;
880                 size_t map_idx;
881 
882                 if (!gelf_getrel(data, i, &rel)) {
883                         pr_warning("relocation: failed to get %d reloc\n", i);
884                         return -LIBBPF_ERRNO__FORMAT;
885                 }
886 
887                 if (!gelf_getsym(symbols,
888                                  GELF_R_SYM(rel.r_info),
889                                  &sym)) {
890                         pr_warning("relocation: symbol %"PRIx64" not found\n",
891                                    GELF_R_SYM(rel.r_info));
892                         return -LIBBPF_ERRNO__FORMAT;
893                 }
894 
895                 if (sym.st_shndx != maps_shndx) {
896                         pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n",
897                                    prog->section_name, sym.st_shndx);
898                         return -LIBBPF_ERRNO__RELOC;
899                 }
900 
901                 insn_idx = rel.r_offset / sizeof(struct bpf_insn);
902                 pr_debug("relocation: insn_idx=%u\n", insn_idx);
903 
904                 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) {
905                         pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",
906                                    insn_idx, insns[insn_idx].code);
907                         return -LIBBPF_ERRNO__RELOC;
908                 }
909 
910                 /* TODO: 'maps' is sorted. We can use bsearch to make it faster. */
911                 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
912                         if (maps[map_idx].offset == sym.st_value) {
913                                 pr_debug("relocation: find map %zd (%s) for insn %u\n",
914                                          map_idx, maps[map_idx].name, insn_idx);
915                                 break;
916                         }
917                 }
918 
919                 if (map_idx >= nr_maps) {
920                         pr_warning("bpf relocation: map_idx %d large than %d\n",
921                                    (int)map_idx, (int)nr_maps - 1);
922                         return -LIBBPF_ERRNO__RELOC;
923                 }
924 
925                 prog->reloc_desc[i].insn_idx = insn_idx;
926                 prog->reloc_desc[i].map_idx = map_idx;
927         }
928         return 0;
929 }
930 
931 static int
932 bpf_object__create_maps(struct bpf_object *obj)
933 {
934         unsigned int i;
935 
936         for (i = 0; i < obj->nr_maps; i++) {
937                 struct bpf_map_def *def = &obj->maps[i].def;
938                 int *pfd = &obj->maps[i].fd;
939 
940                 *pfd = bpf_create_map_name(def->type,
941                                            obj->maps[i].name,
942                                            def->key_size,
943                                            def->value_size,
944                                            def->max_entries,
945                                            def->map_flags);
946                 if (*pfd < 0) {
947                         size_t j;
948                         int err = *pfd;
949 
950                         pr_warning("failed to create map (name: '%s'): %s\n",
951                                    obj->maps[i].name,
952                                    strerror(errno));
953                         for (j = 0; j < i; j++)
954                                 zclose(obj->maps[j].fd);
955                         return err;
956                 }
957                 pr_debug("create map %s: fd=%d\n", obj->maps[i].name, *pfd);
958         }
959 
960         return 0;
961 }
962 
963 static int
964 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj)
965 {
966         int i;
967 
968         if (!prog || !prog->reloc_desc)
969                 return 0;
970 
971         for (i = 0; i < prog->nr_reloc; i++) {
972                 int insn_idx, map_idx;
973                 struct bpf_insn *insns = prog->insns;
974 
975                 insn_idx = prog->reloc_desc[i].insn_idx;
976                 map_idx = prog->reloc_desc[i].map_idx;
977 
978                 if (insn_idx >= (int)prog->insns_cnt) {
979                         pr_warning("relocation out of range: '%s'\n",
980                                    prog->section_name);
981                         return -LIBBPF_ERRNO__RELOC;
982                 }
983                 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD;
984                 insns[insn_idx].imm = obj->maps[map_idx].fd;
985         }
986 
987         zfree(&prog->reloc_desc);
988         prog->nr_reloc = 0;
989         return 0;
990 }
991 
992 
993 static int
994 bpf_object__relocate(struct bpf_object *obj)
995 {
996         struct bpf_program *prog;
997         size_t i;
998         int err;
999 
1000         for (i = 0; i < obj->nr_programs; i++) {
1001                 prog = &obj->programs[i];
1002 
1003                 err = bpf_program__relocate(prog, obj);
1004                 if (err) {
1005                         pr_warning("failed to relocate '%s'\n",
1006                                    prog->section_name);
1007                         return err;
1008                 }
1009         }
1010         return 0;
1011 }
1012 
1013 static int bpf_object__collect_reloc(struct bpf_object *obj)
1014 {
1015         int i, err;
1016 
1017         if (!obj_elf_valid(obj)) {
1018                 pr_warning("Internal error: elf object is closed\n");
1019                 return -LIBBPF_ERRNO__INTERNAL;
1020         }
1021 
1022         for (i = 0; i < obj->efile.nr_reloc; i++) {
1023                 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr;
1024                 Elf_Data *data = obj->efile.reloc[i].data;
1025                 int idx = shdr->sh_info;
1026                 struct bpf_program *prog;
1027                 size_t nr_maps = obj->nr_maps;
1028 
1029                 if (shdr->sh_type != SHT_REL) {
1030                         pr_warning("internal error at %d\n", __LINE__);
1031                         return -LIBBPF_ERRNO__INTERNAL;
1032                 }
1033 
1034                 prog = bpf_object__find_prog_by_idx(obj, idx);
1035                 if (!prog) {
1036                         pr_warning("relocation failed: no %d section\n",
1037                                    idx);
1038                         return -LIBBPF_ERRNO__RELOC;
1039                 }
1040 
1041                 err = bpf_program__collect_reloc(prog, nr_maps,
1042                                                  shdr, data,
1043                                                  obj->efile.symbols,
1044                                                  obj->efile.maps_shndx,
1045                                                  obj->maps);
1046                 if (err)
1047                         return err;
1048         }
1049         return 0;
1050 }
1051 
1052 static int
1053 load_program(enum bpf_prog_type type, const char *name, struct bpf_insn *insns,
1054              int insns_cnt, char *license, u32 kern_version, int *pfd)
1055 {
1056         int ret;
1057         char *log_buf;
1058 
1059         if (!insns || !insns_cnt)
1060                 return -EINVAL;
1061 
1062         log_buf = malloc(BPF_LOG_BUF_SIZE);
1063         if (!log_buf)
1064                 pr_warning("Alloc log buffer for bpf loader error, continue without log\n");
1065 
1066         ret = bpf_load_program_name(type, name, insns, insns_cnt, license,
1067                                     kern_version, log_buf, BPF_LOG_BUF_SIZE);
1068 
1069         if (ret >= 0) {
1070                 *pfd = ret;
1071                 ret = 0;
1072                 goto out;
1073         }
1074 
1075         ret = -LIBBPF_ERRNO__LOAD;
1076         pr_warning("load bpf program failed: %s\n", strerror(errno));
1077 
1078         if (log_buf && log_buf[0] != '\0') {
1079                 ret = -LIBBPF_ERRNO__VERIFY;
1080                 pr_warning("-- BEGIN DUMP LOG ---\n");
1081                 pr_warning("\n%s\n", log_buf);
1082                 pr_warning("-- END LOG --\n");
1083         } else if (insns_cnt >= BPF_MAXINSNS) {
1084                 pr_warning("Program too large (%d insns), at most %d insns\n",
1085                            insns_cnt, BPF_MAXINSNS);
1086                 ret = -LIBBPF_ERRNO__PROG2BIG;
1087         } else {
1088                 /* Wrong program type? */
1089                 if (type != BPF_PROG_TYPE_KPROBE) {
1090                         int fd;
1091 
1092                         fd = bpf_load_program_name(BPF_PROG_TYPE_KPROBE, name,
1093                                                    insns, insns_cnt, license,
1094                                                    kern_version, NULL, 0);
1095                         if (fd >= 0) {
1096                                 close(fd);
1097                                 ret = -LIBBPF_ERRNO__PROGTYPE;
1098                                 goto out;
1099                         }
1100                 }
1101 
1102                 if (log_buf)
1103                         ret = -LIBBPF_ERRNO__KVER;
1104         }
1105 
1106 out:
1107         free(log_buf);
1108         return ret;
1109 }
1110 
1111 static int
1112 bpf_program__load(struct bpf_program *prog,
1113                   char *license, u32 kern_version)
1114 {
1115         int err = 0, fd, i;
1116 
1117         if (prog->instances.nr < 0 || !prog->instances.fds) {
1118                 if (prog->preprocessor) {
1119                         pr_warning("Internal error: can't load program '%s'\n",
1120                                    prog->section_name);
1121                         return -LIBBPF_ERRNO__INTERNAL;
1122                 }
1123 
1124                 prog->instances.fds = malloc(sizeof(int));
1125                 if (!prog->instances.fds) {
1126                         pr_warning("Not enough memory for BPF fds\n");
1127                         return -ENOMEM;
1128                 }
1129                 prog->instances.nr = 1;
1130                 prog->instances.fds[0] = -1;
1131         }
1132 
1133         if (!prog->preprocessor) {
1134                 if (prog->instances.nr != 1) {
1135                         pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n",
1136                                    prog->section_name, prog->instances.nr);
1137                 }
1138                 err = load_program(prog->type, prog->name, prog->insns,
1139                                    prog->insns_cnt, license, kern_version, &fd);
1140                 if (!err)
1141                         prog->instances.fds[0] = fd;
1142                 goto out;
1143         }
1144 
1145         for (i = 0; i < prog->instances.nr; i++) {
1146                 struct bpf_prog_prep_result result;
1147                 bpf_program_prep_t preprocessor = prog->preprocessor;
1148 
1149                 bzero(&result, sizeof(result));
1150                 err = preprocessor(prog, i, prog->insns,
1151                                    prog->insns_cnt, &result);
1152                 if (err) {
1153                         pr_warning("Preprocessing the %dth instance of program '%s' failed\n",
1154                                    i, prog->section_name);
1155                         goto out;
1156                 }
1157 
1158                 if (!result.new_insn_ptr || !result.new_insn_cnt) {
1159                         pr_debug("Skip loading the %dth instance of program '%s'\n",
1160                                  i, prog->section_name);
1161                         prog->instances.fds[i] = -1;
1162                         if (result.pfd)
1163                                 *result.pfd = -1;
1164                         continue;
1165                 }
1166 
1167                 err = load_program(prog->type, prog->name,
1168                                    result.new_insn_ptr,
1169                                    result.new_insn_cnt,
1170                                    license, kern_version, &fd);
1171 
1172                 if (err) {
1173                         pr_warning("Loading the %dth instance of program '%s' failed\n",
1174                                         i, prog->section_name);
1175                         goto out;
1176                 }
1177 
1178                 if (result.pfd)
1179                         *result.pfd = fd;
1180                 prog->instances.fds[i] = fd;
1181         }
1182 out:
1183         if (err)
1184                 pr_warning("failed to load program '%s'\n",
1185                            prog->section_name);
1186         zfree(&prog->insns);
1187         prog->insns_cnt = 0;
1188         return err;
1189 }
1190 
1191 static int
1192 bpf_object__load_progs(struct bpf_object *obj)
1193 {
1194         size_t i;
1195         int err;
1196 
1197         for (i = 0; i < obj->nr_programs; i++) {
1198                 err = bpf_program__load(&obj->programs[i],
1199                                         obj->license,
1200                                         obj->kern_version);
1201                 if (err)
1202                         return err;
1203         }
1204         return 0;
1205 }
1206 
1207 static int bpf_object__validate(struct bpf_object *obj)
1208 {
1209         if (obj->kern_version == 0) {
1210                 pr_warning("%s doesn't provide kernel version\n",
1211                            obj->path);
1212                 return -LIBBPF_ERRNO__KVERSION;
1213         }
1214         return 0;
1215 }
1216 
1217 static struct bpf_object *
1218 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz)
1219 {
1220         struct bpf_object *obj;
1221         int err;
1222 
1223         if (elf_version(EV_CURRENT) == EV_NONE) {
1224                 pr_warning("failed to init libelf for %s\n", path);
1225                 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
1226         }
1227 
1228         obj = bpf_object__new(path, obj_buf, obj_buf_sz);
1229         if (IS_ERR(obj))
1230                 return obj;
1231 
1232         CHECK_ERR(bpf_object__elf_init(obj), err, out);
1233         CHECK_ERR(bpf_object__check_endianness(obj), err, out);
1234         CHECK_ERR(bpf_object__elf_collect(obj), err, out);
1235         CHECK_ERR(bpf_object__collect_reloc(obj), err, out);
1236         CHECK_ERR(bpf_object__validate(obj), err, out);
1237 
1238         bpf_object__elf_finish(obj);
1239         return obj;
1240 out:
1241         bpf_object__close(obj);
1242         return ERR_PTR(err);
1243 }
1244 
1245 struct bpf_object *bpf_object__open(const char *path)
1246 {
1247         /* param validation */
1248         if (!path)
1249                 return NULL;
1250 
1251         pr_debug("loading %s\n", path);
1252 
1253         return __bpf_object__open(path, NULL, 0);
1254 }
1255 
1256 struct bpf_object *bpf_object__open_buffer(void *obj_buf,
1257                                            size_t obj_buf_sz,
1258                                            const char *name)
1259 {
1260         char tmp_name[64];
1261 
1262         /* param validation */
1263         if (!obj_buf || obj_buf_sz <= 0)
1264                 return NULL;
1265 
1266         if (!name) {
1267                 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
1268                          (unsigned long)obj_buf,
1269                          (unsigned long)obj_buf_sz);
1270                 tmp_name[sizeof(tmp_name) - 1] = '\0';
1271                 name = tmp_name;
1272         }
1273         pr_debug("loading object '%s' from buffer\n",
1274                  name);
1275 
1276         return __bpf_object__open(name, obj_buf, obj_buf_sz);
1277 }
1278 
1279 int bpf_object__unload(struct bpf_object *obj)
1280 {
1281         size_t i;
1282 
1283         if (!obj)
1284                 return -EINVAL;
1285 
1286         for (i = 0; i < obj->nr_maps; i++)
1287                 zclose(obj->maps[i].fd);
1288 
1289         for (i = 0; i < obj->nr_programs; i++)
1290                 bpf_program__unload(&obj->programs[i]);
1291 
1292         return 0;
1293 }
1294 
1295 int bpf_object__load(struct bpf_object *obj)
1296 {
1297         int err;
1298 
1299         if (!obj)
1300                 return -EINVAL;
1301 
1302         if (obj->loaded) {
1303                 pr_warning("object should not be loaded twice\n");
1304                 return -EINVAL;
1305         }
1306 
1307         obj->loaded = true;
1308 
1309         CHECK_ERR(bpf_object__create_maps(obj), err, out);
1310         CHECK_ERR(bpf_object__relocate(obj), err, out);
1311         CHECK_ERR(bpf_object__load_progs(obj), err, out);
1312 
1313         return 0;
1314 out:
1315         bpf_object__unload(obj);
1316         pr_warning("failed to load object '%s'\n", obj->path);
1317         return err;
1318 }
1319 
1320 static int check_path(const char *path)
1321 {
1322         struct statfs st_fs;
1323         char *dname, *dir;
1324         int err = 0;
1325 
1326         if (path == NULL)
1327                 return -EINVAL;
1328 
1329         dname = strdup(path);
1330         if (dname == NULL)
1331                 return -ENOMEM;
1332 
1333         dir = dirname(dname);
1334         if (statfs(dir, &st_fs)) {
1335                 pr_warning("failed to statfs %s: %s\n", dir, strerror(errno));
1336                 err = -errno;
1337         }
1338         free(dname);
1339 
1340         if (!err && st_fs.f_type != BPF_FS_MAGIC) {
1341                 pr_warning("specified path %s is not on BPF FS\n", path);
1342                 err = -EINVAL;
1343         }
1344 
1345         return err;
1346 }
1347 
1348 int bpf_program__pin_instance(struct bpf_program *prog, const char *path,
1349                               int instance)
1350 {
1351         int err;
1352 
1353         err = check_path(path);
1354         if (err)
1355                 return err;
1356 
1357         if (prog == NULL) {
1358                 pr_warning("invalid program pointer\n");
1359                 return -EINVAL;
1360         }
1361 
1362         if (instance < 0 || instance >= prog->instances.nr) {
1363                 pr_warning("invalid prog instance %d of prog %s (max %d)\n",
1364                            instance, prog->section_name, prog->instances.nr);
1365                 return -EINVAL;
1366         }
1367 
1368         if (bpf_obj_pin(prog->instances.fds[instance], path)) {
1369                 pr_warning("failed to pin program: %s\n", strerror(errno));
1370                 return -errno;
1371         }
1372         pr_debug("pinned program '%s'\n", path);
1373 
1374         return 0;
1375 }
1376 
1377 static int make_dir(const char *path)
1378 {
1379         int err = 0;
1380 
1381         if (mkdir(path, 0700) && errno != EEXIST)
1382                 err = -errno;
1383 
1384         if (err)
1385                 pr_warning("failed to mkdir %s: %s\n", path, strerror(-err));
1386         return err;
1387 }
1388 
1389 int bpf_program__pin(struct bpf_program *prog, const char *path)
1390 {
1391         int i, err;
1392 
1393         err = check_path(path);
1394         if (err)
1395                 return err;
1396 
1397         if (prog == NULL) {
1398                 pr_warning("invalid program pointer\n");
1399                 return -EINVAL;
1400         }
1401 
1402         if (prog->instances.nr <= 0) {
1403                 pr_warning("no instances of prog %s to pin\n",
1404                            prog->section_name);
1405                 return -EINVAL;
1406         }
1407 
1408         err = make_dir(path);
1409         if (err)
1410                 return err;
1411 
1412         for (i = 0; i < prog->instances.nr; i++) {
1413                 char buf[PATH_MAX];
1414                 int len;
1415 
1416                 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
1417                 if (len < 0)
1418                         return -EINVAL;
1419                 else if (len >= PATH_MAX)
1420                         return -ENAMETOOLONG;
1421 
1422                 err = bpf_program__pin_instance(prog, buf, i);
1423                 if (err)
1424                         return err;
1425         }
1426 
1427         return 0;
1428 }
1429 
1430 int bpf_map__pin(struct bpf_map *map, const char *path)
1431 {
1432         int err;
1433 
1434         err = check_path(path);
1435         if (err)
1436                 return err;
1437 
1438         if (map == NULL) {
1439                 pr_warning("invalid map pointer\n");
1440                 return -EINVAL;
1441         }
1442 
1443         if (bpf_obj_pin(map->fd, path)) {
1444                 pr_warning("failed to pin map: %s\n", strerror(errno));
1445                 return -errno;
1446         }
1447 
1448         pr_debug("pinned map '%s'\n", path);
1449         return 0;
1450 }
1451 
1452 int bpf_object__pin(struct bpf_object *obj, const char *path)
1453 {
1454         struct bpf_program *prog;
1455         struct bpf_map *map;
1456         int err;
1457 
1458         if (!obj)
1459                 return -ENOENT;
1460 
1461         if (!obj->loaded) {
1462                 pr_warning("object not yet loaded; load it first\n");
1463                 return -ENOENT;
1464         }
1465 
1466         err = make_dir(path);
1467         if (err)
1468                 return err;
1469 
1470         bpf_map__for_each(map, obj) {
1471                 char buf[PATH_MAX];
1472                 int len;
1473 
1474                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
1475                                bpf_map__name(map));
1476                 if (len < 0)
1477                         return -EINVAL;
1478                 else if (len >= PATH_MAX)
1479                         return -ENAMETOOLONG;
1480 
1481                 err = bpf_map__pin(map, buf);
1482                 if (err)
1483                         return err;
1484         }
1485 
1486         bpf_object__for_each_program(prog, obj) {
1487                 char buf[PATH_MAX];
1488                 int len;
1489 
1490                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
1491                                prog->section_name);
1492                 if (len < 0)
1493                         return -EINVAL;
1494                 else if (len >= PATH_MAX)
1495                         return -ENAMETOOLONG;
1496 
1497                 err = bpf_program__pin(prog, buf);
1498                 if (err)
1499                         return err;
1500         }
1501 
1502         return 0;
1503 }
1504 
1505 void bpf_object__close(struct bpf_object *obj)
1506 {
1507         size_t i;
1508 
1509         if (!obj)
1510                 return;
1511 
1512         if (obj->clear_priv)
1513                 obj->clear_priv(obj, obj->priv);
1514 
1515         bpf_object__elf_finish(obj);
1516         bpf_object__unload(obj);
1517 
1518         for (i = 0; i < obj->nr_maps; i++) {
1519                 zfree(&obj->maps[i].name);
1520                 if (obj->maps[i].clear_priv)
1521                         obj->maps[i].clear_priv(&obj->maps[i],
1522                                                 obj->maps[i].priv);
1523                 obj->maps[i].priv = NULL;
1524                 obj->maps[i].clear_priv = NULL;
1525         }
1526         zfree(&obj->maps);
1527         obj->nr_maps = 0;
1528 
1529         if (obj->programs && obj->nr_programs) {
1530                 for (i = 0; i < obj->nr_programs; i++)
1531                         bpf_program__exit(&obj->programs[i]);
1532         }
1533         zfree(&obj->programs);
1534 
1535         list_del(&obj->list);
1536         free(obj);
1537 }
1538 
1539 struct bpf_object *
1540 bpf_object__next(struct bpf_object *prev)
1541 {
1542         struct bpf_object *next;
1543 
1544         if (!prev)
1545                 next = list_first_entry(&bpf_objects_list,
1546                                         struct bpf_object,
1547                                         list);
1548         else
1549                 next = list_next_entry(prev, list);
1550 
1551         /* Empty list is noticed here so don't need checking on entry. */
1552         if (&next->list == &bpf_objects_list)
1553                 return NULL;
1554 
1555         return next;
1556 }
1557 
1558 const char *bpf_object__name(struct bpf_object *obj)
1559 {
1560         return obj ? obj->path : ERR_PTR(-EINVAL);
1561 }
1562 
1563 unsigned int bpf_object__kversion(struct bpf_object *obj)
1564 {
1565         return obj ? obj->kern_version : 0;
1566 }
1567 
1568 int bpf_object__set_priv(struct bpf_object *obj, void *priv,
1569                          bpf_object_clear_priv_t clear_priv)
1570 {
1571         if (obj->priv && obj->clear_priv)
1572                 obj->clear_priv(obj, obj->priv);
1573 
1574         obj->priv = priv;
1575         obj->clear_priv = clear_priv;
1576         return 0;
1577 }
1578 
1579 void *bpf_object__priv(struct bpf_object *obj)
1580 {
1581         return obj ? obj->priv : ERR_PTR(-EINVAL);
1582 }
1583 
1584 struct bpf_program *
1585 bpf_program__next(struct bpf_program *prev, struct bpf_object *obj)
1586 {
1587         size_t idx;
1588 
1589         if (!obj->programs)
1590                 return NULL;
1591         /* First handler */
1592         if (prev == NULL)
1593                 return &obj->programs[0];
1594 
1595         if (prev->obj != obj) {
1596                 pr_warning("error: program handler doesn't match object\n");
1597                 return NULL;
1598         }
1599 
1600         idx = (prev - obj->programs) + 1;
1601         if (idx >= obj->nr_programs)
1602                 return NULL;
1603         return &obj->programs[idx];
1604 }
1605 
1606 int bpf_program__set_priv(struct bpf_program *prog, void *priv,
1607                           bpf_program_clear_priv_t clear_priv)
1608 {
1609         if (prog->priv && prog->clear_priv)
1610                 prog->clear_priv(prog, prog->priv);
1611 
1612         prog->priv = priv;
1613         prog->clear_priv = clear_priv;
1614         return 0;
1615 }
1616 
1617 void *bpf_program__priv(struct bpf_program *prog)
1618 {
1619         return prog ? prog->priv : ERR_PTR(-EINVAL);
1620 }
1621 
1622 const char *bpf_program__title(struct bpf_program *prog, bool needs_copy)
1623 {
1624         const char *title;
1625 
1626         title = prog->section_name;
1627         if (needs_copy) {
1628                 title = strdup(title);
1629                 if (!title) {
1630                         pr_warning("failed to strdup program title\n");
1631                         return ERR_PTR(-ENOMEM);
1632                 }
1633         }
1634 
1635         return title;
1636 }
1637 
1638 int bpf_program__fd(struct bpf_program *prog)
1639 {
1640         return bpf_program__nth_fd(prog, 0);
1641 }
1642 
1643 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
1644                           bpf_program_prep_t prep)
1645 {
1646         int *instances_fds;
1647 
1648         if (nr_instances <= 0 || !prep)
1649                 return -EINVAL;
1650 
1651         if (prog->instances.nr > 0 || prog->instances.fds) {
1652                 pr_warning("Can't set pre-processor after loading\n");
1653                 return -EINVAL;
1654         }
1655 
1656         instances_fds = malloc(sizeof(int) * nr_instances);
1657         if (!instances_fds) {
1658                 pr_warning("alloc memory failed for fds\n");
1659                 return -ENOMEM;
1660         }
1661 
1662         /* fill all fd with -1 */
1663         memset(instances_fds, -1, sizeof(int) * nr_instances);
1664 
1665         prog->instances.nr = nr_instances;
1666         prog->instances.fds = instances_fds;
1667         prog->preprocessor = prep;
1668         return 0;
1669 }
1670 
1671 int bpf_program__nth_fd(struct bpf_program *prog, int n)
1672 {
1673         int fd;
1674 
1675         if (n >= prog->instances.nr || n < 0) {
1676                 pr_warning("Can't get the %dth fd from program %s: only %d instances\n",
1677                            n, prog->section_name, prog->instances.nr);
1678                 return -EINVAL;
1679         }
1680 
1681         fd = prog->instances.fds[n];
1682         if (fd < 0) {
1683                 pr_warning("%dth instance of program '%s' is invalid\n",
1684                            n, prog->section_name);
1685                 return -ENOENT;
1686         }
1687 
1688         return fd;
1689 }
1690 
1691 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
1692 {
1693         prog->type = type;
1694 }
1695 
1696 static bool bpf_program__is_type(struct bpf_program *prog,
1697                                  enum bpf_prog_type type)
1698 {
1699         return prog ? (prog->type == type) : false;
1700 }
1701 
1702 #define BPF_PROG_TYPE_FNS(NAME, TYPE)                   \
1703 int bpf_program__set_##NAME(struct bpf_program *prog)   \
1704 {                                                       \
1705         if (!prog)                                      \
1706                 return -EINVAL;                         \
1707         bpf_program__set_type(prog, TYPE);              \
1708         return 0;                                       \
1709 }                                                       \
1710                                                         \
1711 bool bpf_program__is_##NAME(struct bpf_program *prog)   \
1712 {                                                       \
1713         return bpf_program__is_type(prog, TYPE);        \
1714 }                                                       \
1715 
1716 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
1717 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
1718 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS);
1719 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT);
1720 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
1721 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
1722 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
1723 
1724 int bpf_map__fd(struct bpf_map *map)
1725 {
1726         return map ? map->fd : -EINVAL;
1727 }
1728 
1729 const struct bpf_map_def *bpf_map__def(struct bpf_map *map)
1730 {
1731         return map ? &map->def : ERR_PTR(-EINVAL);
1732 }
1733 
1734 const char *bpf_map__name(struct bpf_map *map)
1735 {
1736         return map ? map->name : NULL;
1737 }
1738 
1739 int bpf_map__set_priv(struct bpf_map *map, void *priv,
1740                      bpf_map_clear_priv_t clear_priv)
1741 {
1742         if (!map)
1743                 return -EINVAL;
1744 
1745         if (map->priv) {
1746                 if (map->clear_priv)
1747                         map->clear_priv(map, map->priv);
1748         }
1749 
1750         map->priv = priv;
1751         map->clear_priv = clear_priv;
1752         return 0;
1753 }
1754 
1755 void *bpf_map__priv(struct bpf_map *map)
1756 {
1757         return map ? map->priv : ERR_PTR(-EINVAL);
1758 }
1759 
1760 struct bpf_map *
1761 bpf_map__next(struct bpf_map *prev, struct bpf_object *obj)
1762 {
1763         size_t idx;
1764         struct bpf_map *s, *e;
1765 
1766         if (!obj || !obj->maps)
1767                 return NULL;
1768 
1769         s = obj->maps;
1770         e = obj->maps + obj->nr_maps;
1771 
1772         if (prev == NULL)
1773                 return s;
1774 
1775         if ((prev < s) || (prev >= e)) {
1776                 pr_warning("error in %s: map handler doesn't belong to object\n",
1777                            __func__);
1778                 return NULL;
1779         }
1780 
1781         idx = (prev - obj->maps) + 1;
1782         if (idx >= obj->nr_maps)
1783                 return NULL;
1784         return &obj->maps[idx];
1785 }
1786 
1787 struct bpf_map *
1788 bpf_object__find_map_by_name(struct bpf_object *obj, const char *name)
1789 {
1790         struct bpf_map *pos;
1791 
1792         bpf_map__for_each(pos, obj) {
1793                 if (pos->name && !strcmp(pos->name, name))
1794                         return pos;
1795         }
1796         return NULL;
1797 }
1798 
1799 struct bpf_map *
1800 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
1801 {
1802         int i;
1803 
1804         for (i = 0; i < obj->nr_maps; i++) {
1805                 if (obj->maps[i].offset == offset)
1806                         return &obj->maps[i];
1807         }
1808         return ERR_PTR(-ENOENT);
1809 }
1810 
1811 long libbpf_get_error(const void *ptr)
1812 {
1813         if (IS_ERR(ptr))
1814                 return PTR_ERR(ptr);
1815         return 0;
1816 }
1817 
1818 int bpf_prog_load(const char *file, enum bpf_prog_type type,
1819                   struct bpf_object **pobj, int *prog_fd)
1820 {
1821         struct bpf_program *prog;
1822         struct bpf_object *obj;
1823         int err;
1824 
1825         obj = bpf_object__open(file);
1826         if (IS_ERR(obj))
1827                 return -ENOENT;
1828 
1829         prog = bpf_program__next(NULL, obj);
1830         if (!prog) {
1831                 bpf_object__close(obj);
1832                 return -ENOENT;
1833         }
1834 
1835         bpf_program__set_type(prog, type);
1836         err = bpf_object__load(obj);
1837         if (err) {
1838                 bpf_object__close(obj);
1839                 return -EINVAL;
1840         }
1841 
1842         *pobj = obj;
1843         *prog_fd = bpf_program__fd(prog);
1844         return 0;
1845 }
1846 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp