~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/tools/lib/bpf/btf.c

Version: ~ [ linux-5.14-rc1 ] ~ [ linux-5.13.1 ] ~ [ linux-5.12.16 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.49 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.131 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.197 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.239 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.275 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.275 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
  2 /* Copyright (c) 2018 Facebook */
  3 
  4 #include <byteswap.h>
  5 #include <endian.h>
  6 #include <stdio.h>
  7 #include <stdlib.h>
  8 #include <string.h>
  9 #include <fcntl.h>
 10 #include <unistd.h>
 11 #include <errno.h>
 12 #include <sys/utsname.h>
 13 #include <sys/param.h>
 14 #include <sys/stat.h>
 15 #include <linux/kernel.h>
 16 #include <linux/err.h>
 17 #include <linux/btf.h>
 18 #include <gelf.h>
 19 #include "btf.h"
 20 #include "bpf.h"
 21 #include "libbpf.h"
 22 #include "libbpf_internal.h"
 23 #include "hashmap.h"
 24 #include "strset.h"
 25 
 26 #define BTF_MAX_NR_TYPES 0x7fffffffU
 27 #define BTF_MAX_STR_OFFSET 0x7fffffffU
 28 
 29 static struct btf_type btf_void;
 30 
 31 struct btf {
 32         /* raw BTF data in native endianness */
 33         void *raw_data;
 34         /* raw BTF data in non-native endianness */
 35         void *raw_data_swapped;
 36         __u32 raw_size;
 37         /* whether target endianness differs from the native one */
 38         bool swapped_endian;
 39 
 40         /*
 41          * When BTF is loaded from an ELF or raw memory it is stored
 42          * in a contiguous memory block. The hdr, type_data, and, strs_data
 43          * point inside that memory region to their respective parts of BTF
 44          * representation:
 45          *
 46          * +--------------------------------+
 47          * |  Header  |  Types  |  Strings  |
 48          * +--------------------------------+
 49          * ^          ^         ^
 50          * |          |         |
 51          * hdr        |         |
 52          * types_data-+         |
 53          * strs_data------------+
 54          *
 55          * If BTF data is later modified, e.g., due to types added or
 56          * removed, BTF deduplication performed, etc, this contiguous
 57          * representation is broken up into three independently allocated
 58          * memory regions to be able to modify them independently.
 59          * raw_data is nulled out at that point, but can be later allocated
 60          * and cached again if user calls btf__get_raw_data(), at which point
 61          * raw_data will contain a contiguous copy of header, types, and
 62          * strings:
 63          *
 64          * +----------+  +---------+  +-----------+
 65          * |  Header  |  |  Types  |  |  Strings  |
 66          * +----------+  +---------+  +-----------+
 67          * ^             ^            ^
 68          * |             |            |
 69          * hdr           |            |
 70          * types_data----+            |
 71          * strset__data(strs_set)-----+
 72          *
 73          *               +----------+---------+-----------+
 74          *               |  Header  |  Types  |  Strings  |
 75          * raw_data----->+----------+---------+-----------+
 76          */
 77         struct btf_header *hdr;
 78 
 79         void *types_data;
 80         size_t types_data_cap; /* used size stored in hdr->type_len */
 81 
 82         /* type ID to `struct btf_type *` lookup index
 83          * type_offs[0] corresponds to the first non-VOID type:
 84          *   - for base BTF it's type [1];
 85          *   - for split BTF it's the first non-base BTF type.
 86          */
 87         __u32 *type_offs;
 88         size_t type_offs_cap;
 89         /* number of types in this BTF instance:
 90          *   - doesn't include special [0] void type;
 91          *   - for split BTF counts number of types added on top of base BTF.
 92          */
 93         __u32 nr_types;
 94         /* if not NULL, points to the base BTF on top of which the current
 95          * split BTF is based
 96          */
 97         struct btf *base_btf;
 98         /* BTF type ID of the first type in this BTF instance:
 99          *   - for base BTF it's equal to 1;
100          *   - for split BTF it's equal to biggest type ID of base BTF plus 1.
101          */
102         int start_id;
103         /* logical string offset of this BTF instance:
104          *   - for base BTF it's equal to 0;
105          *   - for split BTF it's equal to total size of base BTF's string section size.
106          */
107         int start_str_off;
108 
109         /* only one of strs_data or strs_set can be non-NULL, depending on
110          * whether BTF is in a modifiable state (strs_set is used) or not
111          * (strs_data points inside raw_data)
112          */
113         void *strs_data;
114         /* a set of unique strings */
115         struct strset *strs_set;
116         /* whether strings are already deduplicated */
117         bool strs_deduped;
118 
119         /* BTF object FD, if loaded into kernel */
120         int fd;
121 
122         /* Pointer size (in bytes) for a target architecture of this BTF */
123         int ptr_sz;
124 };
125 
126 static inline __u64 ptr_to_u64(const void *ptr)
127 {
128         return (__u64) (unsigned long) ptr;
129 }
130 
131 /* Ensure given dynamically allocated memory region pointed to by *data* with
132  * capacity of *cap_cnt* elements each taking *elem_sz* bytes has enough
133  * memory to accomodate *add_cnt* new elements, assuming *cur_cnt* elements
134  * are already used. At most *max_cnt* elements can be ever allocated.
135  * If necessary, memory is reallocated and all existing data is copied over,
136  * new pointer to the memory region is stored at *data, new memory region
137  * capacity (in number of elements) is stored in *cap.
138  * On success, memory pointer to the beginning of unused memory is returned.
139  * On error, NULL is returned.
140  */
141 void *libbpf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz,
142                      size_t cur_cnt, size_t max_cnt, size_t add_cnt)
143 {
144         size_t new_cnt;
145         void *new_data;
146 
147         if (cur_cnt + add_cnt <= *cap_cnt)
148                 return *data + cur_cnt * elem_sz;
149 
150         /* requested more than the set limit */
151         if (cur_cnt + add_cnt > max_cnt)
152                 return NULL;
153 
154         new_cnt = *cap_cnt;
155         new_cnt += new_cnt / 4;           /* expand by 25% */
156         if (new_cnt < 16)                 /* but at least 16 elements */
157                 new_cnt = 16;
158         if (new_cnt > max_cnt)            /* but not exceeding a set limit */
159                 new_cnt = max_cnt;
160         if (new_cnt < cur_cnt + add_cnt)  /* also ensure we have enough memory */
161                 new_cnt = cur_cnt + add_cnt;
162 
163         new_data = libbpf_reallocarray(*data, new_cnt, elem_sz);
164         if (!new_data)
165                 return NULL;
166 
167         /* zero out newly allocated portion of memory */
168         memset(new_data + (*cap_cnt) * elem_sz, 0, (new_cnt - *cap_cnt) * elem_sz);
169 
170         *data = new_data;
171         *cap_cnt = new_cnt;
172         return new_data + cur_cnt * elem_sz;
173 }
174 
175 /* Ensure given dynamically allocated memory region has enough allocated space
176  * to accommodate *need_cnt* elements of size *elem_sz* bytes each
177  */
178 int libbpf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt)
179 {
180         void *p;
181 
182         if (need_cnt <= *cap_cnt)
183                 return 0;
184 
185         p = libbpf_add_mem(data, cap_cnt, elem_sz, *cap_cnt, SIZE_MAX, need_cnt - *cap_cnt);
186         if (!p)
187                 return -ENOMEM;
188 
189         return 0;
190 }
191 
192 static int btf_add_type_idx_entry(struct btf *btf, __u32 type_off)
193 {
194         __u32 *p;
195 
196         p = libbpf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32),
197                            btf->nr_types, BTF_MAX_NR_TYPES, 1);
198         if (!p)
199                 return -ENOMEM;
200 
201         *p = type_off;
202         return 0;
203 }
204 
205 static void btf_bswap_hdr(struct btf_header *h)
206 {
207         h->magic = bswap_16(h->magic);
208         h->hdr_len = bswap_32(h->hdr_len);
209         h->type_off = bswap_32(h->type_off);
210         h->type_len = bswap_32(h->type_len);
211         h->str_off = bswap_32(h->str_off);
212         h->str_len = bswap_32(h->str_len);
213 }
214 
215 static int btf_parse_hdr(struct btf *btf)
216 {
217         struct btf_header *hdr = btf->hdr;
218         __u32 meta_left;
219 
220         if (btf->raw_size < sizeof(struct btf_header)) {
221                 pr_debug("BTF header not found\n");
222                 return -EINVAL;
223         }
224 
225         if (hdr->magic == bswap_16(BTF_MAGIC)) {
226                 btf->swapped_endian = true;
227                 if (bswap_32(hdr->hdr_len) != sizeof(struct btf_header)) {
228                         pr_warn("Can't load BTF with non-native endianness due to unsupported header length %u\n",
229                                 bswap_32(hdr->hdr_len));
230                         return -ENOTSUP;
231                 }
232                 btf_bswap_hdr(hdr);
233         } else if (hdr->magic != BTF_MAGIC) {
234                 pr_debug("Invalid BTF magic:%x\n", hdr->magic);
235                 return -EINVAL;
236         }
237 
238         meta_left = btf->raw_size - sizeof(*hdr);
239         if (meta_left < hdr->str_off + hdr->str_len) {
240                 pr_debug("Invalid BTF total size:%u\n", btf->raw_size);
241                 return -EINVAL;
242         }
243 
244         if (hdr->type_off + hdr->type_len > hdr->str_off) {
245                 pr_debug("Invalid BTF data sections layout: type data at %u + %u, strings data at %u + %u\n",
246                          hdr->type_off, hdr->type_len, hdr->str_off, hdr->str_len);
247                 return -EINVAL;
248         }
249 
250         if (hdr->type_off % 4) {
251                 pr_debug("BTF type section is not aligned to 4 bytes\n");
252                 return -EINVAL;
253         }
254 
255         return 0;
256 }
257 
258 static int btf_parse_str_sec(struct btf *btf)
259 {
260         const struct btf_header *hdr = btf->hdr;
261         const char *start = btf->strs_data;
262         const char *end = start + btf->hdr->str_len;
263 
264         if (btf->base_btf && hdr->str_len == 0)
265                 return 0;
266         if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_STR_OFFSET || end[-1]) {
267                 pr_debug("Invalid BTF string section\n");
268                 return -EINVAL;
269         }
270         if (!btf->base_btf && start[0]) {
271                 pr_debug("Invalid BTF string section\n");
272                 return -EINVAL;
273         }
274         return 0;
275 }
276 
277 static int btf_type_size(const struct btf_type *t)
278 {
279         const int base_size = sizeof(struct btf_type);
280         __u16 vlen = btf_vlen(t);
281 
282         switch (btf_kind(t)) {
283         case BTF_KIND_FWD:
284         case BTF_KIND_CONST:
285         case BTF_KIND_VOLATILE:
286         case BTF_KIND_RESTRICT:
287         case BTF_KIND_PTR:
288         case BTF_KIND_TYPEDEF:
289         case BTF_KIND_FUNC:
290         case BTF_KIND_FLOAT:
291                 return base_size;
292         case BTF_KIND_INT:
293                 return base_size + sizeof(__u32);
294         case BTF_KIND_ENUM:
295                 return base_size + vlen * sizeof(struct btf_enum);
296         case BTF_KIND_ARRAY:
297                 return base_size + sizeof(struct btf_array);
298         case BTF_KIND_STRUCT:
299         case BTF_KIND_UNION:
300                 return base_size + vlen * sizeof(struct btf_member);
301         case BTF_KIND_FUNC_PROTO:
302                 return base_size + vlen * sizeof(struct btf_param);
303         case BTF_KIND_VAR:
304                 return base_size + sizeof(struct btf_var);
305         case BTF_KIND_DATASEC:
306                 return base_size + vlen * sizeof(struct btf_var_secinfo);
307         default:
308                 pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t));
309                 return -EINVAL;
310         }
311 }
312 
313 static void btf_bswap_type_base(struct btf_type *t)
314 {
315         t->name_off = bswap_32(t->name_off);
316         t->info = bswap_32(t->info);
317         t->type = bswap_32(t->type);
318 }
319 
320 static int btf_bswap_type_rest(struct btf_type *t)
321 {
322         struct btf_var_secinfo *v;
323         struct btf_member *m;
324         struct btf_array *a;
325         struct btf_param *p;
326         struct btf_enum *e;
327         __u16 vlen = btf_vlen(t);
328         int i;
329 
330         switch (btf_kind(t)) {
331         case BTF_KIND_FWD:
332         case BTF_KIND_CONST:
333         case BTF_KIND_VOLATILE:
334         case BTF_KIND_RESTRICT:
335         case BTF_KIND_PTR:
336         case BTF_KIND_TYPEDEF:
337         case BTF_KIND_FUNC:
338         case BTF_KIND_FLOAT:
339                 return 0;
340         case BTF_KIND_INT:
341                 *(__u32 *)(t + 1) = bswap_32(*(__u32 *)(t + 1));
342                 return 0;
343         case BTF_KIND_ENUM:
344                 for (i = 0, e = btf_enum(t); i < vlen; i++, e++) {
345                         e->name_off = bswap_32(e->name_off);
346                         e->val = bswap_32(e->val);
347                 }
348                 return 0;
349         case BTF_KIND_ARRAY:
350                 a = btf_array(t);
351                 a->type = bswap_32(a->type);
352                 a->index_type = bswap_32(a->index_type);
353                 a->nelems = bswap_32(a->nelems);
354                 return 0;
355         case BTF_KIND_STRUCT:
356         case BTF_KIND_UNION:
357                 for (i = 0, m = btf_members(t); i < vlen; i++, m++) {
358                         m->name_off = bswap_32(m->name_off);
359                         m->type = bswap_32(m->type);
360                         m->offset = bswap_32(m->offset);
361                 }
362                 return 0;
363         case BTF_KIND_FUNC_PROTO:
364                 for (i = 0, p = btf_params(t); i < vlen; i++, p++) {
365                         p->name_off = bswap_32(p->name_off);
366                         p->type = bswap_32(p->type);
367                 }
368                 return 0;
369         case BTF_KIND_VAR:
370                 btf_var(t)->linkage = bswap_32(btf_var(t)->linkage);
371                 return 0;
372         case BTF_KIND_DATASEC:
373                 for (i = 0, v = btf_var_secinfos(t); i < vlen; i++, v++) {
374                         v->type = bswap_32(v->type);
375                         v->offset = bswap_32(v->offset);
376                         v->size = bswap_32(v->size);
377                 }
378                 return 0;
379         default:
380                 pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t));
381                 return -EINVAL;
382         }
383 }
384 
385 static int btf_parse_type_sec(struct btf *btf)
386 {
387         struct btf_header *hdr = btf->hdr;
388         void *next_type = btf->types_data;
389         void *end_type = next_type + hdr->type_len;
390         int err, type_size;
391 
392         while (next_type + sizeof(struct btf_type) <= end_type) {
393                 if (btf->swapped_endian)
394                         btf_bswap_type_base(next_type);
395 
396                 type_size = btf_type_size(next_type);
397                 if (type_size < 0)
398                         return type_size;
399                 if (next_type + type_size > end_type) {
400                         pr_warn("BTF type [%d] is malformed\n", btf->start_id + btf->nr_types);
401                         return -EINVAL;
402                 }
403 
404                 if (btf->swapped_endian && btf_bswap_type_rest(next_type))
405                         return -EINVAL;
406 
407                 err = btf_add_type_idx_entry(btf, next_type - btf->types_data);
408                 if (err)
409                         return err;
410 
411                 next_type += type_size;
412                 btf->nr_types++;
413         }
414 
415         if (next_type != end_type) {
416                 pr_warn("BTF types data is malformed\n");
417                 return -EINVAL;
418         }
419 
420         return 0;
421 }
422 
423 __u32 btf__get_nr_types(const struct btf *btf)
424 {
425         return btf->start_id + btf->nr_types - 1;
426 }
427 
428 const struct btf *btf__base_btf(const struct btf *btf)
429 {
430         return btf->base_btf;
431 }
432 
433 /* internal helper returning non-const pointer to a type */
434 struct btf_type *btf_type_by_id(struct btf *btf, __u32 type_id)
435 {
436         if (type_id == 0)
437                 return &btf_void;
438         if (type_id < btf->start_id)
439                 return btf_type_by_id(btf->base_btf, type_id);
440         return btf->types_data + btf->type_offs[type_id - btf->start_id];
441 }
442 
443 const struct btf_type *btf__type_by_id(const struct btf *btf, __u32 type_id)
444 {
445         if (type_id >= btf->start_id + btf->nr_types)
446                 return errno = EINVAL, NULL;
447         return btf_type_by_id((struct btf *)btf, type_id);
448 }
449 
450 static int determine_ptr_size(const struct btf *btf)
451 {
452         const struct btf_type *t;
453         const char *name;
454         int i, n;
455 
456         if (btf->base_btf && btf->base_btf->ptr_sz > 0)
457                 return btf->base_btf->ptr_sz;
458 
459         n = btf__get_nr_types(btf);
460         for (i = 1; i <= n; i++) {
461                 t = btf__type_by_id(btf, i);
462                 if (!btf_is_int(t))
463                         continue;
464 
465                 name = btf__name_by_offset(btf, t->name_off);
466                 if (!name)
467                         continue;
468 
469                 if (strcmp(name, "long int") == 0 ||
470                     strcmp(name, "long unsigned int") == 0) {
471                         if (t->size != 4 && t->size != 8)
472                                 continue;
473                         return t->size;
474                 }
475         }
476 
477         return -1;
478 }
479 
480 static size_t btf_ptr_sz(const struct btf *btf)
481 {
482         if (!btf->ptr_sz)
483                 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf);
484         return btf->ptr_sz < 0 ? sizeof(void *) : btf->ptr_sz;
485 }
486 
487 /* Return pointer size this BTF instance assumes. The size is heuristically
488  * determined by looking for 'long' or 'unsigned long' integer type and
489  * recording its size in bytes. If BTF type information doesn't have any such
490  * type, this function returns 0. In the latter case, native architecture's
491  * pointer size is assumed, so will be either 4 or 8, depending on
492  * architecture that libbpf was compiled for. It's possible to override
493  * guessed value by using btf__set_pointer_size() API.
494  */
495 size_t btf__pointer_size(const struct btf *btf)
496 {
497         if (!btf->ptr_sz)
498                 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf);
499 
500         if (btf->ptr_sz < 0)
501                 /* not enough BTF type info to guess */
502                 return 0;
503 
504         return btf->ptr_sz;
505 }
506 
507 /* Override or set pointer size in bytes. Only values of 4 and 8 are
508  * supported.
509  */
510 int btf__set_pointer_size(struct btf *btf, size_t ptr_sz)
511 {
512         if (ptr_sz != 4 && ptr_sz != 8)
513                 return libbpf_err(-EINVAL);
514         btf->ptr_sz = ptr_sz;
515         return 0;
516 }
517 
518 static bool is_host_big_endian(void)
519 {
520 #if __BYTE_ORDER == __LITTLE_ENDIAN
521         return false;
522 #elif __BYTE_ORDER == __BIG_ENDIAN
523         return true;
524 #else
525 # error "Unrecognized __BYTE_ORDER__"
526 #endif
527 }
528 
529 enum btf_endianness btf__endianness(const struct btf *btf)
530 {
531         if (is_host_big_endian())
532                 return btf->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN;
533         else
534                 return btf->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN;
535 }
536 
537 int btf__set_endianness(struct btf *btf, enum btf_endianness endian)
538 {
539         if (endian != BTF_LITTLE_ENDIAN && endian != BTF_BIG_ENDIAN)
540                 return libbpf_err(-EINVAL);
541 
542         btf->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN);
543         if (!btf->swapped_endian) {
544                 free(btf->raw_data_swapped);
545                 btf->raw_data_swapped = NULL;
546         }
547         return 0;
548 }
549 
550 static bool btf_type_is_void(const struct btf_type *t)
551 {
552         return t == &btf_void || btf_is_fwd(t);
553 }
554 
555 static bool btf_type_is_void_or_null(const struct btf_type *t)
556 {
557         return !t || btf_type_is_void(t);
558 }
559 
560 #define MAX_RESOLVE_DEPTH 32
561 
562 __s64 btf__resolve_size(const struct btf *btf, __u32 type_id)
563 {
564         const struct btf_array *array;
565         const struct btf_type *t;
566         __u32 nelems = 1;
567         __s64 size = -1;
568         int i;
569 
570         t = btf__type_by_id(btf, type_id);
571         for (i = 0; i < MAX_RESOLVE_DEPTH && !btf_type_is_void_or_null(t); i++) {
572                 switch (btf_kind(t)) {
573                 case BTF_KIND_INT:
574                 case BTF_KIND_STRUCT:
575                 case BTF_KIND_UNION:
576                 case BTF_KIND_ENUM:
577                 case BTF_KIND_DATASEC:
578                 case BTF_KIND_FLOAT:
579                         size = t->size;
580                         goto done;
581                 case BTF_KIND_PTR:
582                         size = btf_ptr_sz(btf);
583                         goto done;
584                 case BTF_KIND_TYPEDEF:
585                 case BTF_KIND_VOLATILE:
586                 case BTF_KIND_CONST:
587                 case BTF_KIND_RESTRICT:
588                 case BTF_KIND_VAR:
589                         type_id = t->type;
590                         break;
591                 case BTF_KIND_ARRAY:
592                         array = btf_array(t);
593                         if (nelems && array->nelems > UINT32_MAX / nelems)
594                                 return libbpf_err(-E2BIG);
595                         nelems *= array->nelems;
596                         type_id = array->type;
597                         break;
598                 default:
599                         return libbpf_err(-EINVAL);
600                 }
601 
602                 t = btf__type_by_id(btf, type_id);
603         }
604 
605 done:
606         if (size < 0)
607                 return libbpf_err(-EINVAL);
608         if (nelems && size > UINT32_MAX / nelems)
609                 return libbpf_err(-E2BIG);
610 
611         return nelems * size;
612 }
613 
614 int btf__align_of(const struct btf *btf, __u32 id)
615 {
616         const struct btf_type *t = btf__type_by_id(btf, id);
617         __u16 kind = btf_kind(t);
618 
619         switch (kind) {
620         case BTF_KIND_INT:
621         case BTF_KIND_ENUM:
622         case BTF_KIND_FLOAT:
623                 return min(btf_ptr_sz(btf), (size_t)t->size);
624         case BTF_KIND_PTR:
625                 return btf_ptr_sz(btf);
626         case BTF_KIND_TYPEDEF:
627         case BTF_KIND_VOLATILE:
628         case BTF_KIND_CONST:
629         case BTF_KIND_RESTRICT:
630                 return btf__align_of(btf, t->type);
631         case BTF_KIND_ARRAY:
632                 return btf__align_of(btf, btf_array(t)->type);
633         case BTF_KIND_STRUCT:
634         case BTF_KIND_UNION: {
635                 const struct btf_member *m = btf_members(t);
636                 __u16 vlen = btf_vlen(t);
637                 int i, max_align = 1, align;
638 
639                 for (i = 0; i < vlen; i++, m++) {
640                         align = btf__align_of(btf, m->type);
641                         if (align <= 0)
642                                 return libbpf_err(align);
643                         max_align = max(max_align, align);
644                 }
645 
646                 return max_align;
647         }
648         default:
649                 pr_warn("unsupported BTF_KIND:%u\n", btf_kind(t));
650                 return errno = EINVAL, 0;
651         }
652 }
653 
654 int btf__resolve_type(const struct btf *btf, __u32 type_id)
655 {
656         const struct btf_type *t;
657         int depth = 0;
658 
659         t = btf__type_by_id(btf, type_id);
660         while (depth < MAX_RESOLVE_DEPTH &&
661                !btf_type_is_void_or_null(t) &&
662                (btf_is_mod(t) || btf_is_typedef(t) || btf_is_var(t))) {
663                 type_id = t->type;
664                 t = btf__type_by_id(btf, type_id);
665                 depth++;
666         }
667 
668         if (depth == MAX_RESOLVE_DEPTH || btf_type_is_void_or_null(t))
669                 return libbpf_err(-EINVAL);
670 
671         return type_id;
672 }
673 
674 __s32 btf__find_by_name(const struct btf *btf, const char *type_name)
675 {
676         __u32 i, nr_types = btf__get_nr_types(btf);
677 
678         if (!strcmp(type_name, "void"))
679                 return 0;
680 
681         for (i = 1; i <= nr_types; i++) {
682                 const struct btf_type *t = btf__type_by_id(btf, i);
683                 const char *name = btf__name_by_offset(btf, t->name_off);
684 
685                 if (name && !strcmp(type_name, name))
686                         return i;
687         }
688 
689         return libbpf_err(-ENOENT);
690 }
691 
692 __s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name,
693                              __u32 kind)
694 {
695         __u32 i, nr_types = btf__get_nr_types(btf);
696 
697         if (kind == BTF_KIND_UNKN || !strcmp(type_name, "void"))
698                 return 0;
699 
700         for (i = 1; i <= nr_types; i++) {
701                 const struct btf_type *t = btf__type_by_id(btf, i);
702                 const char *name;
703 
704                 if (btf_kind(t) != kind)
705                         continue;
706                 name = btf__name_by_offset(btf, t->name_off);
707                 if (name && !strcmp(type_name, name))
708                         return i;
709         }
710 
711         return libbpf_err(-ENOENT);
712 }
713 
714 static bool btf_is_modifiable(const struct btf *btf)
715 {
716         return (void *)btf->hdr != btf->raw_data;
717 }
718 
719 void btf__free(struct btf *btf)
720 {
721         if (IS_ERR_OR_NULL(btf))
722                 return;
723 
724         if (btf->fd >= 0)
725                 close(btf->fd);
726 
727         if (btf_is_modifiable(btf)) {
728                 /* if BTF was modified after loading, it will have a split
729                  * in-memory representation for header, types, and strings
730                  * sections, so we need to free all of them individually. It
731                  * might still have a cached contiguous raw data present,
732                  * which will be unconditionally freed below.
733                  */
734                 free(btf->hdr);
735                 free(btf->types_data);
736                 strset__free(btf->strs_set);
737         }
738         free(btf->raw_data);
739         free(btf->raw_data_swapped);
740         free(btf->type_offs);
741         free(btf);
742 }
743 
744 static struct btf *btf_new_empty(struct btf *base_btf)
745 {
746         struct btf *btf;
747 
748         btf = calloc(1, sizeof(*btf));
749         if (!btf)
750                 return ERR_PTR(-ENOMEM);
751 
752         btf->nr_types = 0;
753         btf->start_id = 1;
754         btf->start_str_off = 0;
755         btf->fd = -1;
756         btf->ptr_sz = sizeof(void *);
757         btf->swapped_endian = false;
758 
759         if (base_btf) {
760                 btf->base_btf = base_btf;
761                 btf->start_id = btf__get_nr_types(base_btf) + 1;
762                 btf->start_str_off = base_btf->hdr->str_len;
763         }
764 
765         /* +1 for empty string at offset 0 */
766         btf->raw_size = sizeof(struct btf_header) + (base_btf ? 0 : 1);
767         btf->raw_data = calloc(1, btf->raw_size);
768         if (!btf->raw_data) {
769                 free(btf);
770                 return ERR_PTR(-ENOMEM);
771         }
772 
773         btf->hdr = btf->raw_data;
774         btf->hdr->hdr_len = sizeof(struct btf_header);
775         btf->hdr->magic = BTF_MAGIC;
776         btf->hdr->version = BTF_VERSION;
777 
778         btf->types_data = btf->raw_data + btf->hdr->hdr_len;
779         btf->strs_data = btf->raw_data + btf->hdr->hdr_len;
780         btf->hdr->str_len = base_btf ? 0 : 1; /* empty string at offset 0 */
781 
782         return btf;
783 }
784 
785 struct btf *btf__new_empty(void)
786 {
787         return libbpf_ptr(btf_new_empty(NULL));
788 }
789 
790 struct btf *btf__new_empty_split(struct btf *base_btf)
791 {
792         return libbpf_ptr(btf_new_empty(base_btf));
793 }
794 
795 static struct btf *btf_new(const void *data, __u32 size, struct btf *base_btf)
796 {
797         struct btf *btf;
798         int err;
799 
800         btf = calloc(1, sizeof(struct btf));
801         if (!btf)
802                 return ERR_PTR(-ENOMEM);
803 
804         btf->nr_types = 0;
805         btf->start_id = 1;
806         btf->start_str_off = 0;
807 
808         if (base_btf) {
809                 btf->base_btf = base_btf;
810                 btf->start_id = btf__get_nr_types(base_btf) + 1;
811                 btf->start_str_off = base_btf->hdr->str_len;
812         }
813 
814         btf->raw_data = malloc(size);
815         if (!btf->raw_data) {
816                 err = -ENOMEM;
817                 goto done;
818         }
819         memcpy(btf->raw_data, data, size);
820         btf->raw_size = size;
821 
822         btf->hdr = btf->raw_data;
823         err = btf_parse_hdr(btf);
824         if (err)
825                 goto done;
826 
827         btf->strs_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->str_off;
828         btf->types_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->type_off;
829 
830         err = btf_parse_str_sec(btf);
831         err = err ?: btf_parse_type_sec(btf);
832         if (err)
833                 goto done;
834 
835         btf->fd = -1;
836 
837 done:
838         if (err) {
839                 btf__free(btf);
840                 return ERR_PTR(err);
841         }
842 
843         return btf;
844 }
845 
846 struct btf *btf__new(const void *data, __u32 size)
847 {
848         return libbpf_ptr(btf_new(data, size, NULL));
849 }
850 
851 static struct btf *btf_parse_elf(const char *path, struct btf *base_btf,
852                                  struct btf_ext **btf_ext)
853 {
854         Elf_Data *btf_data = NULL, *btf_ext_data = NULL;
855         int err = 0, fd = -1, idx = 0;
856         struct btf *btf = NULL;
857         Elf_Scn *scn = NULL;
858         Elf *elf = NULL;
859         GElf_Ehdr ehdr;
860         size_t shstrndx;
861 
862         if (elf_version(EV_CURRENT) == EV_NONE) {
863                 pr_warn("failed to init libelf for %s\n", path);
864                 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
865         }
866 
867         fd = open(path, O_RDONLY);
868         if (fd < 0) {
869                 err = -errno;
870                 pr_warn("failed to open %s: %s\n", path, strerror(errno));
871                 return ERR_PTR(err);
872         }
873 
874         err = -LIBBPF_ERRNO__FORMAT;
875 
876         elf = elf_begin(fd, ELF_C_READ, NULL);
877         if (!elf) {
878                 pr_warn("failed to open %s as ELF file\n", path);
879                 goto done;
880         }
881         if (!gelf_getehdr(elf, &ehdr)) {
882                 pr_warn("failed to get EHDR from %s\n", path);
883                 goto done;
884         }
885 
886         if (elf_getshdrstrndx(elf, &shstrndx)) {
887                 pr_warn("failed to get section names section index for %s\n",
888                         path);
889                 goto done;
890         }
891 
892         if (!elf_rawdata(elf_getscn(elf, shstrndx), NULL)) {
893                 pr_warn("failed to get e_shstrndx from %s\n", path);
894                 goto done;
895         }
896 
897         while ((scn = elf_nextscn(elf, scn)) != NULL) {
898                 GElf_Shdr sh;
899                 char *name;
900 
901                 idx++;
902                 if (gelf_getshdr(scn, &sh) != &sh) {
903                         pr_warn("failed to get section(%d) header from %s\n",
904                                 idx, path);
905                         goto done;
906                 }
907                 name = elf_strptr(elf, shstrndx, sh.sh_name);
908                 if (!name) {
909                         pr_warn("failed to get section(%d) name from %s\n",
910                                 idx, path);
911                         goto done;
912                 }
913                 if (strcmp(name, BTF_ELF_SEC) == 0) {
914                         btf_data = elf_getdata(scn, 0);
915                         if (!btf_data) {
916                                 pr_warn("failed to get section(%d, %s) data from %s\n",
917                                         idx, name, path);
918                                 goto done;
919                         }
920                         continue;
921                 } else if (btf_ext && strcmp(name, BTF_EXT_ELF_SEC) == 0) {
922                         btf_ext_data = elf_getdata(scn, 0);
923                         if (!btf_ext_data) {
924                                 pr_warn("failed to get section(%d, %s) data from %s\n",
925                                         idx, name, path);
926                                 goto done;
927                         }
928                         continue;
929                 }
930         }
931 
932         err = 0;
933 
934         if (!btf_data) {
935                 err = -ENOENT;
936                 goto done;
937         }
938         btf = btf_new(btf_data->d_buf, btf_data->d_size, base_btf);
939         err = libbpf_get_error(btf);
940         if (err)
941                 goto done;
942 
943         switch (gelf_getclass(elf)) {
944         case ELFCLASS32:
945                 btf__set_pointer_size(btf, 4);
946                 break;
947         case ELFCLASS64:
948                 btf__set_pointer_size(btf, 8);
949                 break;
950         default:
951                 pr_warn("failed to get ELF class (bitness) for %s\n", path);
952                 break;
953         }
954 
955         if (btf_ext && btf_ext_data) {
956                 *btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);
957                 err = libbpf_get_error(*btf_ext);
958                 if (err)
959                         goto done;
960         } else if (btf_ext) {
961                 *btf_ext = NULL;
962         }
963 done:
964         if (elf)
965                 elf_end(elf);
966         close(fd);
967 
968         if (!err)
969                 return btf;
970 
971         if (btf_ext)
972                 btf_ext__free(*btf_ext);
973         btf__free(btf);
974 
975         return ERR_PTR(err);
976 }
977 
978 struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext)
979 {
980         return libbpf_ptr(btf_parse_elf(path, NULL, btf_ext));
981 }
982 
983 struct btf *btf__parse_elf_split(const char *path, struct btf *base_btf)
984 {
985         return libbpf_ptr(btf_parse_elf(path, base_btf, NULL));
986 }
987 
988 static struct btf *btf_parse_raw(const char *path, struct btf *base_btf)
989 {
990         struct btf *btf = NULL;
991         void *data = NULL;
992         FILE *f = NULL;
993         __u16 magic;
994         int err = 0;
995         long sz;
996 
997         f = fopen(path, "rb");
998         if (!f) {
999                 err = -errno;
1000                 goto err_out;
1001         }
1002 
1003         /* check BTF magic */
1004         if (fread(&magic, 1, sizeof(magic), f) < sizeof(magic)) {
1005                 err = -EIO;
1006                 goto err_out;
1007         }
1008         if (magic != BTF_MAGIC && magic != bswap_16(BTF_MAGIC)) {
1009                 /* definitely not a raw BTF */
1010                 err = -EPROTO;
1011                 goto err_out;
1012         }
1013 
1014         /* get file size */
1015         if (fseek(f, 0, SEEK_END)) {
1016                 err = -errno;
1017                 goto err_out;
1018         }
1019         sz = ftell(f);
1020         if (sz < 0) {
1021                 err = -errno;
1022                 goto err_out;
1023         }
1024         /* rewind to the start */
1025         if (fseek(f, 0, SEEK_SET)) {
1026                 err = -errno;
1027                 goto err_out;
1028         }
1029 
1030         /* pre-alloc memory and read all of BTF data */
1031         data = malloc(sz);
1032         if (!data) {
1033                 err = -ENOMEM;
1034                 goto err_out;
1035         }
1036         if (fread(data, 1, sz, f) < sz) {
1037                 err = -EIO;
1038                 goto err_out;
1039         }
1040 
1041         /* finally parse BTF data */
1042         btf = btf_new(data, sz, base_btf);
1043 
1044 err_out:
1045         free(data);
1046         if (f)
1047                 fclose(f);
1048         return err ? ERR_PTR(err) : btf;
1049 }
1050 
1051 struct btf *btf__parse_raw(const char *path)
1052 {
1053         return libbpf_ptr(btf_parse_raw(path, NULL));
1054 }
1055 
1056 struct btf *btf__parse_raw_split(const char *path, struct btf *base_btf)
1057 {
1058         return libbpf_ptr(btf_parse_raw(path, base_btf));
1059 }
1060 
1061 static struct btf *btf_parse(const char *path, struct btf *base_btf, struct btf_ext **btf_ext)
1062 {
1063         struct btf *btf;
1064         int err;
1065 
1066         if (btf_ext)
1067                 *btf_ext = NULL;
1068 
1069         btf = btf_parse_raw(path, base_btf);
1070         err = libbpf_get_error(btf);
1071         if (!err)
1072                 return btf;
1073         if (err != -EPROTO)
1074                 return ERR_PTR(err);
1075         return btf_parse_elf(path, base_btf, btf_ext);
1076 }
1077 
1078 struct btf *btf__parse(const char *path, struct btf_ext **btf_ext)
1079 {
1080         return libbpf_ptr(btf_parse(path, NULL, btf_ext));
1081 }
1082 
1083 struct btf *btf__parse_split(const char *path, struct btf *base_btf)
1084 {
1085         return libbpf_ptr(btf_parse(path, base_btf, NULL));
1086 }
1087 
1088 static int compare_vsi_off(const void *_a, const void *_b)
1089 {
1090         const struct btf_var_secinfo *a = _a;
1091         const struct btf_var_secinfo *b = _b;
1092 
1093         return a->offset - b->offset;
1094 }
1095 
1096 static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf,
1097                              struct btf_type *t)
1098 {
1099         __u32 size = 0, off = 0, i, vars = btf_vlen(t);
1100         const char *name = btf__name_by_offset(btf, t->name_off);
1101         const struct btf_type *t_var;
1102         struct btf_var_secinfo *vsi;
1103         const struct btf_var *var;
1104         int ret;
1105 
1106         if (!name) {
1107                 pr_debug("No name found in string section for DATASEC kind.\n");
1108                 return -ENOENT;
1109         }
1110 
1111         /* .extern datasec size and var offsets were set correctly during
1112          * extern collection step, so just skip straight to sorting variables
1113          */
1114         if (t->size)
1115                 goto sort_vars;
1116 
1117         ret = bpf_object__section_size(obj, name, &size);
1118         if (ret || !size || (t->size && t->size != size)) {
1119                 pr_debug("Invalid size for section %s: %u bytes\n", name, size);
1120                 return -ENOENT;
1121         }
1122 
1123         t->size = size;
1124 
1125         for (i = 0, vsi = btf_var_secinfos(t); i < vars; i++, vsi++) {
1126                 t_var = btf__type_by_id(btf, vsi->type);
1127                 var = btf_var(t_var);
1128 
1129                 if (!btf_is_var(t_var)) {
1130                         pr_debug("Non-VAR type seen in section %s\n", name);
1131                         return -EINVAL;
1132                 }
1133 
1134                 if (var->linkage == BTF_VAR_STATIC)
1135                         continue;
1136 
1137                 name = btf__name_by_offset(btf, t_var->name_off);
1138                 if (!name) {
1139                         pr_debug("No name found in string section for VAR kind\n");
1140                         return -ENOENT;
1141                 }
1142 
1143                 ret = bpf_object__variable_offset(obj, name, &off);
1144                 if (ret) {
1145                         pr_debug("No offset found in symbol table for VAR %s\n",
1146                                  name);
1147                         return -ENOENT;
1148                 }
1149 
1150                 vsi->offset = off;
1151         }
1152 
1153 sort_vars:
1154         qsort(btf_var_secinfos(t), vars, sizeof(*vsi), compare_vsi_off);
1155         return 0;
1156 }
1157 
1158 int btf__finalize_data(struct bpf_object *obj, struct btf *btf)
1159 {
1160         int err = 0;
1161         __u32 i;
1162 
1163         for (i = 1; i <= btf->nr_types; i++) {
1164                 struct btf_type *t = btf_type_by_id(btf, i);
1165 
1166                 /* Loader needs to fix up some of the things compiler
1167                  * couldn't get its hands on while emitting BTF. This
1168                  * is section size and global variable offset. We use
1169                  * the info from the ELF itself for this purpose.
1170                  */
1171                 if (btf_is_datasec(t)) {
1172                         err = btf_fixup_datasec(obj, btf, t);
1173                         if (err)
1174                                 break;
1175                 }
1176         }
1177 
1178         return libbpf_err(err);
1179 }
1180 
1181 static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian);
1182 
1183 int btf__load(struct btf *btf)
1184 {
1185         __u32 log_buf_size = 0, raw_size;
1186         char *log_buf = NULL;
1187         void *raw_data;
1188         int err = 0;
1189 
1190         if (btf->fd >= 0)
1191                 return libbpf_err(-EEXIST);
1192 
1193 retry_load:
1194         if (log_buf_size) {
1195                 log_buf = malloc(log_buf_size);
1196                 if (!log_buf)
1197                         return libbpf_err(-ENOMEM);
1198 
1199                 *log_buf = 0;
1200         }
1201 
1202         raw_data = btf_get_raw_data(btf, &raw_size, false);
1203         if (!raw_data) {
1204                 err = -ENOMEM;
1205                 goto done;
1206         }
1207         /* cache native raw data representation */
1208         btf->raw_size = raw_size;
1209         btf->raw_data = raw_data;
1210 
1211         btf->fd = bpf_load_btf(raw_data, raw_size, log_buf, log_buf_size, false);
1212         if (btf->fd < 0) {
1213                 if (!log_buf || errno == ENOSPC) {
1214                         log_buf_size = max((__u32)BPF_LOG_BUF_SIZE,
1215                                            log_buf_size << 1);
1216                         free(log_buf);
1217                         goto retry_load;
1218                 }
1219 
1220                 err = -errno;
1221                 pr_warn("Error loading BTF: %s(%d)\n", strerror(errno), errno);
1222                 if (*log_buf)
1223                         pr_warn("%s\n", log_buf);
1224                 goto done;
1225         }
1226 
1227 done:
1228         free(log_buf);
1229         return libbpf_err(err);
1230 }
1231 
1232 int btf__fd(const struct btf *btf)
1233 {
1234         return btf->fd;
1235 }
1236 
1237 void btf__set_fd(struct btf *btf, int fd)
1238 {
1239         btf->fd = fd;
1240 }
1241 
1242 static const void *btf_strs_data(const struct btf *btf)
1243 {
1244         return btf->strs_data ? btf->strs_data : strset__data(btf->strs_set);
1245 }
1246 
1247 static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian)
1248 {
1249         struct btf_header *hdr = btf->hdr;
1250         struct btf_type *t;
1251         void *data, *p;
1252         __u32 data_sz;
1253         int i;
1254 
1255         data = swap_endian ? btf->raw_data_swapped : btf->raw_data;
1256         if (data) {
1257                 *size = btf->raw_size;
1258                 return data;
1259         }
1260 
1261         data_sz = hdr->hdr_len + hdr->type_len + hdr->str_len;
1262         data = calloc(1, data_sz);
1263         if (!data)
1264                 return NULL;
1265         p = data;
1266 
1267         memcpy(p, hdr, hdr->hdr_len);
1268         if (swap_endian)
1269                 btf_bswap_hdr(p);
1270         p += hdr->hdr_len;
1271 
1272         memcpy(p, btf->types_data, hdr->type_len);
1273         if (swap_endian) {
1274                 for (i = 0; i < btf->nr_types; i++) {
1275                         t = p + btf->type_offs[i];
1276                         /* btf_bswap_type_rest() relies on native t->info, so
1277                          * we swap base type info after we swapped all the
1278                          * additional information
1279                          */
1280                         if (btf_bswap_type_rest(t))
1281                                 goto err_out;
1282                         btf_bswap_type_base(t);
1283                 }
1284         }
1285         p += hdr->type_len;
1286 
1287         memcpy(p, btf_strs_data(btf), hdr->str_len);
1288         p += hdr->str_len;
1289 
1290         *size = data_sz;
1291         return data;
1292 err_out:
1293         free(data);
1294         return NULL;
1295 }
1296 
1297 const void *btf__get_raw_data(const struct btf *btf_ro, __u32 *size)
1298 {
1299         struct btf *btf = (struct btf *)btf_ro;
1300         __u32 data_sz;
1301         void *data;
1302 
1303         data = btf_get_raw_data(btf, &data_sz, btf->swapped_endian);
1304         if (!data)
1305                 return errno = -ENOMEM, NULL;
1306 
1307         btf->raw_size = data_sz;
1308         if (btf->swapped_endian)
1309                 btf->raw_data_swapped = data;
1310         else
1311                 btf->raw_data = data;
1312         *size = data_sz;
1313         return data;
1314 }
1315 
1316 const char *btf__str_by_offset(const struct btf *btf, __u32 offset)
1317 {
1318         if (offset < btf->start_str_off)
1319                 return btf__str_by_offset(btf->base_btf, offset);
1320         else if (offset - btf->start_str_off < btf->hdr->str_len)
1321                 return btf_strs_data(btf) + (offset - btf->start_str_off);
1322         else
1323                 return errno = EINVAL, NULL;
1324 }
1325 
1326 const char *btf__name_by_offset(const struct btf *btf, __u32 offset)
1327 {
1328         return btf__str_by_offset(btf, offset);
1329 }
1330 
1331 struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf)
1332 {
1333         struct bpf_btf_info btf_info;
1334         __u32 len = sizeof(btf_info);
1335         __u32 last_size;
1336         struct btf *btf;
1337         void *ptr;
1338         int err;
1339 
1340         /* we won't know btf_size until we call bpf_obj_get_info_by_fd(). so
1341          * let's start with a sane default - 4KiB here - and resize it only if
1342          * bpf_obj_get_info_by_fd() needs a bigger buffer.
1343          */
1344         last_size = 4096;
1345         ptr = malloc(last_size);
1346         if (!ptr)
1347                 return ERR_PTR(-ENOMEM);
1348 
1349         memset(&btf_info, 0, sizeof(btf_info));
1350         btf_info.btf = ptr_to_u64(ptr);
1351         btf_info.btf_size = last_size;
1352         err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len);
1353 
1354         if (!err && btf_info.btf_size > last_size) {
1355                 void *temp_ptr;
1356 
1357                 last_size = btf_info.btf_size;
1358                 temp_ptr = realloc(ptr, last_size);
1359                 if (!temp_ptr) {
1360                         btf = ERR_PTR(-ENOMEM);
1361                         goto exit_free;
1362                 }
1363                 ptr = temp_ptr;
1364 
1365                 len = sizeof(btf_info);
1366                 memset(&btf_info, 0, sizeof(btf_info));
1367                 btf_info.btf = ptr_to_u64(ptr);
1368                 btf_info.btf_size = last_size;
1369 
1370                 err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len);
1371         }
1372 
1373         if (err || btf_info.btf_size > last_size) {
1374                 btf = err ? ERR_PTR(-errno) : ERR_PTR(-E2BIG);
1375                 goto exit_free;
1376         }
1377 
1378         btf = btf_new(ptr, btf_info.btf_size, base_btf);
1379 
1380 exit_free:
1381         free(ptr);
1382         return btf;
1383 }
1384 
1385 int btf__get_from_id(__u32 id, struct btf **btf)
1386 {
1387         struct btf *res;
1388         int err, btf_fd;
1389 
1390         *btf = NULL;
1391         btf_fd = bpf_btf_get_fd_by_id(id);
1392         if (btf_fd < 0)
1393                 return libbpf_err(-errno);
1394 
1395         res = btf_get_from_fd(btf_fd, NULL);
1396         err = libbpf_get_error(res);
1397 
1398         close(btf_fd);
1399 
1400         if (err)
1401                 return libbpf_err(err);
1402 
1403         *btf = res;
1404         return 0;
1405 }
1406 
1407 int btf__get_map_kv_tids(const struct btf *btf, const char *map_name,
1408                          __u32 expected_key_size, __u32 expected_value_size,
1409                          __u32 *key_type_id, __u32 *value_type_id)
1410 {
1411         const struct btf_type *container_type;
1412         const struct btf_member *key, *value;
1413         const size_t max_name = 256;
1414         char container_name[max_name];
1415         __s64 key_size, value_size;
1416         __s32 container_id;
1417 
1418         if (snprintf(container_name, max_name, "____btf_map_%s", map_name) == max_name) {
1419                 pr_warn("map:%s length of '____btf_map_%s' is too long\n",
1420                         map_name, map_name);
1421                 return libbpf_err(-EINVAL);
1422         }
1423 
1424         container_id = btf__find_by_name(btf, container_name);
1425         if (container_id < 0) {
1426                 pr_debug("map:%s container_name:%s cannot be found in BTF. Missing BPF_ANNOTATE_KV_PAIR?\n",
1427                          map_name, container_name);
1428                 return libbpf_err(container_id);
1429         }
1430 
1431         container_type = btf__type_by_id(btf, container_id);
1432         if (!container_type) {
1433                 pr_warn("map:%s cannot find BTF type for container_id:%u\n",
1434                         map_name, container_id);
1435                 return libbpf_err(-EINVAL);
1436         }
1437 
1438         if (!btf_is_struct(container_type) || btf_vlen(container_type) < 2) {
1439                 pr_warn("map:%s container_name:%s is an invalid container struct\n",
1440                         map_name, container_name);
1441                 return libbpf_err(-EINVAL);
1442         }
1443 
1444         key = btf_members(container_type);
1445         value = key + 1;
1446 
1447         key_size = btf__resolve_size(btf, key->type);
1448         if (key_size < 0) {
1449                 pr_warn("map:%s invalid BTF key_type_size\n", map_name);
1450                 return libbpf_err(key_size);
1451         }
1452 
1453         if (expected_key_size != key_size) {
1454                 pr_warn("map:%s btf_key_type_size:%u != map_def_key_size:%u\n",
1455                         map_name, (__u32)key_size, expected_key_size);
1456                 return libbpf_err(-EINVAL);
1457         }
1458 
1459         value_size = btf__resolve_size(btf, value->type);
1460         if (value_size < 0) {
1461                 pr_warn("map:%s invalid BTF value_type_size\n", map_name);
1462                 return libbpf_err(value_size);
1463         }
1464 
1465         if (expected_value_size != value_size) {
1466                 pr_warn("map:%s btf_value_type_size:%u != map_def_value_size:%u\n",
1467                         map_name, (__u32)value_size, expected_value_size);
1468                 return libbpf_err(-EINVAL);
1469         }
1470 
1471         *key_type_id = key->type;
1472         *value_type_id = value->type;
1473 
1474         return 0;
1475 }
1476 
1477 static void btf_invalidate_raw_data(struct btf *btf)
1478 {
1479         if (btf->raw_data) {
1480                 free(btf->raw_data);
1481                 btf->raw_data = NULL;
1482         }
1483         if (btf->raw_data_swapped) {
1484                 free(btf->raw_data_swapped);
1485                 btf->raw_data_swapped = NULL;
1486         }
1487 }
1488 
1489 /* Ensure BTF is ready to be modified (by splitting into a three memory
1490  * regions for header, types, and strings). Also invalidate cached
1491  * raw_data, if any.
1492  */
1493 static int btf_ensure_modifiable(struct btf *btf)
1494 {
1495         void *hdr, *types;
1496         struct strset *set = NULL;
1497         int err = -ENOMEM;
1498 
1499         if (btf_is_modifiable(btf)) {
1500                 /* any BTF modification invalidates raw_data */
1501                 btf_invalidate_raw_data(btf);
1502                 return 0;
1503         }
1504 
1505         /* split raw data into three memory regions */
1506         hdr = malloc(btf->hdr->hdr_len);
1507         types = malloc(btf->hdr->type_len);
1508         if (!hdr || !types)
1509                 goto err_out;
1510 
1511         memcpy(hdr, btf->hdr, btf->hdr->hdr_len);
1512         memcpy(types, btf->types_data, btf->hdr->type_len);
1513 
1514         /* build lookup index for all strings */
1515         set = strset__new(BTF_MAX_STR_OFFSET, btf->strs_data, btf->hdr->str_len);
1516         if (IS_ERR(set)) {
1517                 err = PTR_ERR(set);
1518                 goto err_out;
1519         }
1520 
1521         /* only when everything was successful, update internal state */
1522         btf->hdr = hdr;
1523         btf->types_data = types;
1524         btf->types_data_cap = btf->hdr->type_len;
1525         btf->strs_data = NULL;
1526         btf->strs_set = set;
1527         /* if BTF was created from scratch, all strings are guaranteed to be
1528          * unique and deduplicated
1529          */
1530         if (btf->hdr->str_len == 0)
1531                 btf->strs_deduped = true;
1532         if (!btf->base_btf && btf->hdr->str_len == 1)
1533                 btf->strs_deduped = true;
1534 
1535         /* invalidate raw_data representation */
1536         btf_invalidate_raw_data(btf);
1537 
1538         return 0;
1539 
1540 err_out:
1541         strset__free(set);
1542         free(hdr);
1543         free(types);
1544         return err;
1545 }
1546 
1547 /* Find an offset in BTF string section that corresponds to a given string *s*.
1548  * Returns:
1549  *   - >0 offset into string section, if string is found;
1550  *   - -ENOENT, if string is not in the string section;
1551  *   - <0, on any other error.
1552  */
1553 int btf__find_str(struct btf *btf, const char *s)
1554 {
1555         int off;
1556 
1557         if (btf->base_btf) {
1558                 off = btf__find_str(btf->base_btf, s);
1559                 if (off != -ENOENT)
1560                         return off;
1561         }
1562 
1563         /* BTF needs to be in a modifiable state to build string lookup index */
1564         if (btf_ensure_modifiable(btf))
1565                 return libbpf_err(-ENOMEM);
1566 
1567         off = strset__find_str(btf->strs_set, s);
1568         if (off < 0)
1569                 return libbpf_err(off);
1570 
1571         return btf->start_str_off + off;
1572 }
1573 
1574 /* Add a string s to the BTF string section.
1575  * Returns:
1576  *   - > 0 offset into string section, on success;
1577  *   - < 0, on error.
1578  */
1579 int btf__add_str(struct btf *btf, const char *s)
1580 {
1581         int off;
1582 
1583         if (btf->base_btf) {
1584                 off = btf__find_str(btf->base_btf, s);
1585                 if (off != -ENOENT)
1586                         return off;
1587         }
1588 
1589         if (btf_ensure_modifiable(btf))
1590                 return libbpf_err(-ENOMEM);
1591 
1592         off = strset__add_str(btf->strs_set, s);
1593         if (off < 0)
1594                 return libbpf_err(off);
1595 
1596         btf->hdr->str_len = strset__data_size(btf->strs_set);
1597 
1598         return btf->start_str_off + off;
1599 }
1600 
1601 static void *btf_add_type_mem(struct btf *btf, size_t add_sz)
1602 {
1603         return libbpf_add_mem(&btf->types_data, &btf->types_data_cap, 1,
1604                               btf->hdr->type_len, UINT_MAX, add_sz);
1605 }
1606 
1607 static void btf_type_inc_vlen(struct btf_type *t)
1608 {
1609         t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, btf_kflag(t));
1610 }
1611 
1612 static int btf_commit_type(struct btf *btf, int data_sz)
1613 {
1614         int err;
1615 
1616         err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
1617         if (err)
1618                 return libbpf_err(err);
1619 
1620         btf->hdr->type_len += data_sz;
1621         btf->hdr->str_off += data_sz;
1622         btf->nr_types++;
1623         return btf->start_id + btf->nr_types - 1;
1624 }
1625 
1626 struct btf_pipe {
1627         const struct btf *src;
1628         struct btf *dst;
1629 };
1630 
1631 static int btf_rewrite_str(__u32 *str_off, void *ctx)
1632 {
1633         struct btf_pipe *p = ctx;
1634         int off;
1635 
1636         if (!*str_off) /* nothing to do for empty strings */
1637                 return 0;
1638 
1639         off = btf__add_str(p->dst, btf__str_by_offset(p->src, *str_off));
1640         if (off < 0)
1641                 return off;
1642 
1643         *str_off = off;
1644         return 0;
1645 }
1646 
1647 int btf__add_type(struct btf *btf, const struct btf *src_btf, const struct btf_type *src_type)
1648 {
1649         struct btf_pipe p = { .src = src_btf, .dst = btf };
1650         struct btf_type *t;
1651         int sz, err;
1652 
1653         sz = btf_type_size(src_type);
1654         if (sz < 0)
1655                 return libbpf_err(sz);
1656 
1657         /* deconstruct BTF, if necessary, and invalidate raw_data */
1658         if (btf_ensure_modifiable(btf))
1659                 return libbpf_err(-ENOMEM);
1660 
1661         t = btf_add_type_mem(btf, sz);
1662         if (!t)
1663                 return libbpf_err(-ENOMEM);
1664 
1665         memcpy(t, src_type, sz);
1666 
1667         err = btf_type_visit_str_offs(t, btf_rewrite_str, &p);
1668         if (err)
1669                 return libbpf_err(err);
1670 
1671         return btf_commit_type(btf, sz);
1672 }
1673 
1674 /*
1675  * Append new BTF_KIND_INT type with:
1676  *   - *name* - non-empty, non-NULL type name;
1677  *   - *sz* - power-of-2 (1, 2, 4, ..) size of the type, in bytes;
1678  *   - encoding is a combination of BTF_INT_SIGNED, BTF_INT_CHAR, BTF_INT_BOOL.
1679  * Returns:
1680  *   - >0, type ID of newly added BTF type;
1681  *   - <0, on error.
1682  */
1683 int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding)
1684 {
1685         struct btf_type *t;
1686         int sz, name_off;
1687 
1688         /* non-empty name */
1689         if (!name || !name[0])
1690                 return libbpf_err(-EINVAL);
1691         /* byte_sz must be power of 2 */
1692         if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 16)
1693                 return libbpf_err(-EINVAL);
1694         if (encoding & ~(BTF_INT_SIGNED | BTF_INT_CHAR | BTF_INT_BOOL))
1695                 return libbpf_err(-EINVAL);
1696 
1697         /* deconstruct BTF, if necessary, and invalidate raw_data */
1698         if (btf_ensure_modifiable(btf))
1699                 return libbpf_err(-ENOMEM);
1700 
1701         sz = sizeof(struct btf_type) + sizeof(int);
1702         t = btf_add_type_mem(btf, sz);
1703         if (!t)
1704                 return libbpf_err(-ENOMEM);
1705 
1706         /* if something goes wrong later, we might end up with an extra string,
1707          * but that shouldn't be a problem, because BTF can't be constructed
1708          * completely anyway and will most probably be just discarded
1709          */
1710         name_off = btf__add_str(btf, name);
1711         if (name_off < 0)
1712                 return name_off;
1713 
1714         t->name_off = name_off;
1715         t->info = btf_type_info(BTF_KIND_INT, 0, 0);
1716         t->size = byte_sz;
1717         /* set INT info, we don't allow setting legacy bit offset/size */
1718         *(__u32 *)(t + 1) = (encoding << 24) | (byte_sz * 8);
1719 
1720         return btf_commit_type(btf, sz);
1721 }
1722 
1723 /*
1724  * Append new BTF_KIND_FLOAT type with:
1725  *   - *name* - non-empty, non-NULL type name;
1726  *   - *sz* - size of the type, in bytes;
1727  * Returns:
1728  *   - >0, type ID of newly added BTF type;
1729  *   - <0, on error.
1730  */
1731 int btf__add_float(struct btf *btf, const char *name, size_t byte_sz)
1732 {
1733         struct btf_type *t;
1734         int sz, name_off;
1735 
1736         /* non-empty name */
1737         if (!name || !name[0])
1738                 return libbpf_err(-EINVAL);
1739 
1740         /* byte_sz must be one of the explicitly allowed values */
1741         if (byte_sz != 2 && byte_sz != 4 && byte_sz != 8 && byte_sz != 12 &&
1742             byte_sz != 16)
1743                 return libbpf_err(-EINVAL);
1744 
1745         if (btf_ensure_modifiable(btf))
1746                 return libbpf_err(-ENOMEM);
1747 
1748         sz = sizeof(struct btf_type);
1749         t = btf_add_type_mem(btf, sz);
1750         if (!t)
1751                 return libbpf_err(-ENOMEM);
1752 
1753         name_off = btf__add_str(btf, name);
1754         if (name_off < 0)
1755                 return name_off;
1756 
1757         t->name_off = name_off;
1758         t->info = btf_type_info(BTF_KIND_FLOAT, 0, 0);
1759         t->size = byte_sz;
1760 
1761         return btf_commit_type(btf, sz);
1762 }
1763 
1764 /* it's completely legal to append BTF types with type IDs pointing forward to
1765  * types that haven't been appended yet, so we only make sure that id looks
1766  * sane, we can't guarantee that ID will always be valid
1767  */
1768 static int validate_type_id(int id)
1769 {
1770         if (id < 0 || id > BTF_MAX_NR_TYPES)
1771                 return -EINVAL;
1772         return 0;
1773 }
1774 
1775 /* generic append function for PTR, TYPEDEF, CONST/VOLATILE/RESTRICT */
1776 static int btf_add_ref_kind(struct btf *btf, int kind, const char *name, int ref_type_id)
1777 {
1778         struct btf_type *t;
1779         int sz, name_off = 0;
1780 
1781         if (validate_type_id(ref_type_id))
1782                 return libbpf_err(-EINVAL);
1783 
1784         if (btf_ensure_modifiable(btf))
1785                 return libbpf_err(-ENOMEM);
1786 
1787         sz = sizeof(struct btf_type);
1788         t = btf_add_type_mem(btf, sz);
1789         if (!t)
1790                 return libbpf_err(-ENOMEM);
1791 
1792         if (name && name[0]) {
1793                 name_off = btf__add_str(btf, name);
1794                 if (name_off < 0)
1795                         return name_off;
1796         }
1797 
1798         t->name_off = name_off;
1799         t->info = btf_type_info(kind, 0, 0);
1800         t->type = ref_type_id;
1801 
1802         return btf_commit_type(btf, sz);
1803 }
1804 
1805 /*
1806  * Append new BTF_KIND_PTR type with:
1807  *   - *ref_type_id* - referenced type ID, it might not exist yet;
1808  * Returns:
1809  *   - >0, type ID of newly added BTF type;
1810  *   - <0, on error.
1811  */
1812 int btf__add_ptr(struct btf *btf, int ref_type_id)
1813 {
1814         return btf_add_ref_kind(btf, BTF_KIND_PTR, NULL, ref_type_id);
1815 }
1816 
1817 /*
1818  * Append new BTF_KIND_ARRAY type with:
1819  *   - *index_type_id* - type ID of the type describing array index;
1820  *   - *elem_type_id* - type ID of the type describing array element;
1821  *   - *nr_elems* - the size of the array;
1822  * Returns:
1823  *   - >0, type ID of newly added BTF type;
1824  *   - <0, on error.
1825  */
1826 int btf__add_array(struct btf *btf, int index_type_id, int elem_type_id, __u32 nr_elems)
1827 {
1828         struct btf_type *t;
1829         struct btf_array *a;
1830         int sz;
1831 
1832         if (validate_type_id(index_type_id) || validate_type_id(elem_type_id))
1833                 return libbpf_err(-EINVAL);
1834 
1835         if (btf_ensure_modifiable(btf))
1836                 return libbpf_err(-ENOMEM);
1837 
1838         sz = sizeof(struct btf_type) + sizeof(struct btf_array);
1839         t = btf_add_type_mem(btf, sz);
1840         if (!t)
1841                 return libbpf_err(-ENOMEM);
1842 
1843         t->name_off = 0;
1844         t->info = btf_type_info(BTF_KIND_ARRAY, 0, 0);
1845         t->size = 0;
1846 
1847         a = btf_array(t);
1848         a->type = elem_type_id;
1849         a->index_type = index_type_id;
1850         a->nelems = nr_elems;
1851 
1852         return btf_commit_type(btf, sz);
1853 }
1854 
1855 /* generic STRUCT/UNION append function */
1856 static int btf_add_composite(struct btf *btf, int kind, const char *name, __u32 bytes_sz)
1857 {
1858         struct btf_type *t;
1859         int sz, name_off = 0;
1860 
1861         if (btf_ensure_modifiable(btf))
1862                 return libbpf_err(-ENOMEM);
1863 
1864         sz = sizeof(struct btf_type);
1865         t = btf_add_type_mem(btf, sz);
1866         if (!t)
1867                 return libbpf_err(-ENOMEM);
1868 
1869         if (name && name[0]) {
1870                 name_off = btf__add_str(btf, name);
1871                 if (name_off < 0)
1872                         return name_off;
1873         }
1874 
1875         /* start out with vlen=0 and no kflag; this will be adjusted when
1876          * adding each member
1877          */
1878         t->name_off = name_off;
1879         t->info = btf_type_info(kind, 0, 0);
1880         t->size = bytes_sz;
1881 
1882         return btf_commit_type(btf, sz);
1883 }
1884 
1885 /*
1886  * Append new BTF_KIND_STRUCT type with:
1887  *   - *name* - name of the struct, can be NULL or empty for anonymous structs;
1888  *   - *byte_sz* - size of the struct, in bytes;
1889  *
1890  * Struct initially has no fields in it. Fields can be added by
1891  * btf__add_field() right after btf__add_struct() succeeds.
1892  *
1893  * Returns:
1894  *   - >0, type ID of newly added BTF type;
1895  *   - <0, on error.
1896  */
1897 int btf__add_struct(struct btf *btf, const char *name, __u32 byte_sz)
1898 {
1899         return btf_add_composite(btf, BTF_KIND_STRUCT, name, byte_sz);
1900 }
1901 
1902 /*
1903  * Append new BTF_KIND_UNION type with:
1904  *   - *name* - name of the union, can be NULL or empty for anonymous union;
1905  *   - *byte_sz* - size of the union, in bytes;
1906  *
1907  * Union initially has no fields in it. Fields can be added by
1908  * btf__add_field() right after btf__add_union() succeeds. All fields
1909  * should have *bit_offset* of 0.
1910  *
1911  * Returns:
1912  *   - >0, type ID of newly added BTF type;
1913  *   - <0, on error.
1914  */
1915 int btf__add_union(struct btf *btf, const char *name, __u32 byte_sz)
1916 {
1917         return btf_add_composite(btf, BTF_KIND_UNION, name, byte_sz);
1918 }
1919 
1920 static struct btf_type *btf_last_type(struct btf *btf)
1921 {
1922         return btf_type_by_id(btf, btf__get_nr_types(btf));
1923 }
1924 
1925 /*
1926  * Append new field for the current STRUCT/UNION type with:
1927  *   - *name* - name of the field, can be NULL or empty for anonymous field;
1928  *   - *type_id* - type ID for the type describing field type;
1929  *   - *bit_offset* - bit offset of the start of the field within struct/union;
1930  *   - *bit_size* - bit size of a bitfield, 0 for non-bitfield fields;
1931  * Returns:
1932  *   -  0, on success;
1933  *   - <0, on error.
1934  */
1935 int btf__add_field(struct btf *btf, const char *name, int type_id,
1936                    __u32 bit_offset, __u32 bit_size)
1937 {
1938         struct btf_type *t;
1939         struct btf_member *m;
1940         bool is_bitfield;
1941         int sz, name_off = 0;
1942 
1943         /* last type should be union/struct */
1944         if (btf->nr_types == 0)
1945                 return libbpf_err(-EINVAL);
1946         t = btf_last_type(btf);
1947         if (!btf_is_composite(t))
1948                 return libbpf_err(-EINVAL);
1949 
1950         if (validate_type_id(type_id))
1951                 return libbpf_err(-EINVAL);
1952         /* best-effort bit field offset/size enforcement */
1953         is_bitfield = bit_size || (bit_offset % 8 != 0);
1954         if (is_bitfield && (bit_size == 0 || bit_size > 255 || bit_offset > 0xffffff))
1955                 return libbpf_err(-EINVAL);
1956 
1957         /* only offset 0 is allowed for unions */
1958         if (btf_is_union(t) && bit_offset)
1959                 return libbpf_err(-EINVAL);
1960 
1961         /* decompose and invalidate raw data */
1962         if (btf_ensure_modifiable(btf))
1963                 return libbpf_err(-ENOMEM);
1964 
1965         sz = sizeof(struct btf_member);
1966         m = btf_add_type_mem(btf, sz);
1967         if (!m)
1968                 return libbpf_err(-ENOMEM);
1969 
1970         if (name && name[0]) {
1971                 name_off = btf__add_str(btf, name);
1972                 if (name_off < 0)
1973                         return name_off;
1974         }
1975 
1976         m->name_off = name_off;
1977         m->type = type_id;
1978         m->offset = bit_offset | (bit_size << 24);
1979 
1980         /* btf_add_type_mem can invalidate t pointer */
1981         t = btf_last_type(btf);
1982         /* update parent type's vlen and kflag */
1983         t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, is_bitfield || btf_kflag(t));
1984 
1985         btf->hdr->type_len += sz;
1986         btf->hdr->str_off += sz;
1987         return 0;
1988 }
1989 
1990 /*
1991  * Append new BTF_KIND_ENUM type with:
1992  *   - *name* - name of the enum, can be NULL or empty for anonymous enums;
1993  *   - *byte_sz* - size of the enum, in bytes.
1994  *
1995  * Enum initially has no enum values in it (and corresponds to enum forward
1996  * declaration). Enumerator values can be added by btf__add_enum_value()
1997  * immediately after btf__add_enum() succeeds.
1998  *
1999  * Returns:
2000  *   - >0, type ID of newly added BTF type;
2001  *   - <0, on error.
2002  */
2003 int btf__add_enum(struct btf *btf, const char *name, __u32 byte_sz)
2004 {
2005         struct btf_type *t;
2006         int sz, name_off = 0;
2007 
2008         /* byte_sz must be power of 2 */
2009         if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 8)
2010                 return libbpf_err(-EINVAL);
2011 
2012         if (btf_ensure_modifiable(btf))
2013                 return libbpf_err(-ENOMEM);
2014 
2015         sz = sizeof(struct btf_type);
2016         t = btf_add_type_mem(btf, sz);
2017         if (!t)
2018                 return libbpf_err(-ENOMEM);
2019 
2020         if (name && name[0]) {
2021                 name_off = btf__add_str(btf, name);
2022                 if (name_off < 0)
2023                         return name_off;
2024         }
2025 
2026         /* start out with vlen=0; it will be adjusted when adding enum values */
2027         t->name_off = name_off;
2028         t->info = btf_type_info(BTF_KIND_ENUM, 0, 0);
2029         t->size = byte_sz;
2030 
2031         return btf_commit_type(btf, sz);
2032 }
2033 
2034 /*
2035  * Append new enum value for the current ENUM type with:
2036  *   - *name* - name of the enumerator value, can't be NULL or empty;
2037  *   - *value* - integer value corresponding to enum value *name*;
2038  * Returns:
2039  *   -  0, on success;
2040  *   - <0, on error.
2041  */
2042 int btf__add_enum_value(struct btf *btf, const char *name, __s64 value)
2043 {
2044         struct btf_type *t;
2045         struct btf_enum *v;
2046         int sz, name_off;
2047 
2048         /* last type should be BTF_KIND_ENUM */
2049         if (btf->nr_types == 0)
2050                 return libbpf_err(-EINVAL);
2051         t = btf_last_type(btf);
2052         if (!btf_is_enum(t))
2053                 return libbpf_err(-EINVAL);
2054 
2055         /* non-empty name */
2056         if (!name || !name[0])
2057                 return libbpf_err(-EINVAL);
2058         if (value < INT_MIN || value > UINT_MAX)
2059                 return libbpf_err(-E2BIG);
2060 
2061         /* decompose and invalidate raw data */
2062         if (btf_ensure_modifiable(btf))
2063                 return libbpf_err(-ENOMEM);
2064 
2065         sz = sizeof(struct btf_enum);
2066         v = btf_add_type_mem(btf, sz);
2067         if (!v)
2068                 return libbpf_err(-ENOMEM);
2069 
2070         name_off = btf__add_str(btf, name);
2071         if (name_off < 0)
2072                 return name_off;
2073 
2074         v->name_off = name_off;
2075         v->val = value;
2076 
2077         /* update parent type's vlen */
2078         t = btf_last_type(btf);
2079         btf_type_inc_vlen(t);
2080 
2081         btf->hdr->type_len += sz;
2082         btf->hdr->str_off += sz;
2083         return 0;
2084 }
2085 
2086 /*
2087  * Append new BTF_KIND_FWD type with:
2088  *   - *name*, non-empty/non-NULL name;
2089  *   - *fwd_kind*, kind of forward declaration, one of BTF_FWD_STRUCT,
2090  *     BTF_FWD_UNION, or BTF_FWD_ENUM;
2091  * Returns:
2092  *   - >0, type ID of newly added BTF type;
2093  *   - <0, on error.
2094  */
2095 int btf__add_fwd(struct btf *btf, const char *name, enum btf_fwd_kind fwd_kind)
2096 {
2097         if (!name || !name[0])
2098                 return libbpf_err(-EINVAL);
2099 
2100         switch (fwd_kind) {
2101         case BTF_FWD_STRUCT:
2102         case BTF_FWD_UNION: {
2103                 struct btf_type *t;
2104                 int id;
2105 
2106                 id = btf_add_ref_kind(btf, BTF_KIND_FWD, name, 0);
2107                 if (id <= 0)
2108                         return id;
2109                 t = btf_type_by_id(btf, id);
2110                 t->info = btf_type_info(BTF_KIND_FWD, 0, fwd_kind == BTF_FWD_UNION);
2111                 return id;
2112         }
2113         case BTF_FWD_ENUM:
2114                 /* enum forward in BTF currently is just an enum with no enum
2115                  * values; we also assume a standard 4-byte size for it
2116                  */
2117                 return btf__add_enum(btf, name, sizeof(int));
2118         default:
2119                 return libbpf_err(-EINVAL);
2120         }
2121 }
2122 
2123 /*
2124  * Append new BTF_KING_TYPEDEF type with:
2125  *   - *name*, non-empty/non-NULL name;
2126  *   - *ref_type_id* - referenced type ID, it might not exist yet;
2127  * Returns:
2128  *   - >0, type ID of newly added BTF type;
2129  *   - <0, on error.
2130  */
2131 int btf__add_typedef(struct btf *btf, const char *name, int ref_type_id)
2132 {
2133         if (!name || !name[0])
2134                 return libbpf_err(-EINVAL);
2135 
2136         return btf_add_ref_kind(btf, BTF_KIND_TYPEDEF, name, ref_type_id);
2137 }
2138 
2139 /*
2140  * Append new BTF_KIND_VOLATILE type with:
2141  *   - *ref_type_id* - referenced type ID, it might not exist yet;
2142  * Returns:
2143  *   - >0, type ID of newly added BTF type;
2144  *   - <0, on error.
2145  */
2146 int btf__add_volatile(struct btf *btf, int ref_type_id)
2147 {
2148         return btf_add_ref_kind(btf, BTF_KIND_VOLATILE, NULL, ref_type_id);
2149 }
2150 
2151 /*
2152  * Append new BTF_KIND_CONST type with:
2153  *   - *ref_type_id* - referenced type ID, it might not exist yet;
2154  * Returns:
2155  *   - >0, type ID of newly added BTF type;
2156  *   - <0, on error.
2157  */
2158 int btf__add_const(struct btf *btf, int ref_type_id)
2159 {
2160         return btf_add_ref_kind(btf, BTF_KIND_CONST, NULL, ref_type_id);
2161 }
2162 
2163 /*
2164  * Append new BTF_KIND_RESTRICT type with:
2165  *   - *ref_type_id* - referenced type ID, it might not exist yet;
2166  * Returns:
2167  *   - >0, type ID of newly added BTF type;
2168  *   - <0, on error.
2169  */
2170 int btf__add_restrict(struct btf *btf, int ref_type_id)
2171 {
2172         return btf_add_ref_kind(btf, BTF_KIND_RESTRICT, NULL, ref_type_id);
2173 }
2174 
2175 /*
2176  * Append new BTF_KIND_FUNC type with:
2177  *   - *name*, non-empty/non-NULL name;
2178  *   - *proto_type_id* - FUNC_PROTO's type ID, it might not exist yet;
2179  * Returns:
2180  *   - >0, type ID of newly added BTF type;
2181  *   - <0, on error.
2182  */
2183 int btf__add_func(struct btf *btf, const char *name,
2184                   enum btf_func_linkage linkage, int proto_type_id)
2185 {
2186         int id;
2187 
2188         if (!name || !name[0])
2189                 return libbpf_err(-EINVAL);
2190         if (linkage != BTF_FUNC_STATIC && linkage != BTF_FUNC_GLOBAL &&
2191             linkage != BTF_FUNC_EXTERN)
2192                 return libbpf_err(-EINVAL);
2193 
2194         id = btf_add_ref_kind(btf, BTF_KIND_FUNC, name, proto_type_id);
2195         if (id > 0) {
2196                 struct btf_type *t = btf_type_by_id(btf, id);
2197 
2198                 t->info = btf_type_info(BTF_KIND_FUNC, linkage, 0);
2199         }
2200         return libbpf_err(id);
2201 }
2202 
2203 /*
2204  * Append new BTF_KIND_FUNC_PROTO with:
2205  *   - *ret_type_id* - type ID for return result of a function.
2206  *
2207  * Function prototype initially has no arguments, but they can be added by
2208  * btf__add_func_param() one by one, immediately after
2209  * btf__add_func_proto() succeeded.
2210  *
2211  * Returns:
2212  *   - >0, type ID of newly added BTF type;
2213  *   - <0, on error.
2214  */
2215 int btf__add_func_proto(struct btf *btf, int ret_type_id)
2216 {
2217         struct btf_type *t;
2218         int sz;
2219 
2220         if (validate_type_id(ret_type_id))
2221                 return libbpf_err(-EINVAL);
2222 
2223         if (btf_ensure_modifiable(btf))
2224                 return libbpf_err(-ENOMEM);
2225 
2226         sz = sizeof(struct btf_type);
2227         t = btf_add_type_mem(btf, sz);
2228         if (!t)
2229                 return libbpf_err(-ENOMEM);
2230 
2231         /* start out with vlen=0; this will be adjusted when adding enum
2232          * values, if necessary
2233          */
2234         t->name_off = 0;
2235         t->info = btf_type_info(BTF_KIND_FUNC_PROTO, 0, 0);
2236         t->type = ret_type_id;
2237 
2238         return btf_commit_type(btf, sz);
2239 }
2240 
2241 /*
2242  * Append new function parameter for current FUNC_PROTO type with:
2243  *   - *name* - parameter name, can be NULL or empty;
2244  *   - *type_id* - type ID describing the type of the parameter.
2245  * Returns:
2246  *   -  0, on success;
2247  *   - <0, on error.
2248  */
2249 int btf__add_func_param(struct btf *btf, const char *name, int type_id)
2250 {
2251         struct btf_type *t;
2252         struct btf_param *p;
2253         int sz, name_off = 0;
2254 
2255         if (validate_type_id(type_id))
2256                 return libbpf_err(-EINVAL);
2257 
2258         /* last type should be BTF_KIND_FUNC_PROTO */
2259         if (btf->nr_types == 0)
2260                 return libbpf_err(-EINVAL);
2261         t = btf_last_type(btf);
2262         if (!btf_is_func_proto(t))
2263                 return libbpf_err(-EINVAL);
2264 
2265         /* decompose and invalidate raw data */
2266         if (btf_ensure_modifiable(btf))
2267                 return libbpf_err(-ENOMEM);
2268 
2269         sz = sizeof(struct btf_param);
2270         p = btf_add_type_mem(btf, sz);
2271         if (!p)
2272                 return libbpf_err(-ENOMEM);
2273 
2274         if (name && name[0]) {
2275                 name_off = btf__add_str(btf, name);
2276                 if (name_off < 0)
2277                         return name_off;
2278         }
2279 
2280         p->name_off = name_off;
2281         p->type = type_id;
2282 
2283         /* update parent type's vlen */
2284         t = btf_last_type(btf);
2285         btf_type_inc_vlen(t);
2286 
2287         btf->hdr->type_len += sz;
2288         btf->hdr->str_off += sz;
2289         return 0;
2290 }
2291 
2292 /*
2293  * Append new BTF_KIND_VAR type with:
2294  *   - *name* - non-empty/non-NULL name;
2295  *   - *linkage* - variable linkage, one of BTF_VAR_STATIC,
2296  *     BTF_VAR_GLOBAL_ALLOCATED, or BTF_VAR_GLOBAL_EXTERN;
2297  *   - *type_id* - type ID of the type describing the type of the variable.
2298  * Returns:
2299  *   - >0, type ID of newly added BTF type;
2300  *   - <0, on error.
2301  */
2302 int btf__add_var(struct btf *btf, const char *name, int linkage, int type_id)
2303 {
2304         struct btf_type *t;
2305         struct btf_var *v;
2306         int sz, name_off;
2307 
2308         /* non-empty name */
2309         if (!name || !name[0])
2310                 return libbpf_err(-EINVAL);
2311         if (linkage != BTF_VAR_STATIC && linkage != BTF_VAR_GLOBAL_ALLOCATED &&
2312             linkage != BTF_VAR_GLOBAL_EXTERN)
2313                 return libbpf_err(-EINVAL);
2314         if (validate_type_id(type_id))
2315                 return libbpf_err(-EINVAL);
2316 
2317         /* deconstruct BTF, if necessary, and invalidate raw_data */
2318         if (btf_ensure_modifiable(btf))
2319                 return libbpf_err(-ENOMEM);
2320 
2321         sz = sizeof(struct btf_type) + sizeof(struct btf_var);
2322         t = btf_add_type_mem(btf, sz);
2323         if (!t)
2324                 return libbpf_err(-ENOMEM);
2325 
2326         name_off = btf__add_str(btf, name);
2327         if (name_off < 0)
2328                 return name_off;
2329 
2330         t->name_off = name_off;
2331         t->info = btf_type_info(BTF_KIND_VAR, 0, 0);
2332         t->type = type_id;
2333 
2334         v = btf_var(t);
2335         v->linkage = linkage;
2336 
2337         return btf_commit_type(btf, sz);
2338 }
2339 
2340 /*
2341  * Append new BTF_KIND_DATASEC type with:
2342  *   - *name* - non-empty/non-NULL name;
2343  *   - *byte_sz* - data section size, in bytes.
2344  *
2345  * Data section is initially empty. Variables info can be added with
2346  * btf__add_datasec_var_info() calls, after btf__add_datasec() succeeds.
2347  *
2348  * Returns:
2349  *   - >0, type ID of newly added BTF type;
2350  *   - <0, on error.
2351  */
2352 int btf__add_datasec(struct btf *btf, const char *name, __u32 byte_sz)
2353 {
2354         struct btf_type *t;
2355         int sz, name_off;
2356 
2357         /* non-empty name */
2358         if (!name || !name[0])
2359                 return libbpf_err(-EINVAL);
2360 
2361         if (btf_ensure_modifiable(btf))
2362                 return libbpf_err(-ENOMEM);
2363 
2364         sz = sizeof(struct btf_type);
2365         t = btf_add_type_mem(btf, sz);
2366         if (!t)
2367                 return libbpf_err(-ENOMEM);
2368 
2369         name_off = btf__add_str(btf, name);
2370         if (name_off < 0)
2371                 return name_off;
2372 
2373         /* start with vlen=0, which will be update as var_secinfos are added */
2374         t->name_off = name_off;
2375         t->info = btf_type_info(BTF_KIND_DATASEC, 0, 0);
2376         t->size = byte_sz;
2377 
2378         return btf_commit_type(btf, sz);
2379 }
2380 
2381 /*
2382  * Append new data section variable information entry for current DATASEC type:
2383  *   - *var_type_id* - type ID, describing type of the variable;
2384  *   - *offset* - variable offset within data section, in bytes;
2385  *   - *byte_sz* - variable size, in bytes.
2386  *
2387  * Returns:
2388  *   -  0, on success;
2389  *   - <0, on error.
2390  */
2391 int btf__add_datasec_var_info(struct btf *btf, int var_type_id, __u32 offset, __u32 byte_sz)
2392 {
2393         struct btf_type *t;
2394         struct btf_var_secinfo *v;
2395         int sz;
2396 
2397         /* last type should be BTF_KIND_DATASEC */
2398         if (btf->nr_types == 0)
2399                 return libbpf_err(-EINVAL);
2400         t = btf_last_type(btf);
2401         if (!btf_is_datasec(t))
2402                 return libbpf_err(-EINVAL);
2403 
2404         if (validate_type_id(var_type_id))
2405                 return libbpf_err(-EINVAL);
2406 
2407         /* decompose and invalidate raw data */
2408         if (btf_ensure_modifiable(btf))
2409                 return libbpf_err(-ENOMEM);
2410 
2411         sz = sizeof(struct btf_var_secinfo);
2412         v = btf_add_type_mem(btf, sz);
2413         if (!v)
2414                 return libbpf_err(-ENOMEM);
2415 
2416         v->type = var_type_id;
2417         v->offset = offset;
2418         v->size = byte_sz;
2419 
2420         /* update parent type's vlen */
2421         t = btf_last_type(btf);
2422         btf_type_inc_vlen(t);
2423 
2424         btf->hdr->type_len += sz;
2425         btf->hdr->str_off += sz;
2426         return 0;
2427 }
2428 
2429 struct btf_ext_sec_setup_param {
2430         __u32 off;
2431         __u32 len;
2432         __u32 min_rec_size;
2433         struct btf_ext_info *ext_info;
2434         const char *desc;
2435 };
2436 
2437 static int btf_ext_setup_info(struct btf_ext *btf_ext,
2438                               struct btf_ext_sec_setup_param *ext_sec)
2439 {
2440         const struct btf_ext_info_sec *sinfo;
2441         struct btf_ext_info *ext_info;
2442         __u32 info_left, record_size;
2443         /* The start of the info sec (including the __u32 record_size). */
2444         void *info;
2445 
2446         if (ext_sec->len == 0)
2447                 return 0;
2448 
2449         if (ext_sec->off & 0x03) {
2450                 pr_debug(".BTF.ext %s section is not aligned to 4 bytes\n",
2451                      ext_sec->desc);
2452                 return -EINVAL;
2453         }
2454 
2455         info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off;
2456         info_left = ext_sec->len;
2457 
2458         if (btf_ext->data + btf_ext->data_size < info + ext_sec->len) {
2459                 pr_debug("%s section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n",
2460                          ext_sec->desc, ext_sec->off, ext_sec->len);
2461                 return -EINVAL;
2462         }
2463 
2464         /* At least a record size */
2465         if (info_left < sizeof(__u32)) {
2466                 pr_debug(".BTF.ext %s record size not found\n", ext_sec->desc);
2467                 return -EINVAL;
2468         }
2469 
2470         /* The record size needs to meet the minimum standard */
2471         record_size = *(__u32 *)info;
2472         if (record_size < ext_sec->min_rec_size ||
2473             record_size & 0x03) {
2474                 pr_debug("%s section in .BTF.ext has invalid record size %u\n",
2475                          ext_sec->desc, record_size);
2476                 return -EINVAL;
2477         }
2478 
2479         sinfo = info + sizeof(__u32);
2480         info_left -= sizeof(__u32);
2481 
2482         /* If no records, return failure now so .BTF.ext won't be used. */
2483         if (!info_left) {
2484                 pr_debug("%s section in .BTF.ext has no records", ext_sec->desc);
2485                 return -EINVAL;
2486         }
2487 
2488         while (info_left) {
2489                 unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec);
2490                 __u64 total_record_size;
2491                 __u32 num_records;
2492 
2493                 if (info_left < sec_hdrlen) {
2494                         pr_debug("%s section header is not found in .BTF.ext\n",
2495                              ext_sec->desc);
2496                         return -EINVAL;
2497                 }
2498 
2499                 num_records = sinfo->num_info;
2500                 if (num_records == 0) {
2501                         pr_debug("%s section has incorrect num_records in .BTF.ext\n",
2502                              ext_sec->desc);
2503                         return -EINVAL;
2504                 }
2505 
2506                 total_record_size = sec_hdrlen +
2507                                     (__u64)num_records * record_size;
2508                 if (info_left < total_record_size) {
2509                         pr_debug("%s section has incorrect num_records in .BTF.ext\n",
2510                              ext_sec->desc);
2511                         return -EINVAL;
2512                 }
2513 
2514                 info_left -= total_record_size;
2515                 sinfo = (void *)sinfo + total_record_size;
2516         }
2517 
2518         ext_info = ext_sec->ext_info;
2519         ext_info->len = ext_sec->len - sizeof(__u32);
2520         ext_info->rec_size = record_size;
2521         ext_info->info = info + sizeof(__u32);
2522 
2523         return 0;
2524 }
2525 
2526 static int btf_ext_setup_func_info(struct btf_ext *btf_ext)
2527 {
2528         struct btf_ext_sec_setup_param param = {
2529                 .off = btf_ext->hdr->func_info_off,
2530                 .len = btf_ext->hdr->func_info_len,
2531                 .min_rec_size = sizeof(struct bpf_func_info_min),
2532                 .ext_info = &btf_ext->func_info,
2533                 .desc = "func_info"
2534         };
2535 
2536         return btf_ext_setup_info(btf_ext, &param);
2537 }
2538 
2539 static int btf_ext_setup_line_info(struct btf_ext *btf_ext)
2540 {
2541         struct btf_ext_sec_setup_param param = {
2542                 .off = btf_ext->hdr->line_info_off,
2543                 .len = btf_ext->hdr->line_info_len,
2544                 .min_rec_size = sizeof(struct bpf_line_info_min),
2545                 .ext_info = &btf_ext->line_info,
2546                 .desc = "line_info",
2547         };
2548 
2549         return btf_ext_setup_info(btf_ext, &param);
2550 }
2551 
2552 static int btf_ext_setup_core_relos(struct btf_ext *btf_ext)
2553 {
2554         struct btf_ext_sec_setup_param param = {
2555                 .off = btf_ext->hdr->core_relo_off,
2556                 .len = btf_ext->hdr->core_relo_len,
2557                 .min_rec_size = sizeof(struct bpf_core_relo),
2558                 .ext_info = &btf_ext->core_relo_info,
2559                 .desc = "core_relo",
2560         };
2561 
2562         return btf_ext_setup_info(btf_ext, &param);
2563 }
2564 
2565 static int btf_ext_parse_hdr(__u8 *data, __u32 data_size)
2566 {
2567         const struct btf_ext_header *hdr = (struct btf_ext_header *)data;
2568 
2569         if (data_size < offsetofend(struct btf_ext_header, hdr_len) ||
2570             data_size < hdr->hdr_len) {
2571                 pr_debug("BTF.ext header not found");
2572                 return -EINVAL;
2573         }
2574 
2575         if (hdr->magic == bswap_16(BTF_MAGIC)) {
2576                 pr_warn("BTF.ext in non-native endianness is not supported\n");
2577                 return -ENOTSUP;
2578         } else if (hdr->magic != BTF_MAGIC) {
2579                 pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic);
2580                 return -EINVAL;
2581         }
2582 
2583         if (hdr->version != BTF_VERSION) {
2584                 pr_debug("Unsupported BTF.ext version:%u\n", hdr->version);
2585                 return -ENOTSUP;
2586         }
2587 
2588         if (hdr->flags) {
2589                 pr_debug("Unsupported BTF.ext flags:%x\n", hdr->flags);
2590                 return -ENOTSUP;
2591         }
2592 
2593         if (data_size == hdr->hdr_len) {
2594                 pr_debug("BTF.ext has no data\n");
2595                 return -EINVAL;
2596         }
2597 
2598         return 0;
2599 }
2600 
2601 void btf_ext__free(struct btf_ext *btf_ext)
2602 {
2603         if (IS_ERR_OR_NULL(btf_ext))
2604                 return;
2605         free(btf_ext->data);
2606         free(btf_ext);
2607 }
2608 
2609 struct btf_ext *btf_ext__new(__u8 *data, __u32 size)
2610 {
2611         struct btf_ext *btf_ext;
2612         int err;
2613 
2614         err = btf_ext_parse_hdr(data, size);
2615         if (err)
2616                 return libbpf_err_ptr(err);
2617 
2618         btf_ext = calloc(1, sizeof(struct btf_ext));
2619         if (!btf_ext)
2620                 return libbpf_err_ptr(-ENOMEM);
2621 
2622         btf_ext->data_size = size;
2623         btf_ext->data = malloc(size);
2624         if (!btf_ext->data) {
2625                 err = -ENOMEM;
2626                 goto done;
2627         }
2628         memcpy(btf_ext->data, data, size);
2629 
2630         if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, line_info_len)) {
2631                 err = -EINVAL;
2632                 goto done;
2633         }
2634 
2635         err = btf_ext_setup_func_info(btf_ext);
2636         if (err)
2637                 goto done;
2638 
2639         err = btf_ext_setup_line_info(btf_ext);
2640         if (err)
2641                 goto done;
2642 
2643         if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len)) {
2644                 err = -EINVAL;
2645                 goto done;
2646         }
2647 
2648         err = btf_ext_setup_core_relos(btf_ext);
2649         if (err)
2650                 goto done;
2651 
2652 done:
2653         if (err) {
2654                 btf_ext__free(btf_ext);
2655                 return libbpf_err_ptr(err);
2656         }
2657 
2658         return btf_ext;
2659 }
2660 
2661 const void *btf_ext__get_raw_data(const struct btf_ext *btf_ext, __u32 *size)
2662 {
2663         *size = btf_ext->data_size;
2664         return btf_ext->data;
2665 }
2666 
2667 static int btf_ext_reloc_info(const struct btf *btf,
2668                               const struct btf_ext_info *ext_info,
2669                               const char *sec_name, __u32 insns_cnt,
2670                               void **info, __u32 *cnt)
2671 {
2672         __u32 sec_hdrlen = sizeof(struct btf_ext_info_sec);
2673         __u32 i, record_size, existing_len, records_len;
2674         struct btf_ext_info_sec *sinfo;
2675         const char *info_sec_name;
2676         __u64 remain_len;
2677         void *data;
2678 
2679         record_size = ext_info->rec_size;
2680         sinfo = ext_info->info;
2681         remain_len = ext_info->len;
2682         while (remain_len > 0) {
2683                 records_len = sinfo->num_info * record_size;
2684                 info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off);
2685                 if (strcmp(info_sec_name, sec_name)) {
2686                         remain_len -= sec_hdrlen + records_len;
2687                         sinfo = (void *)sinfo + sec_hdrlen + records_len;
2688                         continue;
2689                 }
2690 
2691                 existing_len = (*cnt) * record_size;
2692                 data = realloc(*info, existing_len + records_len);
2693                 if (!data)
2694                         return libbpf_err(-ENOMEM);
2695 
2696                 memcpy(data + existing_len, sinfo->data, records_len);
2697                 /* adjust insn_off only, the rest data will be passed
2698                  * to the kernel.
2699                  */
2700                 for (i = 0; i < sinfo->num_info; i++) {
2701                         __u32 *insn_off;
2702 
2703                         insn_off = data + existing_len + (i * record_size);
2704                         *insn_off = *insn_off / sizeof(struct bpf_insn) + insns_cnt;
2705                 }
2706                 *info = data;
2707                 *cnt += sinfo->num_info;
2708                 return 0;
2709         }
2710 
2711         return libbpf_err(-ENOENT);
2712 }
2713 
2714 int btf_ext__reloc_func_info(const struct btf *btf,
2715                              const struct btf_ext *btf_ext,
2716                              const char *sec_name, __u32 insns_cnt,
2717                              void **func_info, __u32 *cnt)
2718 {
2719         return btf_ext_reloc_info(btf, &btf_ext->func_info, sec_name,
2720                                   insns_cnt, func_info, cnt);
2721 }
2722 
2723 int btf_ext__reloc_line_info(const struct btf *btf,
2724                              const struct btf_ext *btf_ext,
2725                              const char *sec_name, __u32 insns_cnt,
2726                              void **line_info, __u32 *cnt)
2727 {
2728         return btf_ext_reloc_info(btf, &btf_ext->line_info, sec_name,
2729                                   insns_cnt, line_info, cnt);
2730 }
2731 
2732 __u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext)
2733 {
2734         return btf_ext->func_info.rec_size;
2735 }
2736 
2737 __u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext)
2738 {
2739         return btf_ext->line_info.rec_size;
2740 }
2741 
2742 struct btf_dedup;
2743 
2744 static struct btf_dedup *btf_dedup_new(struct btf *btf, struct btf_ext *btf_ext,
2745                                        const struct btf_dedup_opts *opts);
2746 static void btf_dedup_free(struct btf_dedup *d);
2747 static int btf_dedup_prep(struct btf_dedup *d);
2748 static int btf_dedup_strings(struct btf_dedup *d);
2749 static int btf_dedup_prim_types(struct btf_dedup *d);
2750 static int btf_dedup_struct_types(struct btf_dedup *d);
2751 static int btf_dedup_ref_types(struct btf_dedup *d);
2752 static int btf_dedup_compact_types(struct btf_dedup *d);
2753 static int btf_dedup_remap_types(struct btf_dedup *d);
2754 
2755 /*
2756  * Deduplicate BTF types and strings.
2757  *
2758  * BTF dedup algorithm takes as an input `struct btf` representing `.BTF` ELF
2759  * section with all BTF type descriptors and string data. It overwrites that
2760  * memory in-place with deduplicated types and strings without any loss of
2761  * information. If optional `struct btf_ext` representing '.BTF.ext' ELF section
2762  * is provided, all the strings referenced from .BTF.ext section are honored
2763  * and updated to point to the right offsets after deduplication.
2764  *
2765  * If function returns with error, type/string data might be garbled and should
2766  * be discarded.
2767  *
2768  * More verbose and detailed description of both problem btf_dedup is solving,
2769  * as well as solution could be found at:
2770  * https://facebookmicrosites.github.io/bpf/blog/2018/11/14/btf-enhancement.html
2771  *
2772  * Problem description and justification
2773  * =====================================
2774  *
2775  * BTF type information is typically emitted either as a result of conversion
2776  * from DWARF to BTF or directly by compiler. In both cases, each compilation
2777  * unit contains information about a subset of all the types that are used
2778  * in an application. These subsets are frequently overlapping and contain a lot
2779  * of duplicated information when later concatenated together into a single
2780  * binary. This algorithm ensures that each unique type is represented by single
2781  * BTF type descriptor, greatly reducing resulting size of BTF data.
2782  *
2783  * Compilation unit isolation and subsequent duplication of data is not the only
2784  * problem. The same type hierarchy (e.g., struct and all the type that struct
2785  * references) in different compilation units can be represented in BTF to
2786  * various degrees of completeness (or, rather, incompleteness) due to
2787  * struct/union forward declarations.
2788  *
2789  * Let's take a look at an example, that we'll use to better understand the
2790  * problem (and solution). Suppose we have two compilation units, each using
2791  * same `struct S`, but each of them having incomplete type information about
2792  * struct's fields:
2793  *
2794  * // CU #1:
2795  * struct S;
2796  * struct A {
2797  *      int a;
2798  *      struct A* self;
2799  *      struct S* parent;
2800  * };
2801  * struct B;
2802  * struct S {
2803  *      struct A* a_ptr;
2804  *      struct B* b_ptr;
2805  * };
2806  *
2807  * // CU #2:
2808  * struct S;
2809  * struct A;
2810  * struct B {
2811  *      int b;
2812  *      struct B* self;
2813  *      struct S* parent;
2814  * };
2815  * struct S {
2816  *      struct A* a_ptr;
2817  *      struct B* b_ptr;
2818  * };
2819  *
2820  * In case of CU #1, BTF data will know only that `struct B` exist (but no
2821  * more), but will know the complete type information about `struct A`. While
2822  * for CU #2, it will know full type information about `struct B`, but will
2823  * only know about forward declaration of `struct A` (in BTF terms, it will
2824  * have `BTF_KIND_FWD` type descriptor with name `B`).
2825  *
2826  * This compilation unit isolation means that it's possible that there is no
2827  * single CU with complete type information describing structs `S`, `A`, and
2828  * `B`. Also, we might get tons of duplicated and redundant type information.
2829  *
2830  * Additional complication we need to keep in mind comes from the fact that
2831  * types, in general, can form graphs containing cycles, not just DAGs.
2832  *
2833  * While algorithm does deduplication, it also merges and resolves type
2834  * information (unless disabled throught `struct btf_opts`), whenever possible.
2835  * E.g., in the example above with two compilation units having partial type
2836  * information for structs `A` and `B`, the output of algorithm will emit
2837  * a single copy of each BTF type that describes structs `A`, `B`, and `S`
2838  * (as well as type information for `int` and pointers), as if they were defined
2839  * in a single compilation unit as:
2840  *
2841  * struct A {
2842  *      int a;
2843  *      struct A* self;
2844  *      struct S* parent;
2845  * };
2846  * struct B {
2847  *      int b;
2848  *      struct B* self;
2849  *      struct S* parent;
2850  * };
2851  * struct S {
2852  *      struct A* a_ptr;
2853  *      struct B* b_ptr;
2854  * };
2855  *
2856  * Algorithm summary
2857  * =================
2858  *
2859  * Algorithm completes its work in 6 separate passes:
2860  *
2861  * 1. Strings deduplication.
2862  * 2. Primitive types deduplication (int, enum, fwd).
2863  * 3. Struct/union types deduplication.
2864  * 4. Reference types deduplication (pointers, typedefs, arrays, funcs, func
2865  *    protos, and const/volatile/restrict modifiers).
2866  * 5. Types compaction.
2867  * 6. Types remapping.
2868  *
2869  * Algorithm determines canonical type descriptor, which is a single
2870  * representative type for each truly unique type. This canonical type is the
2871  * one that will go into final deduplicated BTF type information. For
2872  * struct/unions, it is also the type that algorithm will merge additional type
2873  * information into (while resolving FWDs), as it discovers it from data in
2874  * other CUs. Each input BTF type eventually gets either mapped to itself, if
2875  * that type is canonical, or to some other type, if that type is equivalent
2876  * and was chosen as canonical representative. This mapping is stored in
2877  * `btf_dedup->map` array. This map is also used to record STRUCT/UNION that
2878  * FWD type got resolved to.
2879  *
2880  * To facilitate fast discovery of canonical types, we also maintain canonical
2881  * index (`btf_dedup->dedup_table`), which maps type descriptor's signature hash
2882  * (i.e., hashed kind, name, size, fields, etc) into a list of canonical types
2883  * that match that signature. With sufficiently good choice of type signature
2884  * hashing function, we can limit number of canonical types for each unique type
2885  * signature to a very small number, allowing to find canonical type for any
2886  * duplicated type very quickly.
2887  *
2888  * Struct/union deduplication is the most critical part and algorithm for
2889  * deduplicating structs/unions is described in greater details in comments for
2890  * `btf_dedup_is_equiv` function.
2891  */
2892 int btf__dedup(struct btf *btf, struct btf_ext *btf_ext,
2893                const struct btf_dedup_opts *opts)
2894 {
2895         struct btf_dedup *d = btf_dedup_new(btf, btf_ext, opts);
2896         int err;
2897 
2898         if (IS_ERR(d)) {
2899                 pr_debug("btf_dedup_new failed: %ld", PTR_ERR(d));
2900                 return libbpf_err(-EINVAL);
2901         }
2902 
2903         if (btf_ensure_modifiable(btf))
2904                 return libbpf_err(-ENOMEM);
2905 
2906         err = btf_dedup_prep(d);
2907         if (err) {
2908                 pr_debug("btf_dedup_prep failed:%d\n", err);
2909                 goto done;
2910         }
2911         err = btf_dedup_strings(d);
2912         if (err < 0) {
2913                 pr_debug("btf_dedup_strings failed:%d\n", err);
2914                 goto done;
2915         }
2916         err = btf_dedup_prim_types(d);
2917         if (err < 0) {
2918                 pr_debug("btf_dedup_prim_types failed:%d\n", err);
2919                 goto done;
2920         }
2921         err = btf_dedup_struct_types(d);
2922         if (err < 0) {
2923                 pr_debug("btf_dedup_struct_types failed:%d\n", err);
2924                 goto done;
2925         }
2926         err = btf_dedup_ref_types(d);
2927         if (err < 0) {
2928                 pr_debug("btf_dedup_ref_types failed:%d\n", err);
2929                 goto done;
2930         }
2931         err = btf_dedup_compact_types(d);
2932         if (err < 0) {
2933                 pr_debug("btf_dedup_compact_types failed:%d\n", err);
2934                 goto done;
2935         }
2936         err = btf_dedup_remap_types(d);
2937         if (err < 0) {
2938                 pr_debug("btf_dedup_remap_types failed:%d\n", err);
2939                 goto done;
2940         }
2941 
2942 done:
2943         btf_dedup_free(d);
2944         return libbpf_err(err);
2945 }
2946 
2947 #define BTF_UNPROCESSED_ID ((__u32)-1)
2948 #define BTF_IN_PROGRESS_ID ((__u32)-2)
2949 
2950 struct btf_dedup {
2951         /* .BTF section to be deduped in-place */
2952         struct btf *btf;
2953         /*
2954          * Optional .BTF.ext section. When provided, any strings referenced
2955          * from it will be taken into account when deduping strings
2956          */
2957         struct btf_ext *btf_ext;
2958         /*
2959          * This is a map from any type's signature hash to a list of possible
2960          * canonical representative type candidates. Hash collisions are
2961          * ignored, so even types of various kinds can share same list of
2962          * candidates, which is fine because we rely on subsequent
2963          * btf_xxx_equal() checks to authoritatively verify type equality.
2964          */
2965         struct hashmap *dedup_table;
2966         /* Canonical types map */
2967         __u32 *map;
2968         /* Hypothetical mapping, used during type graph equivalence checks */
2969         __u32 *hypot_map;
2970         __u32 *hypot_list;
2971         size_t hypot_cnt;
2972         size_t hypot_cap;
2973         /* Whether hypothetical mapping, if successful, would need to adjust
2974          * already canonicalized types (due to a new forward declaration to
2975          * concrete type resolution). In such case, during split BTF dedup
2976          * candidate type would still be considered as different, because base
2977          * BTF is considered to be immutable.
2978          */
2979         bool hypot_adjust_canon;
2980         /* Various option modifying behavior of algorithm */
2981         struct btf_dedup_opts opts;
2982         /* temporary strings deduplication state */
2983         struct strset *strs_set;
2984 };
2985 
2986 static long hash_combine(long h, long value)
2987 {
2988         return h * 31 + value;
2989 }
2990 
2991 #define for_each_dedup_cand(d, node, hash) \
2992         hashmap__for_each_key_entry(d->dedup_table, node, (void *)hash)
2993 
2994 static int btf_dedup_table_add(struct btf_dedup *d, long hash, __u32 type_id)
2995 {
2996         return hashmap__append(d->dedup_table,
2997                                (void *)hash, (void *)(long)type_id);
2998 }
2999 
3000 static int btf_dedup_hypot_map_add(struct btf_dedup *d,
3001                                    __u32 from_id, __u32 to_id)
3002 {
3003         if (d->hypot_cnt == d->hypot_cap) {
3004                 __u32 *new_list;
3005 
3006                 d->hypot_cap += max((size_t)16, d->hypot_cap / 2);
3007                 new_list = libbpf_reallocarray(d->hypot_list, d->hypot_cap, sizeof(__u32));
3008                 if (!new_list)
3009                         return -ENOMEM;
3010                 d->hypot_list = new_list;
3011         }
3012         d->hypot_list[d->hypot_cnt++] = from_id;
3013         d->hypot_map[from_id] = to_id;
3014         return 0;
3015 }
3016 
3017 static void btf_dedup_clear_hypot_map(struct btf_dedup *d)
3018 {
3019         int i;
3020 
3021         for (i = 0; i < d->hypot_cnt; i++)
3022                 d->hypot_map[d->hypot_list[i]] = BTF_UNPROCESSED_ID;
3023         d->hypot_cnt = 0;
3024         d->hypot_adjust_canon = false;
3025 }
3026 
3027 static void btf_dedup_free(struct btf_dedup *d)
3028 {
3029         hashmap__free(d->dedup_table);
3030         d->dedup_table = NULL;
3031 
3032         free(d->map);
3033         d->map = NULL;
3034 
3035         free(d->hypot_map);
3036         d->hypot_map = NULL;
3037 
3038         free(d->hypot_list);
3039         d->hypot_list = NULL;
3040 
3041         free(d);
3042 }
3043 
3044 static size_t btf_dedup_identity_hash_fn(const void *key, void *ctx)
3045 {
3046         return (size_t)key;
3047 }
3048 
3049 static size_t btf_dedup_collision_hash_fn(const void *key, void *ctx)
3050 {
3051         return 0;
3052 }
3053 
3054 static bool btf_dedup_equal_fn(const void *k1, const void *k2, void *ctx)
3055 {
3056         return k1 == k2;
3057 }
3058 
3059 static struct btf_dedup *btf_dedup_new(struct btf *btf, struct btf_ext *btf_ext,
3060                                        const struct btf_dedup_opts *opts)
3061 {
3062         struct btf_dedup *d = calloc(1, sizeof(struct btf_dedup));
3063         hashmap_hash_fn hash_fn = btf_dedup_identity_hash_fn;
3064         int i, err = 0, type_cnt;
3065 
3066         if (!d)
3067                 return ERR_PTR(-ENOMEM);
3068 
3069         d->opts.dont_resolve_fwds = opts && opts->dont_resolve_fwds;
3070         /* dedup_table_size is now used only to force collisions in tests */
3071         if (opts && opts->dedup_table_size == 1)
3072                 hash_fn = btf_dedup_collision_hash_fn;
3073 
3074         d->btf = btf;
3075         d->btf_ext = btf_ext;
3076 
3077         d->dedup_table = hashmap__new(hash_fn, btf_dedup_equal_fn, NULL);
3078         if (IS_ERR(d->dedup_table)) {
3079                 err = PTR_ERR(d->dedup_table);
3080                 d->dedup_table = NULL;
3081                 goto done;
3082         }
3083 
3084         type_cnt = btf__get_nr_types(btf) + 1;
3085         d->map = malloc(sizeof(__u32) * type_cnt);
3086         if (!d->map) {
3087                 err = -ENOMEM;
3088                 goto done;
3089         }
3090         /* special BTF "void" type is made canonical immediately */
3091         d->map[0] = 0;
3092         for (i = 1; i < type_cnt; i++) {
3093                 struct btf_type *t = btf_type_by_id(d->btf, i);
3094 
3095                 /* VAR and DATASEC are never deduped and are self-canonical */
3096                 if (btf_is_var(t) || btf_is_datasec(t))
3097                         d->map[i] = i;
3098                 else
3099                         d->map[i] = BTF_UNPROCESSED_ID;
3100         }
3101 
3102         d->hypot_map = malloc(sizeof(__u32) * type_cnt);
3103         if (!d->hypot_map) {
3104                 err = -ENOMEM;
3105                 goto done;
3106         }
3107         for (i = 0; i < type_cnt; i++)
3108                 d->hypot_map[i] = BTF_UNPROCESSED_ID;
3109 
3110 done:
3111         if (err) {
3112                 btf_dedup_free(d);
3113                 return ERR_PTR(err);
3114         }
3115 
3116         return d;
3117 }
3118 
3119 /*
3120  * Iterate over all possible places in .BTF and .BTF.ext that can reference
3121  * string and pass pointer to it to a provided callback `fn`.
3122  */
3123 static int btf_for_each_str_off(struct btf_dedup *d, str_off_visit_fn fn, void *ctx)
3124 {
3125         int i, r;
3126 
3127         for (i = 0; i < d->btf->nr_types; i++) {
3128                 struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i);
3129 
3130                 r = btf_type_visit_str_offs(t, fn, ctx);
3131                 if (r)
3132                         return r;
3133         }
3134 
3135         if (!d->btf_ext)
3136                 return 0;
3137 
3138         r = btf_ext_visit_str_offs(d->btf_ext, fn, ctx);
3139         if (r)
3140                 return r;
3141 
3142         return 0;
3143 }
3144 
3145 static int strs_dedup_remap_str_off(__u32 *str_off_ptr, void *ctx)
3146 {
3147         struct btf_dedup *d = ctx;
3148         __u32 str_off = *str_off_ptr;
3149         const char *s;
3150         int off, err;
3151 
3152         /* don't touch empty string or string in main BTF */
3153         if (str_off == 0 || str_off < d->btf->start_str_off)
3154                 return 0;
3155 
3156         s = btf__str_by_offset(d->btf, str_off);
3157         if (d->btf->base_btf) {
3158                 err = btf__find_str(d->btf->base_btf, s);
3159                 if (err >= 0) {
3160                         *str_off_ptr = err;
3161                         return 0;
3162                 }
3163                 if (err != -ENOENT)
3164                         return err;
3165         }
3166 
3167         off = strset__add_str(d->strs_set, s);
3168         if (off < 0)
3169                 return off;
3170 
3171         *str_off_ptr = d->btf->start_str_off + off;
3172         return 0;
3173 }
3174 
3175 /*
3176  * Dedup string and filter out those that are not referenced from either .BTF
3177  * or .BTF.ext (if provided) sections.
3178  *
3179  * This is done by building index of all strings in BTF's string section,
3180  * then iterating over all entities that can reference strings (e.g., type
3181  * names, struct field names, .BTF.ext line info, etc) and marking corresponding
3182  * strings as used. After that all used strings are deduped and compacted into
3183  * sequential blob of memory and new offsets are calculated. Then all the string
3184  * references are iterated again and rewritten using new offsets.
3185  */
3186 static int btf_dedup_strings(struct btf_dedup *d)
3187 {
3188         int err;
3189 
3190         if (d->btf->strs_deduped)
3191                 return 0;
3192 
3193         d->strs_set = strset__new(BTF_MAX_STR_OFFSET, NULL, 0);
3194         if (IS_ERR(d->strs_set)) {
3195                 err = PTR_ERR(d->strs_set);
3196                 goto err_out;
3197         }
3198 
3199         if (!d->btf->base_btf) {
3200                 /* insert empty string; we won't be looking it up during strings
3201                  * dedup, but it's good to have it for generic BTF string lookups
3202                  */
3203                 err = strset__add_str(d->strs_set, "");
3204                 if (err < 0)
3205                         goto err_out;
3206         }
3207 
3208         /* remap string offsets */
3209         err = btf_for_each_str_off(d, strs_dedup_remap_str_off, d);
3210         if (err)
3211                 goto err_out;
3212 
3213         /* replace BTF string data and hash with deduped ones */
3214         strset__free(d->btf->strs_set);
3215         d->btf->hdr->str_len = strset__data_size(d->strs_set);
3216         d->btf->strs_set = d->strs_set;
3217         d->strs_set = NULL;
3218         d->btf->strs_deduped = true;
3219         return 0;
3220 
3221 err_out:
3222         strset__free(d->strs_set);
3223         d->strs_set = NULL;
3224 
3225         return err;
3226 }
3227 
3228 static long btf_hash_common(struct btf_type *t)
3229 {
3230         long h;
3231 
3232         h = hash_combine(0, t->name_off);
3233         h = hash_combine(h, t->info);
3234         h = hash_combine(h, t->size);
3235         return h;
3236 }
3237 
3238 static bool btf_equal_common(struct btf_type *t1, struct btf_type *t2)
3239 {
3240         return t1->name_off == t2->name_off &&
3241                t1->info == t2->info &&
3242                t1->size == t2->size;
3243 }
3244 
3245 /* Calculate type signature hash of INT. */
3246 static long btf_hash_int(struct btf_type *t)
3247 {
3248         __u32 info = *(__u32 *)(t + 1);
3249         long h;
3250 
3251         h = btf_hash_common(t);
3252         h = hash_combine(h, info);
3253         return h;
3254 }
3255 
3256 /* Check structural equality of two INTs. */
3257 static bool btf_equal_int(struct btf_type *t1, struct btf_type *t2)
3258 {
3259         __u32 info1, info2;
3260 
3261         if (!btf_equal_common(t1, t2))
3262                 return false;
3263         info1 = *(__u32 *)(t1 + 1);
3264         info2 = *(__u32 *)(t2 + 1);
3265         return info1 == info2;
3266 }
3267 
3268 /* Calculate type signature hash of ENUM. */
3269 static long btf_hash_enum(struct btf_type *t)
3270 {
3271         long h;
3272 
3273         /* don't hash vlen and enum members to support enum fwd resolving */
3274         h = hash_combine(0, t->name_off);
3275         h = hash_combine(h, t->info & ~0xffff);
3276         h = hash_combine(h, t->size);
3277         return h;
3278 }
3279 
3280 /* Check structural equality of two ENUMs. */
3281 static bool btf_equal_enum(struct btf_type *t1, struct btf_type *t2)
3282 {
3283         const struct btf_enum *m1, *m2;
3284         __u16 vlen;
3285         int i;
3286 
3287         if (!btf_equal_common(t1, t2))
3288                 return false;
3289 
3290         vlen = btf_vlen(t1);
3291         m1 = btf_enum(t1);
3292         m2 = btf_enum(t2);
3293         for (i = 0; i < vlen; i++) {
3294                 if (m1->name_off != m2->name_off || m1->val != m2->val)
3295                         return false;
3296                 m1++;
3297                 m2++;
3298         }
3299         return true;
3300 }
3301 
3302 static inline bool btf_is_enum_fwd(struct btf_type *t)
3303 {
3304         return btf_is_enum(t) && btf_vlen(t) == 0;
3305 }
3306 
3307 static bool btf_compat_enum(struct btf_type *t1, struct btf_type *t2)
3308 {
3309         if (!btf_is_enum_fwd(t1) && !btf_is_enum_fwd(t2))
3310                 return btf_equal_enum(t1, t2);
3311         /* ignore vlen when comparing */
3312         return t1->name_off == t2->name_off &&
3313                (t1->info & ~0xffff) == (t2->info & ~0xffff) &&
3314                t1->size == t2->size;
3315 }
3316 
3317 /*
3318  * Calculate type signature hash of STRUCT/UNION, ignoring referenced type IDs,
3319  * as referenced type IDs equivalence is established separately during type
3320  * graph equivalence check algorithm.
3321  */
3322 static long btf_hash_struct(struct btf_type *t)
3323 {
3324         const struct btf_member *member = btf_members(t);
3325         __u32 vlen = btf_vlen(t);
3326         long h = btf_hash_common(t);
3327         int i;
3328 
3329         for (i = 0; i < vlen; i++) {
3330                 h = hash_combine(h, member->name_off);
3331                 h = hash_combine(h, member->offset);
3332                 /* no hashing of referenced type ID, it can be unresolved yet */
3333                 member++;
3334         }
3335         return h;
3336 }
3337 
3338 /*
3339  * Check structural compatibility of two FUNC_PROTOs, ignoring referenced type
3340  * IDs. This check is performed during type graph equivalence check and
3341  * referenced types equivalence is checked separately.
3342  */
3343 static bool btf_shallow_equal_struct(struct btf_type *t1, struct btf_type *t2)
3344 {
3345         const struct btf_member *m1, *m2;
3346         __u16 vlen;
3347         int i;
3348 
3349         if (!btf_equal_common(t1, t2))
3350                 return false;
3351 
3352         vlen = btf_vlen(t1);
3353         m1 = btf_members(t1);
3354         m2 = btf_members(t2);
3355         for (i = 0; i < vlen; i++) {
3356                 if (m1->name_off != m2->name_off || m1->offset != m2->offset)
3357                         return false;
3358                 m1++;
3359                 m2++;
3360         }
3361         return true;
3362 }
3363 
3364 /*
3365  * Calculate type signature hash of ARRAY, including referenced type IDs,
3366  * under assumption that they were already resolved to canonical type IDs and
3367  * are not going to change.
3368  */
3369 static long btf_hash_array(struct btf_type *t)
3370 {
3371         const struct btf_array *info = btf_array(t);
3372         long h = btf_hash_common(t);
3373 
3374         h = hash_combine(h, info->type);
3375         h = hash_combine(h, info->index_type);
3376         h = hash_combine(h, info->nelems);
3377         return h;
3378 }
3379 
3380 /*
3381  * Check exact equality of two ARRAYs, taking into account referenced
3382  * type IDs, under assumption that they were already resolved to canonical
3383  * type IDs and are not going to change.
3384  * This function is called during reference types deduplication to compare
3385  * ARRAY to potential canonical representative.
3386  */
3387 static bool btf_equal_array(struct btf_type *t1, struct btf_type *t2)
3388 {
3389         const struct btf_array *info1, *info2;
3390 
3391         if (!btf_equal_common(t1, t2))
3392                 return false;
3393 
3394         info1 = btf_array(t1);
3395         info2 = btf_array(t2);
3396         return info1->type == info2->type &&
3397                info1->index_type == info2->index_type &&
3398                info1->nelems == info2->nelems;
3399 }
3400 
3401 /*
3402  * Check structural compatibility of two ARRAYs, ignoring referenced type
3403  * IDs. This check is performed during type graph equivalence check and
3404  * referenced types equivalence is checked separately.
3405  */
3406 static bool btf_compat_array(struct btf_type *t1, struct btf_type *t2)
3407 {
3408         if (!btf_equal_common(t1, t2))
3409                 return false;
3410 
3411         return btf_array(t1)->nelems == btf_array(t2)->nelems;
3412 }
3413 
3414 /*
3415  * Calculate type signature hash of FUNC_PROTO, including referenced type IDs,
3416  * under assumption that they were already resolved to canonical type IDs and
3417  * are not going to change.
3418  */
3419 static long btf_hash_fnproto(struct btf_type *t)
3420 {
3421         const struct btf_param *member = btf_params(t);
3422         __u16 vlen = btf_vlen(t);
3423         long h = btf_hash_common(t);
3424         int i;
3425 
3426         for (i = 0; i < vlen; i++) {
3427                 h = hash_combine(h, member->name_off);
3428                 h = hash_combine(h, member->type);
3429                 member++;
3430         }
3431         return h;
3432 }
3433 
3434 /*
3435  * Check exact equality of two FUNC_PROTOs, taking into account referenced
3436  * type IDs, under assumption that they were already resolved to canonical
3437  * type IDs and are not going to change.
3438  * This function is called during reference types deduplication to compare
3439  * FUNC_PROTO to potential canonical representative.
3440  */
3441 static bool btf_equal_fnproto(struct btf_type *t1, struct btf_type *t2)
3442 {
3443         const struct btf_param *m1, *m2;
3444         __u16 vlen;
3445         int i;
3446 
3447         if (!btf_equal_common(t1, t2))
3448                 return false;
3449 
3450         vlen = btf_vlen(t1);
3451         m1 = btf_params(t1);
3452         m2 = btf_params(t2);
3453         for (i = 0; i < vlen; i++) {
3454                 if (m1->name_off != m2->name_off || m1->type != m2->type)
3455                         return false;
3456                 m1++;
3457                 m2++;
3458         }
3459         return true;
3460 }
3461 
3462 /*
3463  * Check structural compatibility of two FUNC_PROTOs, ignoring referenced type
3464  * IDs. This check is performed during type graph equivalence check and
3465  * referenced types equivalence is checked separately.
3466  */
3467 static bool btf_compat_fnproto(struct btf_type *t1, struct btf_type *t2)
3468 {
3469         const struct btf_param *m1, *m2;
3470         __u16 vlen;
3471         int i;
3472 
3473         /* skip return type ID */
3474         if (t1->name_off != t2->name_off || t1->info != t2->info)
3475                 return false;
3476 
3477         vlen = btf_vlen(t1);
3478         m1 = btf_params(t1);
3479         m2 = btf_params(t2);
3480         for (i = 0; i < vlen; i++) {
3481                 if (m1->name_off != m2->name_off)
3482                         return false;
3483                 m1++;
3484                 m2++;
3485         }
3486         return true;
3487 }
3488 
3489 /* Prepare split BTF for deduplication by calculating hashes of base BTF's
3490  * types and initializing the rest of the state (canonical type mapping) for
3491  * the fixed base BTF part.
3492  */
3493 static int btf_dedup_prep(struct btf_dedup *d)
3494 {
3495         struct btf_type *t;
3496         int type_id;
3497         long h;
3498 
3499         if (!d->btf->base_btf)
3500                 return 0;
3501 
3502         for (type_id = 1; type_id < d->btf->start_id; type_id++) {
3503                 t = btf_type_by_id(d->btf, type_id);
3504 
3505                 /* all base BTF types are self-canonical by definition */
3506                 d->map[type_id] = type_id;
3507 
3508                 switch (btf_kind(t)) {
3509                 case BTF_KIND_VAR:
3510                 case BTF_KIND_DATASEC:
3511                         /* VAR and DATASEC are never hash/deduplicated */
3512                         continue;
3513                 case BTF_KIND_CONST:
3514                 case BTF_KIND_VOLATILE:
3515                 case BTF_KIND_RESTRICT:
3516                 case BTF_KIND_PTR:
3517                 case BTF_KIND_FWD:
3518                 case BTF_KIND_TYPEDEF:
3519                 case BTF_KIND_FUNC:
3520                 case BTF_KIND_FLOAT:
3521                         h = btf_hash_common(t);
3522                         break;
3523                 case BTF_KIND_INT:
3524                         h = btf_hash_int(t);
3525                         break;
3526                 case BTF_KIND_ENUM:
3527                         h = btf_hash_enum(t);
3528                         break;
3529                 case BTF_KIND_STRUCT:
3530                 case BTF_KIND_UNION:
3531                         h = btf_hash_struct(t);
3532                         break;
3533                 case BTF_KIND_ARRAY:
3534                         h = btf_hash_array(t);
3535                         break;
3536                 case BTF_KIND_FUNC_PROTO:
3537                         h = btf_hash_fnproto(t);
3538                         break;
3539                 default:
3540                         pr_debug("unknown kind %d for type [%d]\n", btf_kind(t), type_id);
3541                         return -EINVAL;
3542                 }
3543                 if (btf_dedup_table_add(d, h, type_id))
3544                         return -ENOMEM;
3545         }
3546 
3547         return 0;
3548 }
3549 
3550 /*
3551  * Deduplicate primitive types, that can't reference other types, by calculating
3552  * their type signature hash and comparing them with any possible canonical
3553  * candidate. If no canonical candidate matches, type itself is marked as
3554  * canonical and is added into `btf_dedup->dedup_table` as another candidate.
3555  */
3556 static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id)
3557 {
3558         struct btf_type *t = btf_type_by_id(d->btf, type_id);
3559         struct hashmap_entry *hash_entry;
3560         struct btf_type *cand;
3561         /* if we don't find equivalent type, then we are canonical */
3562         __u32 new_id = type_id;
3563         __u32 cand_id;
3564         long h;
3565 
3566         switch (btf_kind(t)) {
3567         case BTF_KIND_CONST:
3568         case BTF_KIND_VOLATILE:
3569         case BTF_KIND_RESTRICT:
3570         case BTF_KIND_PTR:
3571         case BTF_KIND_TYPEDEF:
3572         case BTF_KIND_ARRAY:
3573         case BTF_KIND_STRUCT:
3574         case BTF_KIND_UNION:
3575         case BTF_KIND_FUNC:
3576         case BTF_KIND_FUNC_PROTO:
3577         case BTF_KIND_VAR:
3578         case BTF_KIND_DATASEC:
3579                 return 0;
3580 
3581         case BTF_KIND_INT:
3582                 h = btf_hash_int(t);
3583                 for_each_dedup_cand(d, hash_entry, h) {
3584                         cand_id = (__u32)(long)hash_entry->value;
3585                         cand = btf_type_by_id(d->btf, cand_id);
3586                         if (btf_equal_int(t, cand)) {
3587                                 new_id = cand_id;
3588                                 break;
3589                         }
3590                 }
3591                 break;
3592 
3593         case BTF_KIND_ENUM:
3594                 h = btf_hash_enum(t);
3595                 for_each_dedup_cand(d, hash_entry, h) {
3596                         cand_id = (__u32)(long)hash_entry->value;
3597                         cand = btf_type_by_id(d->btf, cand_id);
3598                         if (btf_equal_enum(t, cand)) {
3599                                 new_id = cand_id;
3600                                 break;
3601                         }
3602                         if (d->opts.dont_resolve_fwds)
3603                                 continue;
3604                         if (btf_compat_enum(t, cand)) {
3605                                 if (btf_is_enum_fwd(t)) {
3606                                         /* resolve fwd to full enum */
3607                                         new_id = cand_id;
3608                                         break;
3609                                 }
3610                                 /* resolve canonical enum fwd to full enum */
3611                                 d->map[cand_id] = type_id;
3612                         }
3613                 }
3614                 break;
3615 
3616         case BTF_KIND_FWD:
3617         case BTF_KIND_FLOAT:
3618                 h = btf_hash_common(t);
3619                 for_each_dedup_cand(d, hash_entry, h) {
3620                         cand_id = (__u32)(long)hash_entry->value;
3621                         cand = btf_type_by_id(d->btf, cand_id);
3622                         if (btf_equal_common(t, cand)) {
3623                                 new_id = cand_id;
3624                                 break;
3625                         }
3626                 }
3627                 break;
3628 
3629         default:
3630                 return -EINVAL;
3631         }
3632 
3633         d->map[type_id] = new_id;
3634         if (type_id == new_id && btf_dedup_table_add(d, h, type_id))
3635                 return -ENOMEM;
3636 
3637         return 0;
3638 }
3639 
3640 static int btf_dedup_prim_types(struct btf_dedup *d)
3641 {
3642         int i, err;
3643 
3644         for (i = 0; i < d->btf->nr_types; i++) {
3645                 err = btf_dedup_prim_type(d, d->btf->start_id + i);
3646                 if (err)
3647                         return err;
3648         }
3649         return 0;
3650 }
3651 
3652 /*
3653  * Check whether type is already mapped into canonical one (could be to itself).
3654  */
3655 static inline bool is_type_mapped(struct btf_dedup *d, uint32_t type_id)
3656 {
3657         return d->map[type_id] <= BTF_MAX_NR_TYPES;
3658 }
3659 
3660 /*
3661  * Resolve type ID into its canonical type ID, if any; otherwise return original
3662  * type ID. If type is FWD and is resolved into STRUCT/UNION already, follow
3663  * STRUCT/UNION link and resolve it into canonical type ID as well.
3664  */
3665 static inline __u32 resolve_type_id(struct btf_dedup *d, __u32 type_id)
3666 {
3667         while (is_type_mapped(d, type_id) && d->map[type_id] != type_id)
3668                 type_id = d->map[type_id];
3669         return type_id;
3670 }
3671 
3672 /*
3673  * Resolve FWD to underlying STRUCT/UNION, if any; otherwise return original
3674  * type ID.
3675  */
3676 static uint32_t resolve_fwd_id(struct btf_dedup *d, uint32_t type_id)
3677 {
3678         __u32 orig_type_id = type_id;
3679 
3680         if (!btf_is_fwd(btf__type_by_id(d->btf, type_id)))
3681                 return type_id;
3682 
3683         while (is_type_mapped(d, type_id) && d->map[type_id] != type_id)
3684                 type_id = d->map[type_id];
3685 
3686         if (!btf_is_fwd(btf__type_by_id(d->btf, type_id)))
3687                 return type_id;
3688 
3689         return orig_type_id;
3690 }
3691 
3692 
3693 static inline __u16 btf_fwd_kind(struct btf_type *t)
3694 {
3695         return btf_kflag(t) ? BTF_KIND_UNION : BTF_KIND_STRUCT;
3696 }
3697 
3698 /* Check if given two types are identical ARRAY definitions */
3699 static int btf_dedup_identical_arrays(struct btf_dedup *d, __u32 id1, __u32 id2)
3700 {
3701         struct btf_type *t1, *t2;
3702 
3703         t1 = btf_type_by_id(d->btf, id1);
3704         t2 = btf_type_by_id(d->btf, id2);
3705         if (!btf_is_array(t1) || !btf_is_array(t2))
3706                 return 0;
3707 
3708         return btf_equal_array(t1, t2);
3709 }
3710 
3711 /*
3712  * Check equivalence of BTF type graph formed by candidate struct/union (we'll
3713  * call it "candidate graph" in this description for brevity) to a type graph
3714  * formed by (potential) canonical struct/union ("canonical graph" for brevity
3715  * here, though keep in mind that not all types in canonical graph are
3716  * necessarily canonical representatives themselves, some of them might be
3717  * duplicates or its uniqueness might not have been established yet).
3718  * Returns:
3719  *  - >0, if type graphs are equivalent;
3720  *  -  0, if not equivalent;
3721  *  - <0, on error.
3722  *
3723  * Algorithm performs side-by-side DFS traversal of both type graphs and checks
3724  * equivalence of BTF types at each step. If at any point BTF types in candidate
3725  * and canonical graphs are not compatible structurally, whole graphs are
3726  * incompatible. If types are structurally equivalent (i.e., all information
3727  * except referenced type IDs is exactly the same), a mapping from `canon_id` to
3728  * a `cand_id` is recored in hypothetical mapping (`btf_dedup->hypot_map`).
3729  * If a type references other types, then those referenced types are checked
3730  * for equivalence recursively.
3731  *
3732  * During DFS traversal, if we find that for current `canon_id` type we
3733  * already have some mapping in hypothetical map, we check for two possible
3734  * situations:
3735  *   - `canon_id` is mapped to exactly the same type as `cand_id`. This will
3736  *     happen when type graphs have cycles. In this case we assume those two
3737  *     types are equivalent.
3738  *   - `canon_id` is mapped to different type. This is contradiction in our
3739  *     hypothetical mapping, because same graph in canonical graph corresponds
3740  *     to two different types in candidate graph, which for equivalent type
3741  *     graphs shouldn't happen. This condition terminates equivalence check
3742  *     with negative result.
3743  *
3744  * If type graphs traversal exhausts types to check and find no contradiction,
3745  * then type graphs are equivalent.
3746  *
3747  * When checking types for equivalence, there is one special case: FWD types.
3748  * If FWD type resolution is allowed and one of the types (either from canonical
3749  * or candidate graph) is FWD and other is STRUCT/UNION (depending on FWD's kind
3750  * flag) and their names match, hypothetical mapping is updated to point from
3751  * FWD to STRUCT/UNION. If graphs will be determined as equivalent successfully,
3752  * this mapping will be used to record FWD -> STRUCT/UNION mapping permanently.
3753  *
3754  * Technically, this could lead to incorrect FWD to STRUCT/UNION resolution,
3755  * if there are two exactly named (or anonymous) structs/unions that are
3756  * compatible structurally, one of which has FWD field, while other is concrete
3757  * STRUCT/UNION, but according to C sources they are different structs/unions
3758  * that are referencing different types with the same name. This is extremely
3759  * unlikely to happen, but btf_dedup API allows to disable FWD resolution if
3760  * this logic is causing problems.
3761  *
3762  * Doing FWD resolution means that both candidate and/or canonical graphs can
3763  * consists of portions of the graph that come from multiple compilation units.
3764  * This is due to the fact that types within single compilation unit are always
3765  * deduplicated and FWDs are already resolved, if referenced struct/union
3766  * definiton is available. So, if we had unresolved FWD and found corresponding
3767  * STRUCT/UNION, they will be from different compilation units. This
3768  * consequently means that when we "link" FWD to corresponding STRUCT/UNION,
3769  * type graph will likely have at least two different BTF types that describe
3770  * same type (e.g., most probably there will be two different BTF types for the
3771  * same 'int' primitive type) and could even have "overlapping" parts of type
3772  * graph that describe same subset of types.
3773  *
3774  * This in turn means that our assumption that each type in canonical graph
3775  * must correspond to exactly one type in candidate graph might not hold
3776  * anymore and will make it harder to detect contradictions using hypothetical
3777  * map. To handle this problem, we allow to follow FWD -> STRUCT/UNION
3778  * resolution only in canonical graph. FWDs in candidate graphs are never
3779  * resolved. To see why it's OK, let's check all possible situations w.r.t. FWDs
3780  * that can occur:
3781  *   - Both types in canonical and candidate graphs are FWDs. If they are
3782  *     structurally equivalent, then they can either be both resolved to the
3783  *     same STRUCT/UNION or not resolved at all. In both cases they are
3784  *     equivalent and there is no need to resolve FWD on candidate side.
3785  *   - Both types in canonical and candidate graphs are concrete STRUCT/UNION,
3786  *     so nothing to resolve as well, algorithm will check equivalence anyway.
3787  *   - Type in canonical graph is FWD, while type in candidate is concrete
3788  *     STRUCT/UNION. In this case candidate graph comes from single compilation
3789  *     unit, so there is exactly one BTF type for each unique C type. After
3790  *     resolving FWD into STRUCT/UNION, there might be more than one BTF type
3791  *     in canonical graph mapping to single BTF type in candidate graph, but
3792  *     because hypothetical mapping maps from canonical to candidate types, it's
3793  *     alright, and we still maintain the property of having single `canon_id`
3794  *     mapping to single `cand_id` (there could be two different `canon_id`
3795  *     mapped to the same `cand_id`, but it's not contradictory).
3796  *   - Type in canonical graph is concrete STRUCT/UNION, while type in candidate
3797  *     graph is FWD. In this case we are just going to check compatibility of
3798  *     STRUCT/UNION and corresponding FWD, and if they are compatible, we'll
3799  *     assume that whatever STRUCT/UNION FWD resolves to must be equivalent to
3800  *     a concrete STRUCT/UNION from canonical graph. If the rest of type graphs
3801  *     turn out equivalent, we'll re-resolve FWD to concrete STRUCT/UNION from
3802  *     canonical graph.
3803  */
3804 static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id,
3805                               __u32 canon_id)
3806 {
3807         struct btf_type *cand_type;
3808         struct btf_type *canon_type;
3809         __u32 hypot_type_id;
3810         __u16 cand_kind;
3811         __u16 canon_kind;
3812         int i, eq;
3813 
3814         /* if both resolve to the same canonical, they must be equivalent */
3815         if (resolve_type_id(d, cand_id) == resolve_type_id(d, canon_id))
3816                 return 1;
3817 
3818         canon_id = resolve_fwd_id(d, canon_id);
3819 
3820         hypot_type_id = d->hypot_map[canon_id];
3821         if (hypot_type_id <= BTF_MAX_NR_TYPES) {
3822                 /* In some cases compiler will generate different DWARF types
3823                  * for *identical* array type definitions and use them for
3824                  * different fields within the *same* struct. This breaks type
3825                  * equivalence check, which makes an assumption that candidate
3826                  * types sub-graph has a consistent and deduped-by-compiler
3827                  * types within a single CU. So work around that by explicitly
3828                  * allowing identical array types here.
3829                  */
3830                 return hypot_type_id == cand_id ||
3831                        btf_dedup_identical_arrays(d, hypot_type_id, cand_id);
3832         }
3833 
3834         if (btf_dedup_hypot_map_add(d, canon_id, cand_id))
3835                 return -ENOMEM;
3836 
3837         cand_type = btf_type_by_id(d->btf, cand_id);
3838         canon_type = btf_type_by_id(d->btf, canon_id);
3839         cand_kind = btf_kind(cand_type);
3840         canon_kind = btf_kind(canon_type);
3841 
3842         if (cand_type->name_off != canon_type->name_off)
3843                 return 0;
3844 
3845         /* FWD <--> STRUCT/UNION equivalence check, if enabled */
3846         if (!d->opts.dont_resolve_fwds
3847             && (cand_kind == BTF_KIND_FWD || canon_kind == BTF_KIND_FWD)
3848             && cand_kind != canon_kind) {
3849                 __u16 real_kind;
3850                 __u16 fwd_kind;
3851 
3852                 if (cand_kind == BTF_KIND_FWD) {
3853                         real_kind = canon_kind;
3854                         fwd_kind = btf_fwd_kind(cand_type);
3855                 } else {
3856                         real_kind = cand_kind;
3857                         fwd_kind = btf_fwd_kind(canon_type);
3858                         /* we'd need to resolve base FWD to STRUCT/UNION */
3859                         if (fwd_kind == real_kind && canon_id < d->btf->start_id)
3860                                 d->hypot_adjust_canon = true;
3861                 }
3862                 return fwd_kind == real_kind;
3863         }
3864 
3865         if (cand_kind != canon_kind)
3866                 return 0;
3867 
3868         switch (cand_kind) {
3869         case BTF_KIND_INT:
3870                 return btf_equal_int(cand_type, canon_type);
3871 
3872         case BTF_KIND_ENUM:
3873                 if (d->opts.dont_resolve_fwds)
3874                         return btf_equal_enum(cand_type, canon_type);
3875                 else
3876                         return btf_compat_enum(cand_type, canon_type);
3877 
3878         case BTF_KIND_FWD:
3879         case BTF_KIND_FLOAT:
3880                 return btf_equal_common(cand_type, canon_type);
3881 
3882         case BTF_KIND_CONST:
3883         case BTF_KIND_VOLATILE:
3884         case BTF_KIND_RESTRICT:
3885         case BTF_KIND_PTR:
3886         case BTF_KIND_TYPEDEF:
3887         case BTF_KIND_FUNC:
3888                 if (cand_type->info != canon_type->info)
3889                         return 0;
3890                 return btf_dedup_is_equiv(d, cand_type->type, canon_type->type);
3891 
3892         case BTF_KIND_ARRAY: {
3893                 const struct btf_array *cand_arr, *canon_arr;
3894 
3895                 if (!btf_compat_array(cand_type, canon_type))
3896                         return 0;
3897                 cand_arr = btf_array(cand_type);
3898                 canon_arr = btf_array(canon_type);
3899                 eq = btf_dedup_is_equiv(d, cand_arr->index_type, canon_arr->index_type);
3900                 if (eq <= 0)
3901                         return eq;
3902                 return btf_dedup_is_equiv(d, cand_arr->type, canon_arr->type);
3903         }
3904 
3905         case BTF_KIND_STRUCT:
3906         case BTF_KIND_UNION: {
3907                 const struct btf_member *cand_m, *canon_m;
3908                 __u16 vlen;
3909 
3910                 if (!btf_shallow_equal_struct(cand_type, canon_type))
3911                         return 0;
3912                 vlen = btf_vlen(cand_type);
3913                 cand_m = btf_members(cand_type);
3914                 canon_m = btf_members(canon_type);
3915                 for (i = 0; i < vlen; i++) {
3916                         eq = btf_dedup_is_equiv(d, cand_m->type, canon_m->type);
3917                         if (eq <= 0)
3918                                 return eq;
3919                         cand_m++;
3920                         canon_m++;
3921                 }
3922 
3923                 return 1;
3924         }
3925 
3926         case BTF_KIND_FUNC_PROTO: {
3927                 const struct btf_param *cand_p, *canon_p;
3928                 __u16 vlen;
3929 
3930                 if (!btf_compat_fnproto(cand_type, canon_type))
3931                         return 0;
3932                 eq = btf_dedup_is_equiv(d, cand_type->type, canon_type->type);
3933                 if (eq <= 0)
3934                         return eq;
3935                 vlen = btf_vlen(cand_type);
3936                 cand_p = btf_params(cand_type);
3937                 canon_p = btf_params(canon_type);
3938                 for (i = 0; i < vlen; i++) {
3939                         eq = btf_dedup_is_equiv(d, cand_p->type, canon_p->type);
3940                         if (eq <= 0)
3941                                 return eq;
3942                         cand_p++;
3943                         canon_p++;
3944                 }
3945                 return 1;
3946         }
3947 
3948         default:
3949                 return -EINVAL;
3950         }
3951         return 0;
3952 }
3953 
3954 /*
3955  * Use hypothetical mapping, produced by successful type graph equivalence
3956  * check, to augment existing struct/union canonical mapping, where possible.
3957  *
3958  * If BTF_KIND_FWD resolution is allowed, this mapping is also used to record
3959  * FWD -> STRUCT/UNION correspondence as well. FWD resolution is bidirectional:
3960  * it doesn't matter if FWD type was part of canonical graph or candidate one,
3961  * we are recording the mapping anyway. As opposed to carefulness required
3962  * for struct/union correspondence mapping (described below), for FWD resolution
3963  * it's not important, as by the time that FWD type (reference type) will be
3964  * deduplicated all structs/unions will be deduped already anyway.
3965  *
3966  * Recording STRUCT/UNION mapping is purely a performance optimization and is
3967  * not required for correctness. It needs to be done carefully to ensure that
3968  * struct/union from candidate's type graph is not mapped into corresponding
3969  * struct/union from canonical type graph that itself hasn't been resolved into
3970  * canonical representative. The only guarantee we have is that canonical
3971  * struct/union was determined as canonical and that won't change. But any
3972  * types referenced through that struct/union fields could have been not yet
3973  * resolved, so in case like that it's too early to establish any kind of
3974  * correspondence between structs/unions.
3975  *
3976  * No canonical correspondence is derived for primitive types (they are already
3977  * deduplicated completely already anyway) or reference types (they rely on
3978  * stability of struct/union canonical relationship for equivalence checks).
3979  */
3980 static void btf_dedup_merge_hypot_map(struct btf_dedup *d)
3981 {
3982         __u32 canon_type_id, targ_type_id;
3983         __u16 t_kind, c_kind;
3984         __u32 t_id, c_id;
3985         int i;
3986 
3987         for (i = 0; i < d->hypot_cnt; i++) {
3988                 canon_type_id = d->hypot_list[i];
3989                 targ_type_id = d->hypot_map[canon_type_id];
3990                 t_id = resolve_type_id(d, targ_type_id);
3991                 c_id = resolve_type_id(d, canon_type_id);
3992                 t_kind = btf_kind(btf__type_by_id(d->btf, t_id));
3993                 c_kind = btf_kind(btf__type_by_id(d->btf, c_id));
3994                 /*
3995                  * Resolve FWD into STRUCT/UNION.
3996                  * It's ok to resolve FWD into STRUCT/UNION that's not yet
3997                  * mapped to canonical representative (as opposed to
3998                  * STRUCT/UNION <--> STRUCT/UNION mapping logic below), because
3999                  * eventually that struct is going to be mapped and all resolved
4000                  * FWDs will automatically resolve to correct canonical
4001                  * representative. This will happen before ref type deduping,
4002                  * which critically depends on stability of these mapping. This
4003                  * stability is not a requirement for STRUCT/UNION equivalence
4004                  * checks, though.
4005                  */
4006 
4007                 /* if it's the split BTF case, we still need to point base FWD
4008                  * to STRUCT/UNION in a split BTF, because FWDs from split BTF
4009                  * will be resolved against base FWD. If we don't point base
4010                  * canonical FWD to the resolved STRUCT/UNION, then all the
4011                  * FWDs in split BTF won't be correctly resolved to a proper
4012                  * STRUCT/UNION.
4013                  */
4014                 if (t_kind != BTF_KIND_FWD && c_kind == BTF_KIND_FWD)
4015                         d->map[c_id] = t_id;
4016 
4017                 /* if graph equivalence determined that we'd need to adjust
4018                  * base canonical types, then we need to only point base FWDs
4019                  * to STRUCTs/UNIONs and do no more modifications. For all
4020                  * other purposes the type graphs were not equivalent.
4021                  */
4022                 if (d->hypot_adjust_canon)
4023                         continue;
4024                 
4025                 if (t_kind == BTF_KIND_FWD && c_kind != BTF_KIND_FWD)
4026                         d->map[t_id] = c_id;
4027 
4028                 if ((t_kind == BTF_KIND_STRUCT || t_kind == BTF_KIND_UNION) &&
4029                     c_kind != BTF_KIND_FWD &&
4030                     is_type_mapped(d, c_id) &&
4031                     !is_type_mapped(d, t_id)) {
4032                         /*
4033                          * as a perf optimization, we can map struct/union
4034                          * that's part of type graph we just verified for
4035                          * equivalence. We can do that for struct/union that has
4036                          * canonical representative only, though.
4037                          */
4038                         d->map[t_id] = c_id;
4039                 }
4040         }
4041 }
4042 
4043 /*
4044  * Deduplicate struct/union types.
4045  *
4046  * For each struct/union type its type signature hash is calculated, taking
4047  * into account type's name, size, number, order and names of fields, but
4048  * ignoring type ID's referenced from fields, because they might not be deduped
4049  * completely until after reference types deduplication phase. This type hash
4050  * is used to iterate over all potential canonical types, sharing same hash.
4051  * For each canonical candidate we check whether type graphs that they form
4052  * (through referenced types in fields and so on) are equivalent using algorithm
4053  * implemented in `btf_dedup_is_equiv`. If such equivalence is found and
4054  * BTF_KIND_FWD resolution is allowed, then hypothetical mapping
4055  * (btf_dedup->hypot_map) produced by aforementioned type graph equivalence
4056  * algorithm is used to record FWD -> STRUCT/UNION mapping. It's also used to
4057  * potentially map other structs/unions to their canonical representatives,
4058  * if such relationship hasn't yet been established. This speeds up algorithm
4059  * by eliminating some of the duplicate work.
4060  *
4061  * If no matching canonical representative was found, struct/union is marked
4062  * as canonical for itself and is added into btf_dedup->dedup_table hash map
4063  * for further look ups.
4064  */
4065 static int btf_dedup_struct_type(struct btf_dedup *d, __u32 type_id)
4066 {
4067         struct btf_type *cand_type, *t;
4068         struct hashmap_entry *hash_entry;
4069         /* if we don't find equivalent type, then we are canonical */
4070         __u32 new_id = type_id;
4071         __u16 kind;
4072         long h;
4073 
4074         /* already deduped or is in process of deduping (loop detected) */
4075         if (d->map[type_id] <= BTF_MAX_NR_TYPES)
4076                 return 0;
4077 
4078         t = btf_type_by_id(d->btf, type_id);
4079         kind = btf_kind(t);
4080 
4081         if (kind != BTF_KIND_STRUCT && kind != BTF_KIND_UNION)
4082                 return 0;
4083 
4084         h = btf_hash_struct(t);
4085         for_each_dedup_cand(d, hash_entry, h) {
4086                 __u32 cand_id = (__u32)(long)hash_entry->value;
4087                 int eq;
4088 
4089                 /*
4090                  * Even though btf_dedup_is_equiv() checks for
4091                  * btf_shallow_equal_struct() internally when checking two
4092                  * structs (unions) for equivalence, we need to guard here
4093                  * from picking matching FWD type as a dedup candidate.
4094                  * This can happen due to hash collision. In such case just
4095                  * relying on btf_dedup_is_equiv() would lead to potentially
4096                  * creating a loop (FWD -> STRUCT and STRUCT -> FWD), because
4097                  * FWD and compatible STRUCT/UNION are considered equivalent.
4098                  */
4099                 cand_type = btf_type_by_id(d->btf, cand_id);
4100                 if (!btf_shallow_equal_struct(t, cand_type))
4101                         continue;
4102 
4103                 btf_dedup_clear_hypot_map(d);
4104                 eq = btf_dedup_is_equiv(d, type_id, cand_id);
4105                 if (eq < 0)
4106                         return eq;
4107                 if (!eq)
4108                         continue;
4109                 btf_dedup_merge_hypot_map(d);
4110                 if (d->hypot_adjust_canon) /* not really equivalent */
4111                         continue;
4112                 new_id = cand_id;
4113                 break;
4114         }
4115 
4116         d->map[type_id] = new_id;
4117         if (type_id == new_id && btf_dedup_table_add(d, h, type_id))
4118                 return -ENOMEM;
4119 
4120         return 0;
4121 }
4122 
4123 static int btf_dedup_struct_types(struct btf_dedup *d)
4124 {
4125         int i, err;
4126 
4127         for (i = 0; i < d->btf->nr_types; i++) {
4128                 err = btf_dedup_struct_type(d, d->btf->start_id + i);
4129                 if (err)
4130                         return err;
4131         }
4132         return 0;
4133 }
4134 
4135 /*
4136  * Deduplicate reference type.
4137  *
4138  * Once all primitive and struct/union types got deduplicated, we can easily
4139  * deduplicate all other (reference) BTF types. This is done in two steps:
4140  *
4141  * 1. Resolve all referenced type IDs into their canonical type IDs. This
4142  * resolution can be done either immediately for primitive or struct/union types
4143  * (because they were deduped in previous two phases) or recursively for
4144  * reference types. Recursion will always terminate at either primitive or
4145  * struct/union type, at which point we can "unwind" chain of reference types
4146  * one by one. There is no danger of encountering cycles because in C type
4147  * system the only way to form type cycle is through struct/union, so any chain
4148  * of reference types, even those taking part in a type cycle, will inevitably
4149  * reach struct/union at some point.
4150  *
4151  * 2. Once all referenced type IDs are resolved into canonical ones, BTF type
4152  * becomes "stable", in the sense that no further deduplication will cause
4153  * any changes to it. With that, it's now possible to calculate type's signature
4154  * hash (this time taking into account referenced type IDs) and loop over all
4155  * potential canonical representatives. If no match was found, current type
4156  * will become canonical representative of itself and will be added into
4157  * btf_dedup->dedup_table as another possible canonical representative.
4158  */
4159 static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id)
4160 {
4161         struct hashmap_entry *hash_entry;
4162         __u32 new_id = type_id, cand_id;
4163         struct btf_type *t, *cand;
4164         /* if we don't find equivalent type, then we are representative type */
4165         int ref_type_id;
4166         long h;
4167 
4168         if (d->map[type_id] == BTF_IN_PROGRESS_ID)
4169                 return -ELOOP;
4170         if (d->map[type_id] <= BTF_MAX_NR_TYPES)
4171                 return resolve_type_id(d, type_id);
4172 
4173         t = btf_type_by_id(d->btf, type_id);
4174         d->map[type_id] = BTF_IN_PROGRESS_ID;
4175 
4176         switch (btf_kind(t)) {
4177         case BTF_KIND_CONST:
4178         case BTF_KIND_VOLATILE:
4179         case BTF_KIND_RESTRICT:
4180         case BTF_KIND_PTR:
4181         case BTF_KIND_TYPEDEF:
4182         case BTF_KIND_FUNC:
4183                 ref_type_id = btf_dedup_ref_type(d, t->type);
4184                 if (ref_type_id < 0)
4185                         return ref_type_id;
4186                 t->type = ref_type_id;
4187 
4188                 h = btf_hash_common(t);
4189                 for_each_dedup_cand(d, hash_entry, h) {
4190                         cand_id = (__u32)(long)hash_entry->value;
4191                         cand = btf_type_by_id(d->btf, cand_id);
4192                         if (btf_equal_common(t, cand)) {
4193                                 new_id = cand_id;
4194                                 break;
4195                         }
4196                 }
4197                 break;
4198 
4199         case BTF_KIND_ARRAY: {
4200                 struct btf_array *info = btf_array(t);
4201 
4202                 ref_type_id = btf_dedup_ref_type(d, info->type);
4203                 if (ref_type_id < 0)
4204                         return ref_type_id;
4205                 info->type = ref_type_id;
4206 
4207                 ref_type_id = btf_dedup_ref_type(d, info->index_type);
4208                 if (ref_type_id < 0)
4209                         return ref_type_id;
4210                 info->index_type = ref_type_id;
4211 
4212                 h = btf_hash_array(t);
4213                 for_each_dedup_cand(d, hash_entry, h) {
4214                         cand_id = (__u32)(long)hash_entry->value;
4215                         cand = btf_type_by_id(d->btf, cand_id);
4216                         if (btf_equal_array(t, cand)) {
4217                                 new_id = cand_id;
4218                                 break;
4219                         }
4220                 }
4221                 break;
4222         }
4223 
4224         case BTF_KIND_FUNC_PROTO: {
4225                 struct btf_param *param;
4226                 __u16 vlen;
4227                 int i;
4228 
4229                 ref_type_id = btf_dedup_ref_type(d, t->type);
4230                 if (ref_type_id < 0)
4231                         return ref_type_id;
4232                 t->type = ref_type_id;
4233 
4234                 vlen = btf_vlen(t);
4235                 param = btf_params(t);
4236                 for (i = 0; i < vlen; i++) {
4237                         ref_type_id = btf_dedup_ref_type(d, param->type);
4238                         if (ref_type_id < 0)
4239                                 return ref_type_id;
4240                         param->type = ref_type_id;
4241                         param++;
4242                 }
4243 
4244                 h = btf_hash_fnproto(t);
4245                 for_each_dedup_cand(d, hash_entry, h) {
4246                         cand_id = (__u32)(long)hash_entry->value;
4247                         cand = btf_type_by_id(d->btf, cand_id);
4248                         if (btf_equal_fnproto(t, cand)) {
4249                                 new_id = cand_id;
4250                                 break;
4251                         }
4252                 }
4253                 break;
4254         }
4255 
4256         default:
4257                 return -EINVAL;
4258         }
4259 
4260         d->map[type_id] = new_id;
4261         if (type_id == new_id && btf_dedup_table_add(d, h, type_id))
4262                 return -ENOMEM;
4263 
4264         return new_id;
4265 }
4266 
4267 static int btf_dedup_ref_types(struct btf_dedup *d)
4268 {
4269         int i, err;
4270 
4271         for (i = 0; i < d->btf->nr_types; i++) {
4272                 err = btf_dedup_ref_type(d, d->btf->start_id + i);
4273                 if (err < 0)
4274                         return err;
4275         }
4276         /* we won't need d->dedup_table anymore */
4277         hashmap__free(d->dedup_table);
4278         d->dedup_table = NULL;
4279         return 0;
4280 }
4281 
4282 /*
4283  * Compact types.
4284  *
4285  * After we established for each type its corresponding canonical representative
4286  * type, we now can eliminate types that are not canonical and leave only
4287  * canonical ones layed out sequentially in memory by copying them over
4288  * duplicates. During compaction btf_dedup->hypot_map array is reused to store
4289  * a map from original type ID to a new compacted type ID, which will be used
4290  * during next phase to "fix up" type IDs, referenced from struct/union and
4291  * reference types.
4292  */
4293 static int btf_dedup_compact_types(struct btf_dedup *d)
4294 {
4295         __u32 *new_offs;
4296         __u32 next_type_id = d->btf->start_id;
4297         const struct btf_type *t;
4298         void *p;
4299         int i, id, len;
4300 
4301         /* we are going to reuse hypot_map to store compaction remapping */
4302         d->hypot_map[0] = 0;
4303         /* base BTF types are not renumbered */
4304         for (id = 1; id < d->btf->start_id; id++)
4305                 d->hypot_map[id] = id;
4306         for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++)
4307                 d->hypot_map[id] = BTF_UNPROCESSED_ID;
4308 
4309         p = d->btf->types_data;
4310 
4311         for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++) {
4312                 if (d->map[id] != id)
4313                         continue;
4314 
4315                 t = btf__type_by_id(d->btf, id);
4316                 len = btf_type_size(t);
4317                 if (len < 0)
4318                         return len;
4319 
4320                 memmove(p, t, len);
4321                 d->hypot_map[id] = next_type_id;
4322                 d->btf->type_offs[next_type_id - d->btf->start_id] = p - d->btf->types_data;
4323                 p += len;
4324                 next_type_id++;
4325         }
4326 
4327         /* shrink struct btf's internal types index and update btf_header */
4328         d->btf->nr_types = next_type_id - d->btf->start_id;
4329         d->btf->type_offs_cap = d->btf->nr_types;
4330         d->btf->hdr->type_len = p - d->btf->types_data;
4331         new_offs = libbpf_reallocarray(d->btf->type_offs, d->btf->type_offs_cap,
4332                                        sizeof(*new_offs));
4333         if (d->btf->type_offs_cap && !new_offs)
4334                 return -ENOMEM;
4335         d->btf->type_offs = new_offs;
4336         d->btf->hdr->str_off = d->btf->hdr->type_len;
4337         d->btf->raw_size = d->btf->hdr->hdr_len + d->btf->hdr->type_len + d->btf->hdr->str_len;
4338         return 0;
4339 }
4340 
4341 /*
4342  * Figure out final (deduplicated and compacted) type ID for provided original
4343  * `type_id` by first resolving it into corresponding canonical type ID and
4344  * then mapping it to a deduplicated type ID, stored in btf_dedup->hypot_map,
4345  * which is populated during compaction phase.
4346  */
4347 static int btf_dedup_remap_type_id(__u32 *type_id, void *ctx)
4348 {
4349         struct btf_dedup *d = ctx;
4350         __u32 resolved_type_id, new_type_id;
4351 
4352         resolved_type_id = resolve_type_id(d, *type_id);
4353         new_type_id = d->hypot_map[resolved_type_id];
4354         if (new_type_id > BTF_MAX_NR_TYPES)
4355                 return -EINVAL;
4356 
4357         *type_id = new_type_id;
4358         return 0;
4359 }
4360 
4361 /*
4362  * Remap referenced type IDs into deduped type IDs.
4363  *
4364  * After BTF types are deduplicated and compacted, their final type IDs may
4365  * differ from original ones. The map from original to a corresponding
4366  * deduped type ID is stored in btf_dedup->hypot_map and is populated during
4367  * compaction phase. During remapping phase we are rewriting all type IDs
4368  * referenced from any BTF type (e.g., struct fields, func proto args, etc) to
4369  * their final deduped type IDs.
4370  */
4371 static int btf_dedup_remap_types(struct btf_dedup *d)
4372 {
4373         int i, r;
4374 
4375         for (i = 0; i < d->btf->nr_types; i++) {
4376                 struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i);
4377 
4378                 r = btf_type_visit_type_ids(t, btf_dedup_remap_type_id, d);
4379                 if (r)
4380                         return r;
4381         }
4382 
4383         if (!d->btf_ext)
4384                 return 0;
4385 
4386         r = btf_ext_visit_type_ids(d->btf_ext, btf_dedup_remap_type_id, d);
4387         if (r)
4388                 return r;
4389 
4390         return 0;
4391 }
4392 
4393 /*
4394  * Probe few well-known locations for vmlinux kernel image and try to load BTF
4395  * data out of it to use for target BTF.
4396  */
4397 struct btf *libbpf_find_kernel_btf(void)
4398 {
4399         struct {
4400                 const char *path_fmt;
4401                 bool raw_btf;
4402         } locations[] = {
4403                 /* try canonical vmlinux BTF through sysfs first */
4404                 { "/sys/kernel/btf/vmlinux", true /* raw BTF */ },
4405                 /* fall back to trying to find vmlinux ELF on disk otherwise */
4406                 { "/boot/vmlinux-%1$s" },
4407                 { "/lib/modules/%1$s/vmlinux-%1$s" },
4408                 { "/lib/modules/%1$s/build/vmlinux" },
4409                 { "/usr/lib/modules/%1$s/kernel/vmlinux" },
4410                 { "/usr/lib/debug/boot/vmlinux-%1$s" },
4411                 { "/usr/lib/debug/boot/vmlinux-%1$s.debug" },
4412                 { "/usr/lib/debug/lib/modules/%1$s/vmlinux" },
4413         };
4414         char path[PATH_MAX + 1];
4415         struct utsname buf;
4416         struct btf *btf;
4417         int i, err;
4418 
4419         uname(&buf);
4420 
4421         for (i = 0; i < ARRAY_SIZE(locations); i++) {
4422                 snprintf(path, PATH_MAX, locations[i].path_fmt, buf.release);
4423 
4424                 if (access(path, R_OK))
4425                         continue;
4426 
4427                 if (locations[i].raw_btf)
4428                         btf = btf__parse_raw(path);
4429                 else
4430                         btf = btf__parse_elf(path, NULL);
4431                 err = libbpf_get_error(btf);
4432                 pr_debug("loading kernel BTF '%s': %d\n", path, err);
4433                 if (err)
4434                         continue;
4435 
4436                 return btf;
4437         }
4438 
4439         pr_warn("failed to find valid kernel BTF\n");
4440         return libbpf_err_ptr(-ESRCH);
4441 }
4442 
4443 int btf_type_visit_type_ids(struct btf_type *t, type_id_visit_fn visit, void *ctx)
4444 {
4445         int i, n, err;
4446 
4447         switch (btf_kind(t)) {
4448         case BTF_KIND_INT:
4449         case BTF_KIND_FLOAT:
4450         case BTF_KIND_ENUM:
4451                 return 0;
4452 
4453         case BTF_KIND_FWD:
4454         case BTF_KIND_CONST:
4455         case BTF_KIND_VOLATILE:
4456         case BTF_KIND_RESTRICT:
4457         case BTF_KIND_PTR:
4458         case BTF_KIND_TYPEDEF:
4459         case BTF_KIND_FUNC:
4460         case BTF_KIND_VAR:
4461                 return visit(&t->type, ctx);
4462 
4463         case BTF_KIND_ARRAY: {
4464                 struct btf_array *a = btf_array(t);
4465 
4466                 err = visit(&a->type, ctx);
4467                 err = err ?: visit(&a->index_type, ctx);
4468                 return err;
4469         }
4470 
4471         case BTF_KIND_STRUCT:
4472         case BTF_KIND_UNION: {
4473                 struct btf_member *m = btf_members(t);
4474 
4475                 for (i = 0, n = btf_vlen(t); i < n; i++, m++) {
4476                         err = visit(&m->type, ctx);
4477                         if (err)
4478                                 return err;
4479                 }
4480                 return 0;
4481         }
4482 
4483         case BTF_KIND_FUNC_PROTO: {
4484                 struct btf_param *m = btf_params(t);
4485 
4486                 err = visit(&t->type, ctx);
4487                 if (err)
4488                         return err;
4489                 for (i = 0, n = btf_vlen(t); i < n; i++, m++) {
4490                         err = visit(&m->type, ctx);
4491                         if (err)
4492                                 return err;
4493                 }
4494                 return 0;
4495         }
4496 
4497         case BTF_KIND_DATASEC: {
4498                 struct btf_var_secinfo *m = btf_var_secinfos(t);
4499 
4500                 for (i = 0, n = btf_vlen(t); i < n; i++, m++) {
4501                         err = visit(&m->type, ctx);
4502                         if (err)
4503                                 return err;
4504                 }
4505                 return 0;
4506         }
4507 
4508         default:
4509                 return -EINVAL;
4510         }
4511 }
4512 
4513 int btf_type_visit_str_offs(struct btf_type *t, str_off_visit_fn visit, void *ctx)
4514 {
4515         int i, n, err;
4516 
4517         err = visit(&t->name_off, ctx);
4518         if (err)
4519                 return err;
4520 
4521         switch (btf_kind(t)) {
4522         case BTF_KIND_STRUCT:
4523         case BTF_KIND_UNION: {
4524                 struct btf_member *m = btf_members(t);
4525 
4526                 for (i = 0, n = btf_vlen(t); i < n; i++, m++) {
4527                         err = visit(&m->name_off, ctx);
4528                         if (err)
4529                                 return err;
4530                 }
4531                 break;
4532         }
4533         case BTF_KIND_ENUM: {
4534                 struct btf_enum *m = btf_enum(t);
4535 
4536                 for (i = 0, n = btf_vlen(t); i < n; i++, m++) {
4537                         err = visit(&m->name_off, ctx);
4538                         if (err)
4539                                 return err;
4540                 }
4541                 break;
4542         }
4543         case BTF_KIND_FUNC_PROTO: {
4544                 struct btf_param *m = btf_params(t);
4545 
4546                 for (i = 0, n = btf_vlen(t); i < n; i++, m++) {
4547                         err = visit(&m->name_off, ctx);
4548                         if (err)
4549                                 return err;
4550                 }
4551                 break;
4552         }
4553         default:
4554                 break;
4555         }
4556 
4557         return 0;
4558 }
4559 
4560 int btf_ext_visit_type_ids(struct btf_ext *btf_ext, type_id_visit_fn visit, void *ctx)
4561 {
4562         const struct btf_ext_info *seg;
4563         struct btf_ext_info_sec *sec;
4564         int i, err;
4565 
4566         seg = &btf_ext->func_info;
4567         for_each_btf_ext_sec(seg, sec) {
4568                 struct bpf_func_info_min *rec;
4569 
4570                 for_each_btf_ext_rec(seg, sec, i, rec) {
4571                         err = visit(&rec->type_id, ctx);
4572                         if (err < 0)
4573                                 return err;
4574                 }
4575         }
4576 
4577         seg = &btf_ext->core_relo_info;
4578         for_each_btf_ext_sec(seg, sec) {
4579                 struct bpf_core_relo *rec;
4580 
4581                 for_each_btf_ext_rec(seg, sec, i, rec) {
4582                         err = visit(&rec->type_id, ctx);
4583                         if (err < 0)
4584                                 return err;
4585                 }
4586         }
4587 
4588         return 0;
4589 }
4590 
4591 int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void *ctx)
4592 {
4593         const struct btf_ext_info *seg;
4594         struct btf_ext_info_sec *sec;
4595         int i, err;
4596 
4597         seg = &btf_ext->func_info;
4598         for_each_btf_ext_sec(seg, sec) {
4599                 err = visit(&sec->sec_name_off, ctx);
4600                 if (err)
4601                         return err;
4602         }
4603 
4604         seg = &btf_ext->line_info;
4605         for_each_btf_ext_sec(seg, sec) {
4606                 struct bpf_line_info_min *rec;
4607 
4608                 err = visit(&sec->sec_name_off, ctx);
4609                 if (err)
4610                         return err;
4611 
4612                 for_each_btf_ext_rec(seg, sec, i, rec) {
4613                         err = visit(&rec->file_name_off, ctx);
4614                         if (err)
4615                                 return err;
4616                         err = visit(&rec->line_off, ctx);
4617                         if (err)
4618                                 return err;
4619                 }
4620         }
4621 
4622         seg = &btf_ext->core_relo_info;
4623         for_each_btf_ext_sec(seg, sec) {
4624                 struct bpf_core_relo *rec;
4625 
4626                 err = visit(&sec->sec_name_off, ctx);
4627                 if (err)
4628                         return err;
4629 
4630                 for_each_btf_ext_rec(seg, sec, i, rec) {
4631                         err = visit(&rec->access_str_off, ctx);
4632                         if (err)
4633                                 return err;
4634                 }
4635         }
4636 
4637         return 0;
4638 }
4639 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp