~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/bpf/syscall.c

Version: ~ [ linux-5.16-rc3 ] ~ [ linux-5.15.5 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.82 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.162 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.218 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.256 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.291 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.293 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
  2  *
  3  * This program is free software; you can redistribute it and/or
  4  * modify it under the terms of version 2 of the GNU General Public
  5  * License as published by the Free Software Foundation.
  6  *
  7  * This program is distributed in the hope that it will be useful, but
  8  * WITHOUT ANY WARRANTY; without even the implied warranty of
  9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 10  * General Public License for more details.
 11  */
 12 #include <linux/bpf.h>
 13 #include <linux/bpf_trace.h>
 14 #include <linux/syscalls.h>
 15 #include <linux/slab.h>
 16 #include <linux/sched/signal.h>
 17 #include <linux/vmalloc.h>
 18 #include <linux/mmzone.h>
 19 #include <linux/anon_inodes.h>
 20 #include <linux/file.h>
 21 #include <linux/license.h>
 22 #include <linux/filter.h>
 23 #include <linux/version.h>
 24 #include <linux/kernel.h>
 25 #include <linux/idr.h>
 26 
 27 #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \
 28                            (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
 29                            (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
 30                            (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
 31 #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
 32 #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map))
 33 
 34 DEFINE_PER_CPU(int, bpf_prog_active);
 35 static DEFINE_IDR(prog_idr);
 36 static DEFINE_SPINLOCK(prog_idr_lock);
 37 static DEFINE_IDR(map_idr);
 38 static DEFINE_SPINLOCK(map_idr_lock);
 39 
 40 int sysctl_unprivileged_bpf_disabled __read_mostly;
 41 
 42 static const struct bpf_map_ops * const bpf_map_types[] = {
 43 #define BPF_PROG_TYPE(_id, _ops)
 44 #define BPF_MAP_TYPE(_id, _ops) \
 45         [_id] = &_ops,
 46 #include <linux/bpf_types.h>
 47 #undef BPF_PROG_TYPE
 48 #undef BPF_MAP_TYPE
 49 };
 50 
 51 static struct bpf_map *find_and_alloc_map(union bpf_attr *attr)
 52 {
 53         struct bpf_map *map;
 54 
 55         if (attr->map_type >= ARRAY_SIZE(bpf_map_types) ||
 56             !bpf_map_types[attr->map_type])
 57                 return ERR_PTR(-EINVAL);
 58 
 59         map = bpf_map_types[attr->map_type]->map_alloc(attr);
 60         if (IS_ERR(map))
 61                 return map;
 62         map->ops = bpf_map_types[attr->map_type];
 63         map->map_type = attr->map_type;
 64         return map;
 65 }
 66 
 67 void *bpf_map_area_alloc(size_t size)
 68 {
 69         /* We definitely need __GFP_NORETRY, so OOM killer doesn't
 70          * trigger under memory pressure as we really just want to
 71          * fail instead.
 72          */
 73         const gfp_t flags = __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO;
 74         void *area;
 75 
 76         if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
 77                 area = kmalloc(size, GFP_USER | flags);
 78                 if (area != NULL)
 79                         return area;
 80         }
 81 
 82         return __vmalloc(size, GFP_KERNEL | flags, PAGE_KERNEL);
 83 }
 84 
 85 void bpf_map_area_free(void *area)
 86 {
 87         kvfree(area);
 88 }
 89 
 90 int bpf_map_precharge_memlock(u32 pages)
 91 {
 92         struct user_struct *user = get_current_user();
 93         unsigned long memlock_limit, cur;
 94 
 95         memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
 96         cur = atomic_long_read(&user->locked_vm);
 97         free_uid(user);
 98         if (cur + pages > memlock_limit)
 99                 return -EPERM;
100         return 0;
101 }
102 
103 static int bpf_map_charge_memlock(struct bpf_map *map)
104 {
105         struct user_struct *user = get_current_user();
106         unsigned long memlock_limit;
107 
108         memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
109 
110         atomic_long_add(map->pages, &user->locked_vm);
111 
112         if (atomic_long_read(&user->locked_vm) > memlock_limit) {
113                 atomic_long_sub(map->pages, &user->locked_vm);
114                 free_uid(user);
115                 return -EPERM;
116         }
117         map->user = user;
118         return 0;
119 }
120 
121 static void bpf_map_uncharge_memlock(struct bpf_map *map)
122 {
123         struct user_struct *user = map->user;
124 
125         atomic_long_sub(map->pages, &user->locked_vm);
126         free_uid(user);
127 }
128 
129 static int bpf_map_alloc_id(struct bpf_map *map)
130 {
131         int id;
132 
133         spin_lock_bh(&map_idr_lock);
134         id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC);
135         if (id > 0)
136                 map->id = id;
137         spin_unlock_bh(&map_idr_lock);
138 
139         if (WARN_ON_ONCE(!id))
140                 return -ENOSPC;
141 
142         return id > 0 ? 0 : id;
143 }
144 
145 static void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock)
146 {
147         unsigned long flags;
148 
149         if (do_idr_lock)
150                 spin_lock_irqsave(&map_idr_lock, flags);
151         else
152                 __acquire(&map_idr_lock);
153 
154         idr_remove(&map_idr, map->id);
155 
156         if (do_idr_lock)
157                 spin_unlock_irqrestore(&map_idr_lock, flags);
158         else
159                 __release(&map_idr_lock);
160 }
161 
162 /* called from workqueue */
163 static void bpf_map_free_deferred(struct work_struct *work)
164 {
165         struct bpf_map *map = container_of(work, struct bpf_map, work);
166 
167         bpf_map_uncharge_memlock(map);
168         /* implementation dependent freeing */
169         map->ops->map_free(map);
170 }
171 
172 static void bpf_map_put_uref(struct bpf_map *map)
173 {
174         if (atomic_dec_and_test(&map->usercnt)) {
175                 if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY)
176                         bpf_fd_array_map_clear(map);
177         }
178 }
179 
180 /* decrement map refcnt and schedule it for freeing via workqueue
181  * (unrelying map implementation ops->map_free() might sleep)
182  */
183 static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock)
184 {
185         if (atomic_dec_and_test(&map->refcnt)) {
186                 /* bpf_map_free_id() must be called first */
187                 bpf_map_free_id(map, do_idr_lock);
188                 INIT_WORK(&map->work, bpf_map_free_deferred);
189                 schedule_work(&map->work);
190         }
191 }
192 
193 void bpf_map_put(struct bpf_map *map)
194 {
195         __bpf_map_put(map, true);
196 }
197 
198 void bpf_map_put_with_uref(struct bpf_map *map)
199 {
200         bpf_map_put_uref(map);
201         bpf_map_put(map);
202 }
203 
204 static int bpf_map_release(struct inode *inode, struct file *filp)
205 {
206         struct bpf_map *map = filp->private_data;
207 
208         if (map->ops->map_release)
209                 map->ops->map_release(map, filp);
210 
211         bpf_map_put_with_uref(map);
212         return 0;
213 }
214 
215 #ifdef CONFIG_PROC_FS
216 static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)
217 {
218         const struct bpf_map *map = filp->private_data;
219         const struct bpf_array *array;
220         u32 owner_prog_type = 0;
221         u32 owner_jited = 0;
222 
223         if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) {
224                 array = container_of(map, struct bpf_array, map);
225                 owner_prog_type = array->owner_prog_type;
226                 owner_jited = array->owner_jited;
227         }
228 
229         seq_printf(m,
230                    "map_type:\t%u\n"
231                    "key_size:\t%u\n"
232                    "value_size:\t%u\n"
233                    "max_entries:\t%u\n"
234                    "map_flags:\t%#x\n"
235                    "memlock:\t%llu\n",
236                    map->map_type,
237                    map->key_size,
238                    map->value_size,
239                    map->max_entries,
240                    map->map_flags,
241                    map->pages * 1ULL << PAGE_SHIFT);
242 
243         if (owner_prog_type) {
244                 seq_printf(m, "owner_prog_type:\t%u\n",
245                            owner_prog_type);
246                 seq_printf(m, "owner_jited:\t%u\n",
247                            owner_jited);
248         }
249 }
250 #endif
251 
252 static const struct file_operations bpf_map_fops = {
253 #ifdef CONFIG_PROC_FS
254         .show_fdinfo    = bpf_map_show_fdinfo,
255 #endif
256         .release        = bpf_map_release,
257 };
258 
259 int bpf_map_new_fd(struct bpf_map *map)
260 {
261         return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
262                                 O_RDWR | O_CLOEXEC);
263 }
264 
265 /* helper macro to check that unused fields 'union bpf_attr' are zero */
266 #define CHECK_ATTR(CMD) \
267         memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
268                    sizeof(attr->CMD##_LAST_FIELD), 0, \
269                    sizeof(*attr) - \
270                    offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
271                    sizeof(attr->CMD##_LAST_FIELD)) != NULL
272 
273 #define BPF_MAP_CREATE_LAST_FIELD inner_map_fd
274 /* called via syscall */
275 static int map_create(union bpf_attr *attr)
276 {
277         struct bpf_map *map;
278         int err;
279 
280         err = CHECK_ATTR(BPF_MAP_CREATE);
281         if (err)
282                 return -EINVAL;
283 
284         /* find map type and init map: hashtable vs rbtree vs bloom vs ... */
285         map = find_and_alloc_map(attr);
286         if (IS_ERR(map))
287                 return PTR_ERR(map);
288 
289         atomic_set(&map->refcnt, 1);
290         atomic_set(&map->usercnt, 1);
291 
292         err = bpf_map_charge_memlock(map);
293         if (err)
294                 goto free_map_nouncharge;
295 
296         err = bpf_map_alloc_id(map);
297         if (err)
298                 goto free_map;
299 
300         err = bpf_map_new_fd(map);
301         if (err < 0) {
302                 /* failed to allocate fd.
303                  * bpf_map_put() is needed because the above
304                  * bpf_map_alloc_id() has published the map
305                  * to the userspace and the userspace may
306                  * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID.
307                  */
308                 bpf_map_put(map);
309                 return err;
310         }
311 
312         trace_bpf_map_create(map, err);
313         return err;
314 
315 free_map:
316         bpf_map_uncharge_memlock(map);
317 free_map_nouncharge:
318         map->ops->map_free(map);
319         return err;
320 }
321 
322 /* if error is returned, fd is released.
323  * On success caller should complete fd access with matching fdput()
324  */
325 struct bpf_map *__bpf_map_get(struct fd f)
326 {
327         if (!f.file)
328                 return ERR_PTR(-EBADF);
329         if (f.file->f_op != &bpf_map_fops) {
330                 fdput(f);
331                 return ERR_PTR(-EINVAL);
332         }
333 
334         return f.file->private_data;
335 }
336 
337 /* prog's and map's refcnt limit */
338 #define BPF_MAX_REFCNT 32768
339 
340 struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref)
341 {
342         if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) {
343                 atomic_dec(&map->refcnt);
344                 return ERR_PTR(-EBUSY);
345         }
346         if (uref)
347                 atomic_inc(&map->usercnt);
348         return map;
349 }
350 
351 struct bpf_map *bpf_map_get_with_uref(u32 ufd)
352 {
353         struct fd f = fdget(ufd);
354         struct bpf_map *map;
355 
356         map = __bpf_map_get(f);
357         if (IS_ERR(map))
358                 return map;
359 
360         map = bpf_map_inc(map, true);
361         fdput(f);
362 
363         return map;
364 }
365 
366 /* map_idr_lock should have been held */
367 static struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map,
368                                             bool uref)
369 {
370         int refold;
371 
372         refold = __atomic_add_unless(&map->refcnt, 1, 0);
373 
374         if (refold >= BPF_MAX_REFCNT) {
375                 __bpf_map_put(map, false);
376                 return ERR_PTR(-EBUSY);
377         }
378 
379         if (!refold)
380                 return ERR_PTR(-ENOENT);
381 
382         if (uref)
383                 atomic_inc(&map->usercnt);
384 
385         return map;
386 }
387 
388 int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
389 {
390         return -ENOTSUPP;
391 }
392 
393 /* last field in 'union bpf_attr' used by this command */
394 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value
395 
396 static int map_lookup_elem(union bpf_attr *attr)
397 {
398         void __user *ukey = u64_to_user_ptr(attr->key);
399         void __user *uvalue = u64_to_user_ptr(attr->value);
400         int ufd = attr->map_fd;
401         struct bpf_map *map;
402         void *key, *value, *ptr;
403         u32 value_size;
404         struct fd f;
405         int err;
406 
407         if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
408                 return -EINVAL;
409 
410         f = fdget(ufd);
411         map = __bpf_map_get(f);
412         if (IS_ERR(map))
413                 return PTR_ERR(map);
414 
415         key = memdup_user(ukey, map->key_size);
416         if (IS_ERR(key)) {
417                 err = PTR_ERR(key);
418                 goto err_put;
419         }
420 
421         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
422             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
423             map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
424                 value_size = round_up(map->value_size, 8) * num_possible_cpus();
425         else if (IS_FD_MAP(map))
426                 value_size = sizeof(u32);
427         else
428                 value_size = map->value_size;
429 
430         err = -ENOMEM;
431         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
432         if (!value)
433                 goto free_key;
434 
435         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
436             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
437                 err = bpf_percpu_hash_copy(map, key, value);
438         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
439                 err = bpf_percpu_array_copy(map, key, value);
440         } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
441                 err = bpf_stackmap_copy(map, key, value);
442         } else if (IS_FD_ARRAY(map)) {
443                 err = bpf_fd_array_map_lookup_elem(map, key, value);
444         } else if (IS_FD_HASH(map)) {
445                 err = bpf_fd_htab_map_lookup_elem(map, key, value);
446         } else {
447                 rcu_read_lock();
448                 ptr = map->ops->map_lookup_elem(map, key);
449                 if (ptr)
450                         memcpy(value, ptr, value_size);
451                 rcu_read_unlock();
452                 err = ptr ? 0 : -ENOENT;
453         }
454 
455         if (err)
456                 goto free_value;
457 
458         err = -EFAULT;
459         if (copy_to_user(uvalue, value, value_size) != 0)
460                 goto free_value;
461 
462         trace_bpf_map_lookup_elem(map, ufd, key, value);
463         err = 0;
464 
465 free_value:
466         kfree(value);
467 free_key:
468         kfree(key);
469 err_put:
470         fdput(f);
471         return err;
472 }
473 
474 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
475 
476 static int map_update_elem(union bpf_attr *attr)
477 {
478         void __user *ukey = u64_to_user_ptr(attr->key);
479         void __user *uvalue = u64_to_user_ptr(attr->value);
480         int ufd = attr->map_fd;
481         struct bpf_map *map;
482         void *key, *value;
483         u32 value_size;
484         struct fd f;
485         int err;
486 
487         if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
488                 return -EINVAL;
489 
490         f = fdget(ufd);
491         map = __bpf_map_get(f);
492         if (IS_ERR(map))
493                 return PTR_ERR(map);
494 
495         key = memdup_user(ukey, map->key_size);
496         if (IS_ERR(key)) {
497                 err = PTR_ERR(key);
498                 goto err_put;
499         }
500 
501         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
502             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
503             map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
504                 value_size = round_up(map->value_size, 8) * num_possible_cpus();
505         else
506                 value_size = map->value_size;
507 
508         err = -ENOMEM;
509         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
510         if (!value)
511                 goto free_key;
512 
513         err = -EFAULT;
514         if (copy_from_user(value, uvalue, value_size) != 0)
515                 goto free_value;
516 
517         /* must increment bpf_prog_active to avoid kprobe+bpf triggering from
518          * inside bpf map update or delete otherwise deadlocks are possible
519          */
520         preempt_disable();
521         __this_cpu_inc(bpf_prog_active);
522         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
523             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
524                 err = bpf_percpu_hash_update(map, key, value, attr->flags);
525         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
526                 err = bpf_percpu_array_update(map, key, value, attr->flags);
527         } else if (map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY ||
528                    map->map_type == BPF_MAP_TYPE_PROG_ARRAY ||
529                    map->map_type == BPF_MAP_TYPE_CGROUP_ARRAY ||
530                    map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) {
531                 rcu_read_lock();
532                 err = bpf_fd_array_map_update_elem(map, f.file, key, value,
533                                                    attr->flags);
534                 rcu_read_unlock();
535         } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
536                 rcu_read_lock();
537                 err = bpf_fd_htab_map_update_elem(map, f.file, key, value,
538                                                   attr->flags);
539                 rcu_read_unlock();
540         } else {
541                 rcu_read_lock();
542                 err = map->ops->map_update_elem(map, key, value, attr->flags);
543                 rcu_read_unlock();
544         }
545         __this_cpu_dec(bpf_prog_active);
546         preempt_enable();
547 
548         if (!err)
549                 trace_bpf_map_update_elem(map, ufd, key, value);
550 free_value:
551         kfree(value);
552 free_key:
553         kfree(key);
554 err_put:
555         fdput(f);
556         return err;
557 }
558 
559 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key
560 
561 static int map_delete_elem(union bpf_attr *attr)
562 {
563         void __user *ukey = u64_to_user_ptr(attr->key);
564         int ufd = attr->map_fd;
565         struct bpf_map *map;
566         struct fd f;
567         void *key;
568         int err;
569 
570         if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
571                 return -EINVAL;
572 
573         f = fdget(ufd);
574         map = __bpf_map_get(f);
575         if (IS_ERR(map))
576                 return PTR_ERR(map);
577 
578         key = memdup_user(ukey, map->key_size);
579         if (IS_ERR(key)) {
580                 err = PTR_ERR(key);
581                 goto err_put;
582         }
583 
584         preempt_disable();
585         __this_cpu_inc(bpf_prog_active);
586         rcu_read_lock();
587         err = map->ops->map_delete_elem(map, key);
588         rcu_read_unlock();
589         __this_cpu_dec(bpf_prog_active);
590         preempt_enable();
591 
592         if (!err)
593                 trace_bpf_map_delete_elem(map, ufd, key);
594         kfree(key);
595 err_put:
596         fdput(f);
597         return err;
598 }
599 
600 /* last field in 'union bpf_attr' used by this command */
601 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
602 
603 static int map_get_next_key(union bpf_attr *attr)
604 {
605         void __user *ukey = u64_to_user_ptr(attr->key);
606         void __user *unext_key = u64_to_user_ptr(attr->next_key);
607         int ufd = attr->map_fd;
608         struct bpf_map *map;
609         void *key, *next_key;
610         struct fd f;
611         int err;
612 
613         if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
614                 return -EINVAL;
615 
616         f = fdget(ufd);
617         map = __bpf_map_get(f);
618         if (IS_ERR(map))
619                 return PTR_ERR(map);
620 
621         if (ukey) {
622                 key = memdup_user(ukey, map->key_size);
623                 if (IS_ERR(key)) {
624                         err = PTR_ERR(key);
625                         goto err_put;
626                 }
627         } else {
628                 key = NULL;
629         }
630 
631         err = -ENOMEM;
632         next_key = kmalloc(map->key_size, GFP_USER);
633         if (!next_key)
634                 goto free_key;
635 
636         rcu_read_lock();
637         err = map->ops->map_get_next_key(map, key, next_key);
638         rcu_read_unlock();
639         if (err)
640                 goto free_next_key;
641 
642         err = -EFAULT;
643         if (copy_to_user(unext_key, next_key, map->key_size) != 0)
644                 goto free_next_key;
645 
646         trace_bpf_map_next_key(map, ufd, key, next_key);
647         err = 0;
648 
649 free_next_key:
650         kfree(next_key);
651 free_key:
652         kfree(key);
653 err_put:
654         fdput(f);
655         return err;
656 }
657 
658 static const struct bpf_verifier_ops * const bpf_prog_types[] = {
659 #define BPF_PROG_TYPE(_id, _ops) \
660         [_id] = &_ops,
661 #define BPF_MAP_TYPE(_id, _ops)
662 #include <linux/bpf_types.h>
663 #undef BPF_PROG_TYPE
664 #undef BPF_MAP_TYPE
665 };
666 
667 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
668 {
669         if (type >= ARRAY_SIZE(bpf_prog_types) || !bpf_prog_types[type])
670                 return -EINVAL;
671 
672         prog->aux->ops = bpf_prog_types[type];
673         prog->type = type;
674         return 0;
675 }
676 
677 /* drop refcnt on maps used by eBPF program and free auxilary data */
678 static void free_used_maps(struct bpf_prog_aux *aux)
679 {
680         int i;
681 
682         for (i = 0; i < aux->used_map_cnt; i++)
683                 bpf_map_put(aux->used_maps[i]);
684 
685         kfree(aux->used_maps);
686 }
687 
688 int __bpf_prog_charge(struct user_struct *user, u32 pages)
689 {
690         unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
691         unsigned long user_bufs;
692 
693         if (user) {
694                 user_bufs = atomic_long_add_return(pages, &user->locked_vm);
695                 if (user_bufs > memlock_limit) {
696                         atomic_long_sub(pages, &user->locked_vm);
697                         return -EPERM;
698                 }
699         }
700 
701         return 0;
702 }
703 
704 void __bpf_prog_uncharge(struct user_struct *user, u32 pages)
705 {
706         if (user)
707                 atomic_long_sub(pages, &user->locked_vm);
708 }
709 
710 static int bpf_prog_charge_memlock(struct bpf_prog *prog)
711 {
712         struct user_struct *user = get_current_user();
713         int ret;
714 
715         ret = __bpf_prog_charge(user, prog->pages);
716         if (ret) {
717                 free_uid(user);
718                 return ret;
719         }
720 
721         prog->aux->user = user;
722         return 0;
723 }
724 
725 static void bpf_prog_uncharge_memlock(struct bpf_prog *prog)
726 {
727         struct user_struct *user = prog->aux->user;
728 
729         __bpf_prog_uncharge(user, prog->pages);
730         free_uid(user);
731 }
732 
733 static int bpf_prog_alloc_id(struct bpf_prog *prog)
734 {
735         int id;
736 
737         spin_lock_bh(&prog_idr_lock);
738         id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC);
739         if (id > 0)
740                 prog->aux->id = id;
741         spin_unlock_bh(&prog_idr_lock);
742 
743         /* id is in [1, INT_MAX) */
744         if (WARN_ON_ONCE(!id))
745                 return -ENOSPC;
746 
747         return id > 0 ? 0 : id;
748 }
749 
750 static void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock)
751 {
752         /* cBPF to eBPF migrations are currently not in the idr store. */
753         if (!prog->aux->id)
754                 return;
755 
756         if (do_idr_lock)
757                 spin_lock_bh(&prog_idr_lock);
758         else
759                 __acquire(&prog_idr_lock);
760 
761         idr_remove(&prog_idr, prog->aux->id);
762 
763         if (do_idr_lock)
764                 spin_unlock_bh(&prog_idr_lock);
765         else
766                 __release(&prog_idr_lock);
767 }
768 
769 static void __bpf_prog_put_rcu(struct rcu_head *rcu)
770 {
771         struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
772 
773         free_used_maps(aux);
774         bpf_prog_uncharge_memlock(aux->prog);
775         bpf_prog_free(aux->prog);
776 }
777 
778 static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)
779 {
780         if (atomic_dec_and_test(&prog->aux->refcnt)) {
781                 trace_bpf_prog_put_rcu(prog);
782                 /* bpf_prog_free_id() must be called first */
783                 bpf_prog_free_id(prog, do_idr_lock);
784                 bpf_prog_kallsyms_del(prog);
785                 call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
786         }
787 }
788 
789 void bpf_prog_put(struct bpf_prog *prog)
790 {
791         __bpf_prog_put(prog, true);
792 }
793 EXPORT_SYMBOL_GPL(bpf_prog_put);
794 
795 static int bpf_prog_release(struct inode *inode, struct file *filp)
796 {
797         struct bpf_prog *prog = filp->private_data;
798 
799         bpf_prog_put(prog);
800         return 0;
801 }
802 
803 #ifdef CONFIG_PROC_FS
804 static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
805 {
806         const struct bpf_prog *prog = filp->private_data;
807         char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
808 
809         bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
810         seq_printf(m,
811                    "prog_type:\t%u\n"
812                    "prog_jited:\t%u\n"
813                    "prog_tag:\t%s\n"
814                    "memlock:\t%llu\n",
815                    prog->type,
816                    prog->jited,
817                    prog_tag,
818                    prog->pages * 1ULL << PAGE_SHIFT);
819 }
820 #endif
821 
822 static const struct file_operations bpf_prog_fops = {
823 #ifdef CONFIG_PROC_FS
824         .show_fdinfo    = bpf_prog_show_fdinfo,
825 #endif
826         .release        = bpf_prog_release,
827 };
828 
829 int bpf_prog_new_fd(struct bpf_prog *prog)
830 {
831         return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
832                                 O_RDWR | O_CLOEXEC);
833 }
834 
835 static struct bpf_prog *____bpf_prog_get(struct fd f)
836 {
837         if (!f.file)
838                 return ERR_PTR(-EBADF);
839         if (f.file->f_op != &bpf_prog_fops) {
840                 fdput(f);
841                 return ERR_PTR(-EINVAL);
842         }
843 
844         return f.file->private_data;
845 }
846 
847 struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i)
848 {
849         if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) {
850                 atomic_sub(i, &prog->aux->refcnt);
851                 return ERR_PTR(-EBUSY);
852         }
853         return prog;
854 }
855 EXPORT_SYMBOL_GPL(bpf_prog_add);
856 
857 void bpf_prog_sub(struct bpf_prog *prog, int i)
858 {
859         /* Only to be used for undoing previous bpf_prog_add() in some
860          * error path. We still know that another entity in our call
861          * path holds a reference to the program, thus atomic_sub() can
862          * be safely used in such cases!
863          */
864         WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0);
865 }
866 EXPORT_SYMBOL_GPL(bpf_prog_sub);
867 
868 struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog)
869 {
870         return bpf_prog_add(prog, 1);
871 }
872 EXPORT_SYMBOL_GPL(bpf_prog_inc);
873 
874 /* prog_idr_lock should have been held */
875 static struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog)
876 {
877         int refold;
878 
879         refold = __atomic_add_unless(&prog->aux->refcnt, 1, 0);
880 
881         if (refold >= BPF_MAX_REFCNT) {
882                 __bpf_prog_put(prog, false);
883                 return ERR_PTR(-EBUSY);
884         }
885 
886         if (!refold)
887                 return ERR_PTR(-ENOENT);
888 
889         return prog;
890 }
891 
892 static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *type)
893 {
894         struct fd f = fdget(ufd);
895         struct bpf_prog *prog;
896 
897         prog = ____bpf_prog_get(f);
898         if (IS_ERR(prog))
899                 return prog;
900         if (type && prog->type != *type) {
901                 prog = ERR_PTR(-EINVAL);
902                 goto out;
903         }
904 
905         prog = bpf_prog_inc(prog);
906 out:
907         fdput(f);
908         return prog;
909 }
910 
911 struct bpf_prog *bpf_prog_get(u32 ufd)
912 {
913         return __bpf_prog_get(ufd, NULL);
914 }
915 
916 struct bpf_prog *bpf_prog_get_type(u32 ufd, enum bpf_prog_type type)
917 {
918         struct bpf_prog *prog = __bpf_prog_get(ufd, &type);
919 
920         if (!IS_ERR(prog))
921                 trace_bpf_prog_get_type(prog);
922         return prog;
923 }
924 EXPORT_SYMBOL_GPL(bpf_prog_get_type);
925 
926 /* last field in 'union bpf_attr' used by this command */
927 #define BPF_PROG_LOAD_LAST_FIELD prog_flags
928 
929 static int bpf_prog_load(union bpf_attr *attr)
930 {
931         enum bpf_prog_type type = attr->prog_type;
932         struct bpf_prog *prog;
933         int err;
934         char license[128];
935         bool is_gpl;
936 
937         if (CHECK_ATTR(BPF_PROG_LOAD))
938                 return -EINVAL;
939 
940         if (attr->prog_flags & ~BPF_F_STRICT_ALIGNMENT)
941                 return -EINVAL;
942 
943         /* copy eBPF program license from user space */
944         if (strncpy_from_user(license, u64_to_user_ptr(attr->license),
945                               sizeof(license) - 1) < 0)
946                 return -EFAULT;
947         license[sizeof(license) - 1] = 0;
948 
949         /* eBPF programs must be GPL compatible to use GPL-ed functions */
950         is_gpl = license_is_gpl_compatible(license);
951 
952         if (attr->insn_cnt == 0 || attr->insn_cnt > BPF_MAXINSNS)
953                 return -E2BIG;
954 
955         if (type == BPF_PROG_TYPE_KPROBE &&
956             attr->kern_version != LINUX_VERSION_CODE)
957                 return -EINVAL;
958 
959         if (type != BPF_PROG_TYPE_SOCKET_FILTER &&
960             type != BPF_PROG_TYPE_CGROUP_SKB &&
961             !capable(CAP_SYS_ADMIN))
962                 return -EPERM;
963 
964         /* plain bpf_prog allocation */
965         prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
966         if (!prog)
967                 return -ENOMEM;
968 
969         err = bpf_prog_charge_memlock(prog);
970         if (err)
971                 goto free_prog_nouncharge;
972 
973         prog->len = attr->insn_cnt;
974 
975         err = -EFAULT;
976         if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns),
977                            bpf_prog_insn_size(prog)) != 0)
978                 goto free_prog;
979 
980         prog->orig_prog = NULL;
981         prog->jited = 0;
982 
983         atomic_set(&prog->aux->refcnt, 1);
984         prog->gpl_compatible = is_gpl ? 1 : 0;
985 
986         /* find program type: socket_filter vs tracing_filter */
987         err = find_prog_type(type, prog);
988         if (err < 0)
989                 goto free_prog;
990 
991         /* run eBPF verifier */
992         err = bpf_check(&prog, attr);
993         if (err < 0)
994                 goto free_used_maps;
995 
996         /* eBPF program is ready to be JITed */
997         prog = bpf_prog_select_runtime(prog, &err);
998         if (err < 0)
999                 goto free_used_maps;
1000 
1001         err = bpf_prog_alloc_id(prog);
1002         if (err)
1003                 goto free_used_maps;
1004 
1005         err = bpf_prog_new_fd(prog);
1006         if (err < 0) {
1007                 /* failed to allocate fd.
1008                  * bpf_prog_put() is needed because the above
1009                  * bpf_prog_alloc_id() has published the prog
1010                  * to the userspace and the userspace may
1011                  * have refcnt-ed it through BPF_PROG_GET_FD_BY_ID.
1012                  */
1013                 bpf_prog_put(prog);
1014                 return err;
1015         }
1016 
1017         bpf_prog_kallsyms_add(prog);
1018         trace_bpf_prog_load(prog, err);
1019         return err;
1020 
1021 free_used_maps:
1022         free_used_maps(prog->aux);
1023 free_prog:
1024         bpf_prog_uncharge_memlock(prog);
1025 free_prog_nouncharge:
1026         bpf_prog_free(prog);
1027         return err;
1028 }
1029 
1030 #define BPF_OBJ_LAST_FIELD bpf_fd
1031 
1032 static int bpf_obj_pin(const union bpf_attr *attr)
1033 {
1034         if (CHECK_ATTR(BPF_OBJ))
1035                 return -EINVAL;
1036 
1037         return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname));
1038 }
1039 
1040 static int bpf_obj_get(const union bpf_attr *attr)
1041 {
1042         if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0)
1043                 return -EINVAL;
1044 
1045         return bpf_obj_get_user(u64_to_user_ptr(attr->pathname));
1046 }
1047 
1048 #ifdef CONFIG_CGROUP_BPF
1049 
1050 #define BPF_PROG_ATTACH_LAST_FIELD attach_flags
1051 
1052 static int bpf_prog_attach(const union bpf_attr *attr)
1053 {
1054         enum bpf_prog_type ptype;
1055         struct bpf_prog *prog;
1056         struct cgroup *cgrp;
1057         int ret;
1058 
1059         if (!capable(CAP_NET_ADMIN))
1060                 return -EPERM;
1061 
1062         if (CHECK_ATTR(BPF_PROG_ATTACH))
1063                 return -EINVAL;
1064 
1065         if (attr->attach_flags & ~BPF_F_ALLOW_OVERRIDE)
1066                 return -EINVAL;
1067 
1068         switch (attr->attach_type) {
1069         case BPF_CGROUP_INET_INGRESS:
1070         case BPF_CGROUP_INET_EGRESS:
1071                 ptype = BPF_PROG_TYPE_CGROUP_SKB;
1072                 break;
1073         case BPF_CGROUP_INET_SOCK_CREATE:
1074                 ptype = BPF_PROG_TYPE_CGROUP_SOCK;
1075                 break;
1076         case BPF_CGROUP_SOCK_OPS:
1077                 ptype = BPF_PROG_TYPE_SOCK_OPS;
1078                 break;
1079         default:
1080                 return -EINVAL;
1081         }
1082 
1083         prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
1084         if (IS_ERR(prog))
1085                 return PTR_ERR(prog);
1086 
1087         cgrp = cgroup_get_from_fd(attr->target_fd);
1088         if (IS_ERR(cgrp)) {
1089                 bpf_prog_put(prog);
1090                 return PTR_ERR(cgrp);
1091         }
1092 
1093         ret = cgroup_bpf_update(cgrp, prog, attr->attach_type,
1094                                 attr->attach_flags & BPF_F_ALLOW_OVERRIDE);
1095         if (ret)
1096                 bpf_prog_put(prog);
1097         cgroup_put(cgrp);
1098 
1099         return ret;
1100 }
1101 
1102 #define BPF_PROG_DETACH_LAST_FIELD attach_type
1103 
1104 static int bpf_prog_detach(const union bpf_attr *attr)
1105 {
1106         struct cgroup *cgrp;
1107         int ret;
1108 
1109         if (!capable(CAP_NET_ADMIN))
1110                 return -EPERM;
1111 
1112         if (CHECK_ATTR(BPF_PROG_DETACH))
1113                 return -EINVAL;
1114 
1115         switch (attr->attach_type) {
1116         case BPF_CGROUP_INET_INGRESS:
1117         case BPF_CGROUP_INET_EGRESS:
1118         case BPF_CGROUP_INET_SOCK_CREATE:
1119         case BPF_CGROUP_SOCK_OPS:
1120                 cgrp = cgroup_get_from_fd(attr->target_fd);
1121                 if (IS_ERR(cgrp))
1122                         return PTR_ERR(cgrp);
1123 
1124                 ret = cgroup_bpf_update(cgrp, NULL, attr->attach_type, false);
1125                 cgroup_put(cgrp);
1126                 break;
1127 
1128         default:
1129                 return -EINVAL;
1130         }
1131 
1132         return ret;
1133 }
1134 
1135 #endif /* CONFIG_CGROUP_BPF */
1136 
1137 #define BPF_PROG_TEST_RUN_LAST_FIELD test.duration
1138 
1139 static int bpf_prog_test_run(const union bpf_attr *attr,
1140                              union bpf_attr __user *uattr)
1141 {
1142         struct bpf_prog *prog;
1143         int ret = -ENOTSUPP;
1144 
1145         if (CHECK_ATTR(BPF_PROG_TEST_RUN))
1146                 return -EINVAL;
1147 
1148         prog = bpf_prog_get(attr->test.prog_fd);
1149         if (IS_ERR(prog))
1150                 return PTR_ERR(prog);
1151 
1152         if (prog->aux->ops->test_run)
1153                 ret = prog->aux->ops->test_run(prog, attr, uattr);
1154 
1155         bpf_prog_put(prog);
1156         return ret;
1157 }
1158 
1159 #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id
1160 
1161 static int bpf_obj_get_next_id(const union bpf_attr *attr,
1162                                union bpf_attr __user *uattr,
1163                                struct idr *idr,
1164                                spinlock_t *lock)
1165 {
1166         u32 next_id = attr->start_id;
1167         int err = 0;
1168 
1169         if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX)
1170                 return -EINVAL;
1171 
1172         if (!capable(CAP_SYS_ADMIN))
1173                 return -EPERM;
1174 
1175         next_id++;
1176         spin_lock_bh(lock);
1177         if (!idr_get_next(idr, &next_id))
1178                 err = -ENOENT;
1179         spin_unlock_bh(lock);
1180 
1181         if (!err)
1182                 err = put_user(next_id, &uattr->next_id);
1183 
1184         return err;
1185 }
1186 
1187 #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id
1188 
1189 static int bpf_prog_get_fd_by_id(const union bpf_attr *attr)
1190 {
1191         struct bpf_prog *prog;
1192         u32 id = attr->prog_id;
1193         int fd;
1194 
1195         if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID))
1196                 return -EINVAL;
1197 
1198         if (!capable(CAP_SYS_ADMIN))
1199                 return -EPERM;
1200 
1201         spin_lock_bh(&prog_idr_lock);
1202         prog = idr_find(&prog_idr, id);
1203         if (prog)
1204                 prog = bpf_prog_inc_not_zero(prog);
1205         else
1206                 prog = ERR_PTR(-ENOENT);
1207         spin_unlock_bh(&prog_idr_lock);
1208 
1209         if (IS_ERR(prog))
1210                 return PTR_ERR(prog);
1211 
1212         fd = bpf_prog_new_fd(prog);
1213         if (fd < 0)
1214                 bpf_prog_put(prog);
1215 
1216         return fd;
1217 }
1218 
1219 #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD map_id
1220 
1221 static int bpf_map_get_fd_by_id(const union bpf_attr *attr)
1222 {
1223         struct bpf_map *map;
1224         u32 id = attr->map_id;
1225         int fd;
1226 
1227         if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID))
1228                 return -EINVAL;
1229 
1230         if (!capable(CAP_SYS_ADMIN))
1231                 return -EPERM;
1232 
1233         spin_lock_bh(&map_idr_lock);
1234         map = idr_find(&map_idr, id);
1235         if (map)
1236                 map = bpf_map_inc_not_zero(map, true);
1237         else
1238                 map = ERR_PTR(-ENOENT);
1239         spin_unlock_bh(&map_idr_lock);
1240 
1241         if (IS_ERR(map))
1242                 return PTR_ERR(map);
1243 
1244         fd = bpf_map_new_fd(map);
1245         if (fd < 0)
1246                 bpf_map_put(map);
1247 
1248         return fd;
1249 }
1250 
1251 static int check_uarg_tail_zero(void __user *uaddr,
1252                                 size_t expected_size,
1253                                 size_t actual_size)
1254 {
1255         unsigned char __user *addr;
1256         unsigned char __user *end;
1257         unsigned char val;
1258         int err;
1259 
1260         if (actual_size <= expected_size)
1261                 return 0;
1262 
1263         addr = uaddr + expected_size;
1264         end  = uaddr + actual_size;
1265 
1266         for (; addr < end; addr++) {
1267                 err = get_user(val, addr);
1268                 if (err)
1269                         return err;
1270                 if (val)
1271                         return -E2BIG;
1272         }
1273 
1274         return 0;
1275 }
1276 
1277 static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
1278                                    const union bpf_attr *attr,
1279                                    union bpf_attr __user *uattr)
1280 {
1281         struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);
1282         struct bpf_prog_info info = {};
1283         u32 info_len = attr->info.info_len;
1284         char __user *uinsns;
1285         u32 ulen;
1286         int err;
1287 
1288         err = check_uarg_tail_zero(uinfo, sizeof(info), info_len);
1289         if (err)
1290                 return err;
1291         info_len = min_t(u32, sizeof(info), info_len);
1292 
1293         if (copy_from_user(&info, uinfo, info_len))
1294                 return -EFAULT;
1295 
1296         info.type = prog->type;
1297         info.id = prog->aux->id;
1298 
1299         memcpy(info.tag, prog->tag, sizeof(prog->tag));
1300 
1301         if (!capable(CAP_SYS_ADMIN)) {
1302                 info.jited_prog_len = 0;
1303                 info.xlated_prog_len = 0;
1304                 goto done;
1305         }
1306 
1307         ulen = info.jited_prog_len;
1308         info.jited_prog_len = prog->jited_len;
1309         if (info.jited_prog_len && ulen) {
1310                 uinsns = u64_to_user_ptr(info.jited_prog_insns);
1311                 ulen = min_t(u32, info.jited_prog_len, ulen);
1312                 if (copy_to_user(uinsns, prog->bpf_func, ulen))
1313                         return -EFAULT;
1314         }
1315 
1316         ulen = info.xlated_prog_len;
1317         info.xlated_prog_len = bpf_prog_insn_size(prog);
1318         if (info.xlated_prog_len && ulen) {
1319                 uinsns = u64_to_user_ptr(info.xlated_prog_insns);
1320                 ulen = min_t(u32, info.xlated_prog_len, ulen);
1321                 if (copy_to_user(uinsns, prog->insnsi, ulen))
1322                         return -EFAULT;
1323         }
1324 
1325 done:
1326         if (copy_to_user(uinfo, &info, info_len) ||
1327             put_user(info_len, &uattr->info.info_len))
1328                 return -EFAULT;
1329 
1330         return 0;
1331 }
1332 
1333 static int bpf_map_get_info_by_fd(struct bpf_map *map,
1334                                   const union bpf_attr *attr,
1335                                   union bpf_attr __user *uattr)
1336 {
1337         struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info);
1338         struct bpf_map_info info = {};
1339         u32 info_len = attr->info.info_len;
1340         int err;
1341 
1342         err = check_uarg_tail_zero(uinfo, sizeof(info), info_len);
1343         if (err)
1344                 return err;
1345         info_len = min_t(u32, sizeof(info), info_len);
1346 
1347         info.type = map->map_type;
1348         info.id = map->id;
1349         info.key_size = map->key_size;
1350         info.value_size = map->value_size;
1351         info.max_entries = map->max_entries;
1352         info.map_flags = map->map_flags;
1353 
1354         if (copy_to_user(uinfo, &info, info_len) ||
1355             put_user(info_len, &uattr->info.info_len))
1356                 return -EFAULT;
1357 
1358         return 0;
1359 }
1360 
1361 #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info
1362 
1363 static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
1364                                   union bpf_attr __user *uattr)
1365 {
1366         int ufd = attr->info.bpf_fd;
1367         struct fd f;
1368         int err;
1369 
1370         if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD))
1371                 return -EINVAL;
1372 
1373         f = fdget(ufd);
1374         if (!f.file)
1375                 return -EBADFD;
1376 
1377         if (f.file->f_op == &bpf_prog_fops)
1378                 err = bpf_prog_get_info_by_fd(f.file->private_data, attr,
1379                                               uattr);
1380         else if (f.file->f_op == &bpf_map_fops)
1381                 err = bpf_map_get_info_by_fd(f.file->private_data, attr,
1382                                              uattr);
1383         else
1384                 err = -EINVAL;
1385 
1386         fdput(f);
1387         return err;
1388 }
1389 
1390 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
1391 {
1392         union bpf_attr attr = {};
1393         int err;
1394 
1395         if (!capable(CAP_SYS_ADMIN) && sysctl_unprivileged_bpf_disabled)
1396                 return -EPERM;
1397 
1398         if (!access_ok(VERIFY_READ, uattr, 1))
1399                 return -EFAULT;
1400 
1401         if (size > PAGE_SIZE)   /* silly large */
1402                 return -E2BIG;
1403 
1404         /* If we're handed a bigger struct than we know of,
1405          * ensure all the unknown bits are 0 - i.e. new
1406          * user-space does not rely on any kernel feature
1407          * extensions we dont know about yet.
1408          */
1409         err = check_uarg_tail_zero(uattr, sizeof(attr), size);
1410         if (err)
1411                 return err;
1412         size = min_t(u32, size, sizeof(attr));
1413 
1414         /* copy attributes from user space, may be less than sizeof(bpf_attr) */
1415         if (copy_from_user(&attr, uattr, size) != 0)
1416                 return -EFAULT;
1417 
1418         switch (cmd) {
1419         case BPF_MAP_CREATE:
1420                 err = map_create(&attr);
1421                 break;
1422         case BPF_MAP_LOOKUP_ELEM:
1423                 err = map_lookup_elem(&attr);
1424                 break;
1425         case BPF_MAP_UPDATE_ELEM:
1426                 err = map_update_elem(&attr);
1427                 break;
1428         case BPF_MAP_DELETE_ELEM:
1429                 err = map_delete_elem(&attr);
1430                 break;
1431         case BPF_MAP_GET_NEXT_KEY:
1432                 err = map_get_next_key(&attr);
1433                 break;
1434         case BPF_PROG_LOAD:
1435                 err = bpf_prog_load(&attr);
1436                 break;
1437         case BPF_OBJ_PIN:
1438                 err = bpf_obj_pin(&attr);
1439                 break;
1440         case BPF_OBJ_GET:
1441                 err = bpf_obj_get(&attr);
1442                 break;
1443 #ifdef CONFIG_CGROUP_BPF
1444         case BPF_PROG_ATTACH:
1445                 err = bpf_prog_attach(&attr);
1446                 break;
1447         case BPF_PROG_DETACH:
1448                 err = bpf_prog_detach(&attr);
1449                 break;
1450 #endif
1451         case BPF_PROG_TEST_RUN:
1452                 err = bpf_prog_test_run(&attr, uattr);
1453                 break;
1454         case BPF_PROG_GET_NEXT_ID:
1455                 err = bpf_obj_get_next_id(&attr, uattr,
1456                                           &prog_idr, &prog_idr_lock);
1457                 break;
1458         case BPF_MAP_GET_NEXT_ID:
1459                 err = bpf_obj_get_next_id(&attr, uattr,
1460                                           &map_idr, &map_idr_lock);
1461                 break;
1462         case BPF_PROG_GET_FD_BY_ID:
1463                 err = bpf_prog_get_fd_by_id(&attr);
1464                 break;
1465         case BPF_MAP_GET_FD_BY_ID:
1466                 err = bpf_map_get_fd_by_id(&attr);
1467                 break;
1468         case BPF_OBJ_GET_INFO_BY_FD:
1469                 err = bpf_obj_get_info_by_fd(&attr, uattr);
1470                 break;
1471         default:
1472                 err = -EINVAL;
1473                 break;
1474         }
1475 
1476         return err;
1477 }
1478 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp