~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/bpf/syscall.c

Version: ~ [ linux-5.5-rc1 ] ~ [ linux-5.4.2 ] ~ [ linux-5.3.15 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.88 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.158 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.206 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.206 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.78 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
  2  *
  3  * This program is free software; you can redistribute it and/or
  4  * modify it under the terms of version 2 of the GNU General Public
  5  * License as published by the Free Software Foundation.
  6  *
  7  * This program is distributed in the hope that it will be useful, but
  8  * WITHOUT ANY WARRANTY; without even the implied warranty of
  9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 10  * General Public License for more details.
 11  */
 12 #include <linux/bpf.h>
 13 #include <linux/syscalls.h>
 14 #include <linux/slab.h>
 15 #include <linux/anon_inodes.h>
 16 #include <linux/file.h>
 17 #include <linux/license.h>
 18 #include <linux/filter.h>
 19 #include <linux/version.h>
 20 
 21 static LIST_HEAD(bpf_map_types);
 22 
 23 static struct bpf_map *find_and_alloc_map(union bpf_attr *attr)
 24 {
 25         struct bpf_map_type_list *tl;
 26         struct bpf_map *map;
 27 
 28         list_for_each_entry(tl, &bpf_map_types, list_node) {
 29                 if (tl->type == attr->map_type) {
 30                         map = tl->ops->map_alloc(attr);
 31                         if (IS_ERR(map))
 32                                 return map;
 33                         map->ops = tl->ops;
 34                         map->map_type = attr->map_type;
 35                         return map;
 36                 }
 37         }
 38         return ERR_PTR(-EINVAL);
 39 }
 40 
 41 /* boot time registration of different map implementations */
 42 void bpf_register_map_type(struct bpf_map_type_list *tl)
 43 {
 44         list_add(&tl->list_node, &bpf_map_types);
 45 }
 46 
 47 /* called from workqueue */
 48 static void bpf_map_free_deferred(struct work_struct *work)
 49 {
 50         struct bpf_map *map = container_of(work, struct bpf_map, work);
 51 
 52         /* implementation dependent freeing */
 53         map->ops->map_free(map);
 54 }
 55 
 56 /* decrement map refcnt and schedule it for freeing via workqueue
 57  * (unrelying map implementation ops->map_free() might sleep)
 58  */
 59 void bpf_map_put(struct bpf_map *map)
 60 {
 61         if (atomic_dec_and_test(&map->refcnt)) {
 62                 INIT_WORK(&map->work, bpf_map_free_deferred);
 63                 schedule_work(&map->work);
 64         }
 65 }
 66 
 67 static int bpf_map_release(struct inode *inode, struct file *filp)
 68 {
 69         struct bpf_map *map = filp->private_data;
 70 
 71         if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY)
 72                 /* prog_array stores refcnt-ed bpf_prog pointers
 73                  * release them all when user space closes prog_array_fd
 74                  */
 75                 bpf_fd_array_map_clear(map);
 76 
 77         bpf_map_put(map);
 78         return 0;
 79 }
 80 
 81 static const struct file_operations bpf_map_fops = {
 82         .release = bpf_map_release,
 83 };
 84 
 85 /* helper macro to check that unused fields 'union bpf_attr' are zero */
 86 #define CHECK_ATTR(CMD) \
 87         memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
 88                    sizeof(attr->CMD##_LAST_FIELD), 0, \
 89                    sizeof(*attr) - \
 90                    offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
 91                    sizeof(attr->CMD##_LAST_FIELD)) != NULL
 92 
 93 #define BPF_MAP_CREATE_LAST_FIELD max_entries
 94 /* called via syscall */
 95 static int map_create(union bpf_attr *attr)
 96 {
 97         struct bpf_map *map;
 98         int err;
 99 
100         err = CHECK_ATTR(BPF_MAP_CREATE);
101         if (err)
102                 return -EINVAL;
103 
104         /* find map type and init map: hashtable vs rbtree vs bloom vs ... */
105         map = find_and_alloc_map(attr);
106         if (IS_ERR(map))
107                 return PTR_ERR(map);
108 
109         atomic_set(&map->refcnt, 1);
110 
111         err = anon_inode_getfd("bpf-map", &bpf_map_fops, map, O_RDWR | O_CLOEXEC);
112 
113         if (err < 0)
114                 /* failed to allocate fd */
115                 goto free_map;
116 
117         return err;
118 
119 free_map:
120         map->ops->map_free(map);
121         return err;
122 }
123 
124 /* if error is returned, fd is released.
125  * On success caller should complete fd access with matching fdput()
126  */
127 struct bpf_map *bpf_map_get(struct fd f)
128 {
129         struct bpf_map *map;
130 
131         if (!f.file)
132                 return ERR_PTR(-EBADF);
133 
134         if (f.file->f_op != &bpf_map_fops) {
135                 fdput(f);
136                 return ERR_PTR(-EINVAL);
137         }
138 
139         map = f.file->private_data;
140 
141         return map;
142 }
143 
144 /* helper to convert user pointers passed inside __aligned_u64 fields */
145 static void __user *u64_to_ptr(__u64 val)
146 {
147         return (void __user *) (unsigned long) val;
148 }
149 
150 /* last field in 'union bpf_attr' used by this command */
151 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value
152 
153 static int map_lookup_elem(union bpf_attr *attr)
154 {
155         void __user *ukey = u64_to_ptr(attr->key);
156         void __user *uvalue = u64_to_ptr(attr->value);
157         int ufd = attr->map_fd;
158         struct bpf_map *map;
159         void *key, *value, *ptr;
160         struct fd f;
161         int err;
162 
163         if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
164                 return -EINVAL;
165 
166         f = fdget(ufd);
167         map = bpf_map_get(f);
168         if (IS_ERR(map))
169                 return PTR_ERR(map);
170 
171         err = -ENOMEM;
172         key = kmalloc(map->key_size, GFP_USER);
173         if (!key)
174                 goto err_put;
175 
176         err = -EFAULT;
177         if (copy_from_user(key, ukey, map->key_size) != 0)
178                 goto free_key;
179 
180         err = -ENOMEM;
181         value = kmalloc(map->value_size, GFP_USER);
182         if (!value)
183                 goto free_key;
184 
185         rcu_read_lock();
186         ptr = map->ops->map_lookup_elem(map, key);
187         if (ptr)
188                 memcpy(value, ptr, map->value_size);
189         rcu_read_unlock();
190 
191         err = -ENOENT;
192         if (!ptr)
193                 goto free_value;
194 
195         err = -EFAULT;
196         if (copy_to_user(uvalue, value, map->value_size) != 0)
197                 goto free_value;
198 
199         err = 0;
200 
201 free_value:
202         kfree(value);
203 free_key:
204         kfree(key);
205 err_put:
206         fdput(f);
207         return err;
208 }
209 
210 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
211 
212 static int map_update_elem(union bpf_attr *attr)
213 {
214         void __user *ukey = u64_to_ptr(attr->key);
215         void __user *uvalue = u64_to_ptr(attr->value);
216         int ufd = attr->map_fd;
217         struct bpf_map *map;
218         void *key, *value;
219         struct fd f;
220         int err;
221 
222         if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
223                 return -EINVAL;
224 
225         f = fdget(ufd);
226         map = bpf_map_get(f);
227         if (IS_ERR(map))
228                 return PTR_ERR(map);
229 
230         err = -ENOMEM;
231         key = kmalloc(map->key_size, GFP_USER);
232         if (!key)
233                 goto err_put;
234 
235         err = -EFAULT;
236         if (copy_from_user(key, ukey, map->key_size) != 0)
237                 goto free_key;
238 
239         err = -ENOMEM;
240         value = kmalloc(map->value_size, GFP_USER);
241         if (!value)
242                 goto free_key;
243 
244         err = -EFAULT;
245         if (copy_from_user(value, uvalue, map->value_size) != 0)
246                 goto free_value;
247 
248         /* eBPF program that use maps are running under rcu_read_lock(),
249          * therefore all map accessors rely on this fact, so do the same here
250          */
251         rcu_read_lock();
252         err = map->ops->map_update_elem(map, key, value, attr->flags);
253         rcu_read_unlock();
254 
255 free_value:
256         kfree(value);
257 free_key:
258         kfree(key);
259 err_put:
260         fdput(f);
261         return err;
262 }
263 
264 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key
265 
266 static int map_delete_elem(union bpf_attr *attr)
267 {
268         void __user *ukey = u64_to_ptr(attr->key);
269         int ufd = attr->map_fd;
270         struct bpf_map *map;
271         struct fd f;
272         void *key;
273         int err;
274 
275         if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
276                 return -EINVAL;
277 
278         f = fdget(ufd);
279         map = bpf_map_get(f);
280         if (IS_ERR(map))
281                 return PTR_ERR(map);
282 
283         err = -ENOMEM;
284         key = kmalloc(map->key_size, GFP_USER);
285         if (!key)
286                 goto err_put;
287 
288         err = -EFAULT;
289         if (copy_from_user(key, ukey, map->key_size) != 0)
290                 goto free_key;
291 
292         rcu_read_lock();
293         err = map->ops->map_delete_elem(map, key);
294         rcu_read_unlock();
295 
296 free_key:
297         kfree(key);
298 err_put:
299         fdput(f);
300         return err;
301 }
302 
303 /* last field in 'union bpf_attr' used by this command */
304 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
305 
306 static int map_get_next_key(union bpf_attr *attr)
307 {
308         void __user *ukey = u64_to_ptr(attr->key);
309         void __user *unext_key = u64_to_ptr(attr->next_key);
310         int ufd = attr->map_fd;
311         struct bpf_map *map;
312         void *key, *next_key;
313         struct fd f;
314         int err;
315 
316         if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
317                 return -EINVAL;
318 
319         f = fdget(ufd);
320         map = bpf_map_get(f);
321         if (IS_ERR(map))
322                 return PTR_ERR(map);
323 
324         err = -ENOMEM;
325         key = kmalloc(map->key_size, GFP_USER);
326         if (!key)
327                 goto err_put;
328 
329         err = -EFAULT;
330         if (copy_from_user(key, ukey, map->key_size) != 0)
331                 goto free_key;
332 
333         err = -ENOMEM;
334         next_key = kmalloc(map->key_size, GFP_USER);
335         if (!next_key)
336                 goto free_key;
337 
338         rcu_read_lock();
339         err = map->ops->map_get_next_key(map, key, next_key);
340         rcu_read_unlock();
341         if (err)
342                 goto free_next_key;
343 
344         err = -EFAULT;
345         if (copy_to_user(unext_key, next_key, map->key_size) != 0)
346                 goto free_next_key;
347 
348         err = 0;
349 
350 free_next_key:
351         kfree(next_key);
352 free_key:
353         kfree(key);
354 err_put:
355         fdput(f);
356         return err;
357 }
358 
359 static LIST_HEAD(bpf_prog_types);
360 
361 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
362 {
363         struct bpf_prog_type_list *tl;
364 
365         list_for_each_entry(tl, &bpf_prog_types, list_node) {
366                 if (tl->type == type) {
367                         prog->aux->ops = tl->ops;
368                         prog->type = type;
369                         return 0;
370                 }
371         }
372 
373         return -EINVAL;
374 }
375 
376 void bpf_register_prog_type(struct bpf_prog_type_list *tl)
377 {
378         list_add(&tl->list_node, &bpf_prog_types);
379 }
380 
381 /* fixup insn->imm field of bpf_call instructions:
382  * if (insn->imm == BPF_FUNC_map_lookup_elem)
383  *      insn->imm = bpf_map_lookup_elem - __bpf_call_base;
384  * else if (insn->imm == BPF_FUNC_map_update_elem)
385  *      insn->imm = bpf_map_update_elem - __bpf_call_base;
386  * else ...
387  *
388  * this function is called after eBPF program passed verification
389  */
390 static void fixup_bpf_calls(struct bpf_prog *prog)
391 {
392         const struct bpf_func_proto *fn;
393         int i;
394 
395         for (i = 0; i < prog->len; i++) {
396                 struct bpf_insn *insn = &prog->insnsi[i];
397 
398                 if (insn->code == (BPF_JMP | BPF_CALL)) {
399                         /* we reach here when program has bpf_call instructions
400                          * and it passed bpf_check(), means that
401                          * ops->get_func_proto must have been supplied, check it
402                          */
403                         BUG_ON(!prog->aux->ops->get_func_proto);
404 
405                         if (insn->imm == BPF_FUNC_tail_call) {
406                                 /* mark bpf_tail_call as different opcode
407                                  * to avoid conditional branch in
408                                  * interpeter for every normal call
409                                  * and to prevent accidental JITing by
410                                  * JIT compiler that doesn't support
411                                  * bpf_tail_call yet
412                                  */
413                                 insn->imm = 0;
414                                 insn->code |= BPF_X;
415                                 continue;
416                         }
417 
418                         fn = prog->aux->ops->get_func_proto(insn->imm);
419                         /* all functions that have prototype and verifier allowed
420                          * programs to call them, must be real in-kernel functions
421                          */
422                         BUG_ON(!fn->func);
423                         insn->imm = fn->func - __bpf_call_base;
424                 }
425         }
426 }
427 
428 /* drop refcnt on maps used by eBPF program and free auxilary data */
429 static void free_used_maps(struct bpf_prog_aux *aux)
430 {
431         int i;
432 
433         for (i = 0; i < aux->used_map_cnt; i++)
434                 bpf_map_put(aux->used_maps[i]);
435 
436         kfree(aux->used_maps);
437 }
438 
439 static void __prog_put_rcu(struct rcu_head *rcu)
440 {
441         struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
442 
443         free_used_maps(aux);
444         bpf_prog_free(aux->prog);
445 }
446 
447 /* version of bpf_prog_put() that is called after a grace period */
448 void bpf_prog_put_rcu(struct bpf_prog *prog)
449 {
450         if (atomic_dec_and_test(&prog->aux->refcnt)) {
451                 prog->aux->prog = prog;
452                 call_rcu(&prog->aux->rcu, __prog_put_rcu);
453         }
454 }
455 
456 void bpf_prog_put(struct bpf_prog *prog)
457 {
458         if (atomic_dec_and_test(&prog->aux->refcnt)) {
459                 free_used_maps(prog->aux);
460                 bpf_prog_free(prog);
461         }
462 }
463 EXPORT_SYMBOL_GPL(bpf_prog_put);
464 
465 static int bpf_prog_release(struct inode *inode, struct file *filp)
466 {
467         struct bpf_prog *prog = filp->private_data;
468 
469         bpf_prog_put_rcu(prog);
470         return 0;
471 }
472 
473 static const struct file_operations bpf_prog_fops = {
474         .release = bpf_prog_release,
475 };
476 
477 static struct bpf_prog *get_prog(struct fd f)
478 {
479         struct bpf_prog *prog;
480 
481         if (!f.file)
482                 return ERR_PTR(-EBADF);
483 
484         if (f.file->f_op != &bpf_prog_fops) {
485                 fdput(f);
486                 return ERR_PTR(-EINVAL);
487         }
488 
489         prog = f.file->private_data;
490 
491         return prog;
492 }
493 
494 /* called by sockets/tracing/seccomp before attaching program to an event
495  * pairs with bpf_prog_put()
496  */
497 struct bpf_prog *bpf_prog_get(u32 ufd)
498 {
499         struct fd f = fdget(ufd);
500         struct bpf_prog *prog;
501 
502         prog = get_prog(f);
503 
504         if (IS_ERR(prog))
505                 return prog;
506 
507         atomic_inc(&prog->aux->refcnt);
508         fdput(f);
509         return prog;
510 }
511 EXPORT_SYMBOL_GPL(bpf_prog_get);
512 
513 /* last field in 'union bpf_attr' used by this command */
514 #define BPF_PROG_LOAD_LAST_FIELD kern_version
515 
516 static int bpf_prog_load(union bpf_attr *attr)
517 {
518         enum bpf_prog_type type = attr->prog_type;
519         struct bpf_prog *prog;
520         int err;
521         char license[128];
522         bool is_gpl;
523 
524         if (CHECK_ATTR(BPF_PROG_LOAD))
525                 return -EINVAL;
526 
527         /* copy eBPF program license from user space */
528         if (strncpy_from_user(license, u64_to_ptr(attr->license),
529                               sizeof(license) - 1) < 0)
530                 return -EFAULT;
531         license[sizeof(license) - 1] = 0;
532 
533         /* eBPF programs must be GPL compatible to use GPL-ed functions */
534         is_gpl = license_is_gpl_compatible(license);
535 
536         if (attr->insn_cnt >= BPF_MAXINSNS)
537                 return -EINVAL;
538 
539         if (type == BPF_PROG_TYPE_KPROBE &&
540             attr->kern_version != LINUX_VERSION_CODE)
541                 return -EINVAL;
542 
543         /* plain bpf_prog allocation */
544         prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
545         if (!prog)
546                 return -ENOMEM;
547 
548         prog->len = attr->insn_cnt;
549 
550         err = -EFAULT;
551         if (copy_from_user(prog->insns, u64_to_ptr(attr->insns),
552                            prog->len * sizeof(struct bpf_insn)) != 0)
553                 goto free_prog;
554 
555         prog->orig_prog = NULL;
556         prog->jited = false;
557 
558         atomic_set(&prog->aux->refcnt, 1);
559         prog->gpl_compatible = is_gpl;
560 
561         /* find program type: socket_filter vs tracing_filter */
562         err = find_prog_type(type, prog);
563         if (err < 0)
564                 goto free_prog;
565 
566         /* run eBPF verifier */
567         err = bpf_check(&prog, attr);
568         if (err < 0)
569                 goto free_used_maps;
570 
571         /* fixup BPF_CALL->imm field */
572         fixup_bpf_calls(prog);
573 
574         /* eBPF program is ready to be JITed */
575         err = bpf_prog_select_runtime(prog);
576         if (err < 0)
577                 goto free_used_maps;
578 
579         err = anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, O_RDWR | O_CLOEXEC);
580         if (err < 0)
581                 /* failed to allocate fd */
582                 goto free_used_maps;
583 
584         return err;
585 
586 free_used_maps:
587         free_used_maps(prog->aux);
588 free_prog:
589         bpf_prog_free(prog);
590         return err;
591 }
592 
593 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
594 {
595         union bpf_attr attr = {};
596         int err;
597 
598         /* the syscall is limited to root temporarily. This restriction will be
599          * lifted when security audit is clean. Note that eBPF+tracing must have
600          * this restriction, since it may pass kernel data to user space
601          */
602         if (!capable(CAP_SYS_ADMIN))
603                 return -EPERM;
604 
605         if (!access_ok(VERIFY_READ, uattr, 1))
606                 return -EFAULT;
607 
608         if (size > PAGE_SIZE)   /* silly large */
609                 return -E2BIG;
610 
611         /* If we're handed a bigger struct than we know of,
612          * ensure all the unknown bits are 0 - i.e. new
613          * user-space does not rely on any kernel feature
614          * extensions we dont know about yet.
615          */
616         if (size > sizeof(attr)) {
617                 unsigned char __user *addr;
618                 unsigned char __user *end;
619                 unsigned char val;
620 
621                 addr = (void __user *)uattr + sizeof(attr);
622                 end  = (void __user *)uattr + size;
623 
624                 for (; addr < end; addr++) {
625                         err = get_user(val, addr);
626                         if (err)
627                                 return err;
628                         if (val)
629                                 return -E2BIG;
630                 }
631                 size = sizeof(attr);
632         }
633 
634         /* copy attributes from user space, may be less than sizeof(bpf_attr) */
635         if (copy_from_user(&attr, uattr, size) != 0)
636                 return -EFAULT;
637 
638         switch (cmd) {
639         case BPF_MAP_CREATE:
640                 err = map_create(&attr);
641                 break;
642         case BPF_MAP_LOOKUP_ELEM:
643                 err = map_lookup_elem(&attr);
644                 break;
645         case BPF_MAP_UPDATE_ELEM:
646                 err = map_update_elem(&attr);
647                 break;
648         case BPF_MAP_DELETE_ELEM:
649                 err = map_delete_elem(&attr);
650                 break;
651         case BPF_MAP_GET_NEXT_KEY:
652                 err = map_get_next_key(&attr);
653                 break;
654         case BPF_PROG_LOAD:
655                 err = bpf_prog_load(&attr);
656                 break;
657         default:
658                 err = -EINVAL;
659                 break;
660         }
661 
662         return err;
663 }
664 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp