~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/bpf/cgroup.c

Version: ~ [ linux-5.16-rc3 ] ~ [ linux-5.15.5 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.82 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.162 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.218 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.256 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.291 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.293 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-only
  2 /*
  3  * Functions to manage eBPF programs attached to cgroups
  4  *
  5  * Copyright (c) 2016 Daniel Mack
  6  */
  7 
  8 #include <linux/kernel.h>
  9 #include <linux/atomic.h>
 10 #include <linux/cgroup.h>
 11 #include <linux/filter.h>
 12 #include <linux/slab.h>
 13 #include <linux/sysctl.h>
 14 #include <linux/string.h>
 15 #include <linux/bpf.h>
 16 #include <linux/bpf-cgroup.h>
 17 #include <net/sock.h>
 18 #include <net/bpf_sk_storage.h>
 19 
 20 #include "../cgroup/cgroup-internal.h"
 21 
 22 DEFINE_STATIC_KEY_FALSE(cgroup_bpf_enabled_key);
 23 EXPORT_SYMBOL(cgroup_bpf_enabled_key);
 24 
 25 void cgroup_bpf_offline(struct cgroup *cgrp)
 26 {
 27         cgroup_get(cgrp);
 28         percpu_ref_kill(&cgrp->bpf.refcnt);
 29 }
 30 
 31 /**
 32  * cgroup_bpf_release() - put references of all bpf programs and
 33  *                        release all cgroup bpf data
 34  * @work: work structure embedded into the cgroup to modify
 35  */
 36 static void cgroup_bpf_release(struct work_struct *work)
 37 {
 38         struct cgroup *p, *cgrp = container_of(work, struct cgroup,
 39                                                bpf.release_work);
 40         enum bpf_cgroup_storage_type stype;
 41         struct bpf_prog_array *old_array;
 42         unsigned int type;
 43 
 44         mutex_lock(&cgroup_mutex);
 45 
 46         for (type = 0; type < ARRAY_SIZE(cgrp->bpf.progs); type++) {
 47                 struct list_head *progs = &cgrp->bpf.progs[type];
 48                 struct bpf_prog_list *pl, *tmp;
 49 
 50                 list_for_each_entry_safe(pl, tmp, progs, node) {
 51                         list_del(&pl->node);
 52                         bpf_prog_put(pl->prog);
 53                         for_each_cgroup_storage_type(stype) {
 54                                 bpf_cgroup_storage_unlink(pl->storage[stype]);
 55                                 bpf_cgroup_storage_free(pl->storage[stype]);
 56                         }
 57                         kfree(pl);
 58                         static_branch_dec(&cgroup_bpf_enabled_key);
 59                 }
 60                 old_array = rcu_dereference_protected(
 61                                 cgrp->bpf.effective[type],
 62                                 lockdep_is_held(&cgroup_mutex));
 63                 bpf_prog_array_free(old_array);
 64         }
 65 
 66         mutex_unlock(&cgroup_mutex);
 67 
 68         for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p))
 69                 cgroup_bpf_put(p);
 70 
 71         percpu_ref_exit(&cgrp->bpf.refcnt);
 72         cgroup_put(cgrp);
 73 }
 74 
 75 /**
 76  * cgroup_bpf_release_fn() - callback used to schedule releasing
 77  *                           of bpf cgroup data
 78  * @ref: percpu ref counter structure
 79  */
 80 static void cgroup_bpf_release_fn(struct percpu_ref *ref)
 81 {
 82         struct cgroup *cgrp = container_of(ref, struct cgroup, bpf.refcnt);
 83 
 84         INIT_WORK(&cgrp->bpf.release_work, cgroup_bpf_release);
 85         queue_work(system_wq, &cgrp->bpf.release_work);
 86 }
 87 
 88 /* count number of elements in the list.
 89  * it's slow but the list cannot be long
 90  */
 91 static u32 prog_list_length(struct list_head *head)
 92 {
 93         struct bpf_prog_list *pl;
 94         u32 cnt = 0;
 95 
 96         list_for_each_entry(pl, head, node) {
 97                 if (!pl->prog)
 98                         continue;
 99                 cnt++;
100         }
101         return cnt;
102 }
103 
104 /* if parent has non-overridable prog attached,
105  * disallow attaching new programs to the descendent cgroup.
106  * if parent has overridable or multi-prog, allow attaching
107  */
108 static bool hierarchy_allows_attach(struct cgroup *cgrp,
109                                     enum bpf_attach_type type,
110                                     u32 new_flags)
111 {
112         struct cgroup *p;
113 
114         p = cgroup_parent(cgrp);
115         if (!p)
116                 return true;
117         do {
118                 u32 flags = p->bpf.flags[type];
119                 u32 cnt;
120 
121                 if (flags & BPF_F_ALLOW_MULTI)
122                         return true;
123                 cnt = prog_list_length(&p->bpf.progs[type]);
124                 WARN_ON_ONCE(cnt > 1);
125                 if (cnt == 1)
126                         return !!(flags & BPF_F_ALLOW_OVERRIDE);
127                 p = cgroup_parent(p);
128         } while (p);
129         return true;
130 }
131 
132 /* compute a chain of effective programs for a given cgroup:
133  * start from the list of programs in this cgroup and add
134  * all parent programs.
135  * Note that parent's F_ALLOW_OVERRIDE-type program is yielding
136  * to programs in this cgroup
137  */
138 static int compute_effective_progs(struct cgroup *cgrp,
139                                    enum bpf_attach_type type,
140                                    struct bpf_prog_array **array)
141 {
142         enum bpf_cgroup_storage_type stype;
143         struct bpf_prog_array *progs;
144         struct bpf_prog_list *pl;
145         struct cgroup *p = cgrp;
146         int cnt = 0;
147 
148         /* count number of effective programs by walking parents */
149         do {
150                 if (cnt == 0 || (p->bpf.flags[type] & BPF_F_ALLOW_MULTI))
151                         cnt += prog_list_length(&p->bpf.progs[type]);
152                 p = cgroup_parent(p);
153         } while (p);
154 
155         progs = bpf_prog_array_alloc(cnt, GFP_KERNEL);
156         if (!progs)
157                 return -ENOMEM;
158 
159         /* populate the array with effective progs */
160         cnt = 0;
161         p = cgrp;
162         do {
163                 if (cnt > 0 && !(p->bpf.flags[type] & BPF_F_ALLOW_MULTI))
164                         continue;
165 
166                 list_for_each_entry(pl, &p->bpf.progs[type], node) {
167                         if (!pl->prog)
168                                 continue;
169 
170                         progs->items[cnt].prog = pl->prog;
171                         for_each_cgroup_storage_type(stype)
172                                 progs->items[cnt].cgroup_storage[stype] =
173                                         pl->storage[stype];
174                         cnt++;
175                 }
176         } while ((p = cgroup_parent(p)));
177 
178         *array = progs;
179         return 0;
180 }
181 
182 static void activate_effective_progs(struct cgroup *cgrp,
183                                      enum bpf_attach_type type,
184                                      struct bpf_prog_array *old_array)
185 {
186         old_array = rcu_replace_pointer(cgrp->bpf.effective[type], old_array,
187                                         lockdep_is_held(&cgroup_mutex));
188         /* free prog array after grace period, since __cgroup_bpf_run_*()
189          * might be still walking the array
190          */
191         bpf_prog_array_free(old_array);
192 }
193 
194 /**
195  * cgroup_bpf_inherit() - inherit effective programs from parent
196  * @cgrp: the cgroup to modify
197  */
198 int cgroup_bpf_inherit(struct cgroup *cgrp)
199 {
200 /* has to use marco instead of const int, since compiler thinks
201  * that array below is variable length
202  */
203 #define NR ARRAY_SIZE(cgrp->bpf.effective)
204         struct bpf_prog_array *arrays[NR] = {};
205         struct cgroup *p;
206         int ret, i;
207 
208         ret = percpu_ref_init(&cgrp->bpf.refcnt, cgroup_bpf_release_fn, 0,
209                               GFP_KERNEL);
210         if (ret)
211                 return ret;
212 
213         for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p))
214                 cgroup_bpf_get(p);
215 
216         for (i = 0; i < NR; i++)
217                 INIT_LIST_HEAD(&cgrp->bpf.progs[i]);
218 
219         for (i = 0; i < NR; i++)
220                 if (compute_effective_progs(cgrp, i, &arrays[i]))
221                         goto cleanup;
222 
223         for (i = 0; i < NR; i++)
224                 activate_effective_progs(cgrp, i, arrays[i]);
225 
226         return 0;
227 cleanup:
228         for (i = 0; i < NR; i++)
229                 bpf_prog_array_free(arrays[i]);
230 
231         for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p))
232                 cgroup_bpf_put(p);
233 
234         percpu_ref_exit(&cgrp->bpf.refcnt);
235 
236         return -ENOMEM;
237 }
238 
239 static int update_effective_progs(struct cgroup *cgrp,
240                                   enum bpf_attach_type type)
241 {
242         struct cgroup_subsys_state *css;
243         int err;
244 
245         /* allocate and recompute effective prog arrays */
246         css_for_each_descendant_pre(css, &cgrp->self) {
247                 struct cgroup *desc = container_of(css, struct cgroup, self);
248 
249                 if (percpu_ref_is_zero(&desc->bpf.refcnt))
250                         continue;
251 
252                 err = compute_effective_progs(desc, type, &desc->bpf.inactive);
253                 if (err)
254                         goto cleanup;
255         }
256 
257         /* all allocations were successful. Activate all prog arrays */
258         css_for_each_descendant_pre(css, &cgrp->self) {
259                 struct cgroup *desc = container_of(css, struct cgroup, self);
260 
261                 if (percpu_ref_is_zero(&desc->bpf.refcnt)) {
262                         if (unlikely(desc->bpf.inactive)) {
263                                 bpf_prog_array_free(desc->bpf.inactive);
264                                 desc->bpf.inactive = NULL;
265                         }
266                         continue;
267                 }
268 
269                 activate_effective_progs(desc, type, desc->bpf.inactive);
270                 desc->bpf.inactive = NULL;
271         }
272 
273         return 0;
274 
275 cleanup:
276         /* oom while computing effective. Free all computed effective arrays
277          * since they were not activated
278          */
279         css_for_each_descendant_pre(css, &cgrp->self) {
280                 struct cgroup *desc = container_of(css, struct cgroup, self);
281 
282                 bpf_prog_array_free(desc->bpf.inactive);
283                 desc->bpf.inactive = NULL;
284         }
285 
286         return err;
287 }
288 
289 #define BPF_CGROUP_MAX_PROGS 64
290 
291 /**
292  * __cgroup_bpf_attach() - Attach the program to a cgroup, and
293  *                         propagate the change to descendants
294  * @cgrp: The cgroup which descendants to traverse
295  * @prog: A program to attach
296  * @type: Type of attach operation
297  * @flags: Option flags
298  *
299  * Must be called with cgroup_mutex held.
300  */
301 int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog,
302                         enum bpf_attach_type type, u32 flags)
303 {
304         struct list_head *progs = &cgrp->bpf.progs[type];
305         struct bpf_prog *old_prog = NULL;
306         struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {};
307         struct bpf_cgroup_storage *old_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {};
308         enum bpf_cgroup_storage_type stype;
309         struct bpf_prog_list *pl;
310         bool pl_was_allocated;
311         int err;
312 
313         if ((flags & BPF_F_ALLOW_OVERRIDE) && (flags & BPF_F_ALLOW_MULTI))
314                 /* invalid combination */
315                 return -EINVAL;
316 
317         if (!hierarchy_allows_attach(cgrp, type, flags))
318                 return -EPERM;
319 
320         if (!list_empty(progs) && cgrp->bpf.flags[type] != flags)
321                 /* Disallow attaching non-overridable on top
322                  * of existing overridable in this cgroup.
323                  * Disallow attaching multi-prog if overridable or none
324                  */
325                 return -EPERM;
326 
327         if (prog_list_length(progs) >= BPF_CGROUP_MAX_PROGS)
328                 return -E2BIG;
329 
330         for_each_cgroup_storage_type(stype) {
331                 storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
332                 if (IS_ERR(storage[stype])) {
333                         storage[stype] = NULL;
334                         for_each_cgroup_storage_type(stype)
335                                 bpf_cgroup_storage_free(storage[stype]);
336                         return -ENOMEM;
337                 }
338         }
339 
340         if (flags & BPF_F_ALLOW_MULTI) {
341                 list_for_each_entry(pl, progs, node) {
342                         if (pl->prog == prog) {
343                                 /* disallow attaching the same prog twice */
344                                 for_each_cgroup_storage_type(stype)
345                                         bpf_cgroup_storage_free(storage[stype]);
346                                 return -EINVAL;
347                         }
348                 }
349 
350                 pl = kmalloc(sizeof(*pl), GFP_KERNEL);
351                 if (!pl) {
352                         for_each_cgroup_storage_type(stype)
353                                 bpf_cgroup_storage_free(storage[stype]);
354                         return -ENOMEM;
355                 }
356 
357                 pl_was_allocated = true;
358                 pl->prog = prog;
359                 for_each_cgroup_storage_type(stype)
360                         pl->storage[stype] = storage[stype];
361                 list_add_tail(&pl->node, progs);
362         } else {
363                 if (list_empty(progs)) {
364                         pl = kmalloc(sizeof(*pl), GFP_KERNEL);
365                         if (!pl) {
366                                 for_each_cgroup_storage_type(stype)
367                                         bpf_cgroup_storage_free(storage[stype]);
368                                 return -ENOMEM;
369                         }
370                         pl_was_allocated = true;
371                         list_add_tail(&pl->node, progs);
372                 } else {
373                         pl = list_first_entry(progs, typeof(*pl), node);
374                         old_prog = pl->prog;
375                         for_each_cgroup_storage_type(stype) {
376                                 old_storage[stype] = pl->storage[stype];
377                                 bpf_cgroup_storage_unlink(old_storage[stype]);
378                         }
379                         pl_was_allocated = false;
380                 }
381                 pl->prog = prog;
382                 for_each_cgroup_storage_type(stype)
383                         pl->storage[stype] = storage[stype];
384         }
385 
386         cgrp->bpf.flags[type] = flags;
387 
388         err = update_effective_progs(cgrp, type);
389         if (err)
390                 goto cleanup;
391 
392         static_branch_inc(&cgroup_bpf_enabled_key);
393         for_each_cgroup_storage_type(stype) {
394                 if (!old_storage[stype])
395                         continue;
396                 bpf_cgroup_storage_free(old_storage[stype]);
397         }
398         if (old_prog) {
399                 bpf_prog_put(old_prog);
400                 static_branch_dec(&cgroup_bpf_enabled_key);
401         }
402         for_each_cgroup_storage_type(stype)
403                 bpf_cgroup_storage_link(storage[stype], cgrp, type);
404         return 0;
405 
406 cleanup:
407         /* and cleanup the prog list */
408         pl->prog = old_prog;
409         for_each_cgroup_storage_type(stype) {
410                 bpf_cgroup_storage_free(pl->storage[stype]);
411                 pl->storage[stype] = old_storage[stype];
412                 bpf_cgroup_storage_link(old_storage[stype], cgrp, type);
413         }
414         if (pl_was_allocated) {
415                 list_del(&pl->node);
416                 kfree(pl);
417         }
418         return err;
419 }
420 
421 /**
422  * __cgroup_bpf_detach() - Detach the program from a cgroup, and
423  *                         propagate the change to descendants
424  * @cgrp: The cgroup which descendants to traverse
425  * @prog: A program to detach or NULL
426  * @type: Type of detach operation
427  *
428  * Must be called with cgroup_mutex held.
429  */
430 int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
431                         enum bpf_attach_type type)
432 {
433         struct list_head *progs = &cgrp->bpf.progs[type];
434         enum bpf_cgroup_storage_type stype;
435         u32 flags = cgrp->bpf.flags[type];
436         struct bpf_prog *old_prog = NULL;
437         struct bpf_prog_list *pl;
438         int err;
439 
440         if (flags & BPF_F_ALLOW_MULTI) {
441                 if (!prog)
442                         /* to detach MULTI prog the user has to specify valid FD
443                          * of the program to be detached
444                          */
445                         return -EINVAL;
446         } else {
447                 if (list_empty(progs))
448                         /* report error when trying to detach and nothing is attached */
449                         return -ENOENT;
450         }
451 
452         if (flags & BPF_F_ALLOW_MULTI) {
453                 /* find the prog and detach it */
454                 list_for_each_entry(pl, progs, node) {
455                         if (pl->prog != prog)
456                                 continue;
457                         old_prog = prog;
458                         /* mark it deleted, so it's ignored while
459                          * recomputing effective
460                          */
461                         pl->prog = NULL;
462                         break;
463                 }
464                 if (!old_prog)
465                         return -ENOENT;
466         } else {
467                 /* to maintain backward compatibility NONE and OVERRIDE cgroups
468                  * allow detaching with invalid FD (prog==NULL)
469                  */
470                 pl = list_first_entry(progs, typeof(*pl), node);
471                 old_prog = pl->prog;
472                 pl->prog = NULL;
473         }
474 
475         err = update_effective_progs(cgrp, type);
476         if (err)
477                 goto cleanup;
478 
479         /* now can actually delete it from this cgroup list */
480         list_del(&pl->node);
481         for_each_cgroup_storage_type(stype) {
482                 bpf_cgroup_storage_unlink(pl->storage[stype]);
483                 bpf_cgroup_storage_free(pl->storage[stype]);
484         }
485         kfree(pl);
486         if (list_empty(progs))
487                 /* last program was detached, reset flags to zero */
488                 cgrp->bpf.flags[type] = 0;
489 
490         bpf_prog_put(old_prog);
491         static_branch_dec(&cgroup_bpf_enabled_key);
492         return 0;
493 
494 cleanup:
495         /* and restore back old_prog */
496         pl->prog = old_prog;
497         return err;
498 }
499 
500 /* Must be called with cgroup_mutex held to avoid races. */
501 int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
502                        union bpf_attr __user *uattr)
503 {
504         __u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids);
505         enum bpf_attach_type type = attr->query.attach_type;
506         struct list_head *progs = &cgrp->bpf.progs[type];
507         u32 flags = cgrp->bpf.flags[type];
508         struct bpf_prog_array *effective;
509         int cnt, ret = 0, i;
510 
511         effective = rcu_dereference_protected(cgrp->bpf.effective[type],
512                                               lockdep_is_held(&cgroup_mutex));
513 
514         if (attr->query.query_flags & BPF_F_QUERY_EFFECTIVE)
515                 cnt = bpf_prog_array_length(effective);
516         else
517                 cnt = prog_list_length(progs);
518 
519         if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags)))
520                 return -EFAULT;
521         if (copy_to_user(&uattr->query.prog_cnt, &cnt, sizeof(cnt)))
522                 return -EFAULT;
523         if (attr->query.prog_cnt == 0 || !prog_ids || !cnt)
524                 /* return early if user requested only program count + flags */
525                 return 0;
526         if (attr->query.prog_cnt < cnt) {
527                 cnt = attr->query.prog_cnt;
528                 ret = -ENOSPC;
529         }
530 
531         if (attr->query.query_flags & BPF_F_QUERY_EFFECTIVE) {
532                 return bpf_prog_array_copy_to_user(effective, prog_ids, cnt);
533         } else {
534                 struct bpf_prog_list *pl;
535                 u32 id;
536 
537                 i = 0;
538                 list_for_each_entry(pl, progs, node) {
539                         id = pl->prog->aux->id;
540                         if (copy_to_user(prog_ids + i, &id, sizeof(id)))
541                                 return -EFAULT;
542                         if (++i == cnt)
543                                 break;
544                 }
545         }
546         return ret;
547 }
548 
549 int cgroup_bpf_prog_attach(const union bpf_attr *attr,
550                            enum bpf_prog_type ptype, struct bpf_prog *prog)
551 {
552         struct cgroup *cgrp;
553         int ret;
554 
555         cgrp = cgroup_get_from_fd(attr->target_fd);
556         if (IS_ERR(cgrp))
557                 return PTR_ERR(cgrp);
558 
559         ret = cgroup_bpf_attach(cgrp, prog, attr->attach_type,
560                                 attr->attach_flags);
561         cgroup_put(cgrp);
562         return ret;
563 }
564 
565 int cgroup_bpf_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype)
566 {
567         struct bpf_prog *prog;
568         struct cgroup *cgrp;
569         int ret;
570 
571         cgrp = cgroup_get_from_fd(attr->target_fd);
572         if (IS_ERR(cgrp))
573                 return PTR_ERR(cgrp);
574 
575         prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
576         if (IS_ERR(prog))
577                 prog = NULL;
578 
579         ret = cgroup_bpf_detach(cgrp, prog, attr->attach_type, 0);
580         if (prog)
581                 bpf_prog_put(prog);
582 
583         cgroup_put(cgrp);
584         return ret;
585 }
586 
587 int cgroup_bpf_prog_query(const union bpf_attr *attr,
588                           union bpf_attr __user *uattr)
589 {
590         struct cgroup *cgrp;
591         int ret;
592 
593         cgrp = cgroup_get_from_fd(attr->query.target_fd);
594         if (IS_ERR(cgrp))
595                 return PTR_ERR(cgrp);
596 
597         ret = cgroup_bpf_query(cgrp, attr, uattr);
598 
599         cgroup_put(cgrp);
600         return ret;
601 }
602 
603 /**
604  * __cgroup_bpf_run_filter_skb() - Run a program for packet filtering
605  * @sk: The socket sending or receiving traffic
606  * @skb: The skb that is being sent or received
607  * @type: The type of program to be exectuted
608  *
609  * If no socket is passed, or the socket is not of type INET or INET6,
610  * this function does nothing and returns 0.
611  *
612  * The program type passed in via @type must be suitable for network
613  * filtering. No further check is performed to assert that.
614  *
615  * For egress packets, this function can return:
616  *   NET_XMIT_SUCCESS    (0)    - continue with packet output
617  *   NET_XMIT_DROP       (1)    - drop packet and notify TCP to call cwr
618  *   NET_XMIT_CN         (2)    - continue with packet output and notify TCP
619  *                                to call cwr
620  *   -EPERM                     - drop packet
621  *
622  * For ingress packets, this function will return -EPERM if any
623  * attached program was found and if it returned != 1 during execution.
624  * Otherwise 0 is returned.
625  */
626 int __cgroup_bpf_run_filter_skb(struct sock *sk,
627                                 struct sk_buff *skb,
628                                 enum bpf_attach_type type)
629 {
630         unsigned int offset = skb->data - skb_network_header(skb);
631         struct sock *save_sk;
632         void *saved_data_end;
633         struct cgroup *cgrp;
634         int ret;
635 
636         if (!sk || !sk_fullsock(sk))
637                 return 0;
638 
639         if (sk->sk_family != AF_INET && sk->sk_family != AF_INET6)
640                 return 0;
641 
642         cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
643         save_sk = skb->sk;
644         skb->sk = sk;
645         __skb_push(skb, offset);
646 
647         /* compute pointers for the bpf prog */
648         bpf_compute_and_save_data_end(skb, &saved_data_end);
649 
650         if (type == BPF_CGROUP_INET_EGRESS) {
651                 ret = BPF_PROG_CGROUP_INET_EGRESS_RUN_ARRAY(
652                         cgrp->bpf.effective[type], skb, __bpf_prog_run_save_cb);
653         } else {
654                 ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], skb,
655                                           __bpf_prog_run_save_cb);
656                 ret = (ret == 1 ? 0 : -EPERM);
657         }
658         bpf_restore_data_end(skb, saved_data_end);
659         __skb_pull(skb, offset);
660         skb->sk = save_sk;
661 
662         return ret;
663 }
664 EXPORT_SYMBOL(__cgroup_bpf_run_filter_skb);
665 
666 /**
667  * __cgroup_bpf_run_filter_sk() - Run a program on a sock
668  * @sk: sock structure to manipulate
669  * @type: The type of program to be exectuted
670  *
671  * socket is passed is expected to be of type INET or INET6.
672  *
673  * The program type passed in via @type must be suitable for sock
674  * filtering. No further check is performed to assert that.
675  *
676  * This function will return %-EPERM if any if an attached program was found
677  * and if it returned != 1 during execution. In all other cases, 0 is returned.
678  */
679 int __cgroup_bpf_run_filter_sk(struct sock *sk,
680                                enum bpf_attach_type type)
681 {
682         struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
683         int ret;
684 
685         ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], sk, BPF_PROG_RUN);
686         return ret == 1 ? 0 : -EPERM;
687 }
688 EXPORT_SYMBOL(__cgroup_bpf_run_filter_sk);
689 
690 /**
691  * __cgroup_bpf_run_filter_sock_addr() - Run a program on a sock and
692  *                                       provided by user sockaddr
693  * @sk: sock struct that will use sockaddr
694  * @uaddr: sockaddr struct provided by user
695  * @type: The type of program to be exectuted
696  * @t_ctx: Pointer to attach type specific context
697  *
698  * socket is expected to be of type INET or INET6.
699  *
700  * This function will return %-EPERM if an attached program is found and
701  * returned value != 1 during execution. In all other cases, 0 is returned.
702  */
703 int __cgroup_bpf_run_filter_sock_addr(struct sock *sk,
704                                       struct sockaddr *uaddr,
705                                       enum bpf_attach_type type,
706                                       void *t_ctx)
707 {
708         struct bpf_sock_addr_kern ctx = {
709                 .sk = sk,
710                 .uaddr = uaddr,
711                 .t_ctx = t_ctx,
712         };
713         struct sockaddr_storage unspec;
714         struct cgroup *cgrp;
715         int ret;
716 
717         /* Check socket family since not all sockets represent network
718          * endpoint (e.g. AF_UNIX).
719          */
720         if (sk->sk_family != AF_INET && sk->sk_family != AF_INET6)
721                 return 0;
722 
723         if (!ctx.uaddr) {
724                 memset(&unspec, 0, sizeof(unspec));
725                 ctx.uaddr = (struct sockaddr *)&unspec;
726         }
727 
728         cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
729         ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx, BPF_PROG_RUN);
730 
731         return ret == 1 ? 0 : -EPERM;
732 }
733 EXPORT_SYMBOL(__cgroup_bpf_run_filter_sock_addr);
734 
735 /**
736  * __cgroup_bpf_run_filter_sock_ops() - Run a program on a sock
737  * @sk: socket to get cgroup from
738  * @sock_ops: bpf_sock_ops_kern struct to pass to program. Contains
739  * sk with connection information (IP addresses, etc.) May not contain
740  * cgroup info if it is a req sock.
741  * @type: The type of program to be exectuted
742  *
743  * socket passed is expected to be of type INET or INET6.
744  *
745  * The program type passed in via @type must be suitable for sock_ops
746  * filtering. No further check is performed to assert that.
747  *
748  * This function will return %-EPERM if any if an attached program was found
749  * and if it returned != 1 during execution. In all other cases, 0 is returned.
750  */
751 int __cgroup_bpf_run_filter_sock_ops(struct sock *sk,
752                                      struct bpf_sock_ops_kern *sock_ops,
753                                      enum bpf_attach_type type)
754 {
755         struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
756         int ret;
757 
758         ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], sock_ops,
759                                  BPF_PROG_RUN);
760         return ret == 1 ? 0 : -EPERM;
761 }
762 EXPORT_SYMBOL(__cgroup_bpf_run_filter_sock_ops);
763 
764 int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor,
765                                       short access, enum bpf_attach_type type)
766 {
767         struct cgroup *cgrp;
768         struct bpf_cgroup_dev_ctx ctx = {
769                 .access_type = (access << 16) | dev_type,
770                 .major = major,
771                 .minor = minor,
772         };
773         int allow = 1;
774 
775         rcu_read_lock();
776         cgrp = task_dfl_cgroup(current);
777         allow = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx,
778                                    BPF_PROG_RUN);
779         rcu_read_unlock();
780 
781         return !allow;
782 }
783 EXPORT_SYMBOL(__cgroup_bpf_check_dev_permission);
784 
785 static const struct bpf_func_proto *
786 cgroup_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
787 {
788         switch (func_id) {
789         case BPF_FUNC_map_lookup_elem:
790                 return &bpf_map_lookup_elem_proto;
791         case BPF_FUNC_map_update_elem:
792                 return &bpf_map_update_elem_proto;
793         case BPF_FUNC_map_delete_elem:
794                 return &bpf_map_delete_elem_proto;
795         case BPF_FUNC_map_push_elem:
796                 return &bpf_map_push_elem_proto;
797         case BPF_FUNC_map_pop_elem:
798                 return &bpf_map_pop_elem_proto;
799         case BPF_FUNC_map_peek_elem:
800                 return &bpf_map_peek_elem_proto;
801         case BPF_FUNC_get_current_uid_gid:
802                 return &bpf_get_current_uid_gid_proto;
803         case BPF_FUNC_get_local_storage:
804                 return &bpf_get_local_storage_proto;
805         case BPF_FUNC_get_current_cgroup_id:
806                 return &bpf_get_current_cgroup_id_proto;
807         case BPF_FUNC_trace_printk:
808                 if (capable(CAP_SYS_ADMIN))
809                         return bpf_get_trace_printk_proto();
810                 /* fall through */
811         default:
812                 return NULL;
813         }
814 }
815 
816 static const struct bpf_func_proto *
817 cgroup_dev_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
818 {
819         return cgroup_base_func_proto(func_id, prog);
820 }
821 
822 static bool cgroup_dev_is_valid_access(int off, int size,
823                                        enum bpf_access_type type,
824                                        const struct bpf_prog *prog,
825                                        struct bpf_insn_access_aux *info)
826 {
827         const int size_default = sizeof(__u32);
828 
829         if (type == BPF_WRITE)
830                 return false;
831 
832         if (off < 0 || off + size > sizeof(struct bpf_cgroup_dev_ctx))
833                 return false;
834         /* The verifier guarantees that size > 0. */
835         if (off % size != 0)
836                 return false;
837 
838         switch (off) {
839         case bpf_ctx_range(struct bpf_cgroup_dev_ctx, access_type):
840                 bpf_ctx_record_field_size(info, size_default);
841                 if (!bpf_ctx_narrow_access_ok(off, size, size_default))
842                         return false;
843                 break;
844         default:
845                 if (size != size_default)
846                         return false;
847         }
848 
849         return true;
850 }
851 
852 const struct bpf_prog_ops cg_dev_prog_ops = {
853 };
854 
855 const struct bpf_verifier_ops cg_dev_verifier_ops = {
856         .get_func_proto         = cgroup_dev_func_proto,
857         .is_valid_access        = cgroup_dev_is_valid_access,
858 };
859 
860 /**
861  * __cgroup_bpf_run_filter_sysctl - Run a program on sysctl
862  *
863  * @head: sysctl table header
864  * @table: sysctl table
865  * @write: sysctl is being read (= 0) or written (= 1)
866  * @buf: pointer to buffer passed by user space
867  * @pcount: value-result argument: value is size of buffer pointed to by @buf,
868  *      result is size of @new_buf if program set new value, initial value
869  *      otherwise
870  * @ppos: value-result argument: value is position at which read from or write
871  *      to sysctl is happening, result is new position if program overrode it,
872  *      initial value otherwise
873  * @new_buf: pointer to pointer to new buffer that will be allocated if program
874  *      overrides new value provided by user space on sysctl write
875  *      NOTE: it's caller responsibility to free *new_buf if it was set
876  * @type: type of program to be executed
877  *
878  * Program is run when sysctl is being accessed, either read or written, and
879  * can allow or deny such access.
880  *
881  * This function will return %-EPERM if an attached program is found and
882  * returned value != 1 during execution. In all other cases 0 is returned.
883  */
884 int __cgroup_bpf_run_filter_sysctl(struct ctl_table_header *head,
885                                    struct ctl_table *table, int write,
886                                    void __user *buf, size_t *pcount,
887                                    loff_t *ppos, void **new_buf,
888                                    enum bpf_attach_type type)
889 {
890         struct bpf_sysctl_kern ctx = {
891                 .head = head,
892                 .table = table,
893                 .write = write,
894                 .ppos = ppos,
895                 .cur_val = NULL,
896                 .cur_len = PAGE_SIZE,
897                 .new_val = NULL,
898                 .new_len = 0,
899                 .new_updated = 0,
900         };
901         struct cgroup *cgrp;
902         int ret;
903 
904         ctx.cur_val = kmalloc_track_caller(ctx.cur_len, GFP_KERNEL);
905         if (ctx.cur_val) {
906                 mm_segment_t old_fs;
907                 loff_t pos = 0;
908 
909                 old_fs = get_fs();
910                 set_fs(KERNEL_DS);
911                 if (table->proc_handler(table, 0, (void __user *)ctx.cur_val,
912                                         &ctx.cur_len, &pos)) {
913                         /* Let BPF program decide how to proceed. */
914                         ctx.cur_len = 0;
915                 }
916                 set_fs(old_fs);
917         } else {
918                 /* Let BPF program decide how to proceed. */
919                 ctx.cur_len = 0;
920         }
921 
922         if (write && buf && *pcount) {
923                 /* BPF program should be able to override new value with a
924                  * buffer bigger than provided by user.
925                  */
926                 ctx.new_val = kmalloc_track_caller(PAGE_SIZE, GFP_KERNEL);
927                 ctx.new_len = min_t(size_t, PAGE_SIZE, *pcount);
928                 if (!ctx.new_val ||
929                     copy_from_user(ctx.new_val, buf, ctx.new_len))
930                         /* Let BPF program decide how to proceed. */
931                         ctx.new_len = 0;
932         }
933 
934         rcu_read_lock();
935         cgrp = task_dfl_cgroup(current);
936         ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx, BPF_PROG_RUN);
937         rcu_read_unlock();
938 
939         kfree(ctx.cur_val);
940 
941         if (ret == 1 && ctx.new_updated) {
942                 *new_buf = ctx.new_val;
943                 *pcount = ctx.new_len;
944         } else {
945                 kfree(ctx.new_val);
946         }
947 
948         return ret == 1 ? 0 : -EPERM;
949 }
950 EXPORT_SYMBOL(__cgroup_bpf_run_filter_sysctl);
951 
952 #ifdef CONFIG_NET
953 static bool __cgroup_bpf_prog_array_is_empty(struct cgroup *cgrp,
954                                              enum bpf_attach_type attach_type)
955 {
956         struct bpf_prog_array *prog_array;
957         bool empty;
958 
959         rcu_read_lock();
960         prog_array = rcu_dereference(cgrp->bpf.effective[attach_type]);
961         empty = bpf_prog_array_is_empty(prog_array);
962         rcu_read_unlock();
963 
964         return empty;
965 }
966 
967 static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen)
968 {
969         if (unlikely(max_optlen > PAGE_SIZE) || max_optlen < 0)
970                 return -EINVAL;
971 
972         ctx->optval = kzalloc(max_optlen, GFP_USER);
973         if (!ctx->optval)
974                 return -ENOMEM;
975 
976         ctx->optval_end = ctx->optval + max_optlen;
977 
978         return 0;
979 }
980 
981 static void sockopt_free_buf(struct bpf_sockopt_kern *ctx)
982 {
983         kfree(ctx->optval);
984 }
985 
986 int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level,
987                                        int *optname, char __user *optval,
988                                        int *optlen, char **kernel_optval)
989 {
990         struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
991         struct bpf_sockopt_kern ctx = {
992                 .sk = sk,
993                 .level = *level,
994                 .optname = *optname,
995         };
996         int ret, max_optlen;
997 
998         /* Opportunistic check to see whether we have any BPF program
999          * attached to the hook so we don't waste time allocating
1000          * memory and locking the socket.
1001          */
1002         if (!cgroup_bpf_enabled ||
1003             __cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_SETSOCKOPT))
1004                 return 0;
1005 
1006         /* Allocate a bit more than the initial user buffer for
1007          * BPF program. The canonical use case is overriding
1008          * TCP_CONGESTION(nv) to TCP_CONGESTION(cubic).
1009          */
1010         max_optlen = max_t(int, 16, *optlen);
1011 
1012         ret = sockopt_alloc_buf(&ctx, max_optlen);
1013         if (ret)
1014                 return ret;
1015 
1016         ctx.optlen = *optlen;
1017 
1018         if (copy_from_user(ctx.optval, optval, *optlen) != 0) {
1019                 ret = -EFAULT;
1020                 goto out;
1021         }
1022 
1023         lock_sock(sk);
1024         ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[BPF_CGROUP_SETSOCKOPT],
1025                                  &ctx, BPF_PROG_RUN);
1026         release_sock(sk);
1027 
1028         if (!ret) {
1029                 ret = -EPERM;
1030                 goto out;
1031         }
1032 
1033         if (ctx.optlen == -1) {
1034                 /* optlen set to -1, bypass kernel */
1035                 ret = 1;
1036         } else if (ctx.optlen > max_optlen || ctx.optlen < -1) {
1037                 /* optlen is out of bounds */
1038                 ret = -EFAULT;
1039         } else {
1040                 /* optlen within bounds, run kernel handler */
1041                 ret = 0;
1042 
1043                 /* export any potential modifications */
1044                 *level = ctx.level;
1045                 *optname = ctx.optname;
1046                 *optlen = ctx.optlen;
1047                 *kernel_optval = ctx.optval;
1048         }
1049 
1050 out:
1051         if (ret)
1052                 sockopt_free_buf(&ctx);
1053         return ret;
1054 }
1055 EXPORT_SYMBOL(__cgroup_bpf_run_filter_setsockopt);
1056 
1057 int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level,
1058                                        int optname, char __user *optval,
1059                                        int __user *optlen, int max_optlen,
1060                                        int retval)
1061 {
1062         struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
1063         struct bpf_sockopt_kern ctx = {
1064                 .sk = sk,
1065                 .level = level,
1066                 .optname = optname,
1067                 .retval = retval,
1068         };
1069         int ret;
1070 
1071         /* Opportunistic check to see whether we have any BPF program
1072          * attached to the hook so we don't waste time allocating
1073          * memory and locking the socket.
1074          */
1075         if (!cgroup_bpf_enabled ||
1076             __cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_GETSOCKOPT))
1077                 return retval;
1078 
1079         ret = sockopt_alloc_buf(&ctx, max_optlen);
1080         if (ret)
1081                 return ret;
1082 
1083         ctx.optlen = max_optlen;
1084 
1085         if (!retval) {
1086                 /* If kernel getsockopt finished successfully,
1087                  * copy whatever was returned to the user back
1088                  * into our temporary buffer. Set optlen to the
1089                  * one that kernel returned as well to let
1090                  * BPF programs inspect the value.
1091                  */
1092 
1093                 if (get_user(ctx.optlen, optlen)) {
1094                         ret = -EFAULT;
1095                         goto out;
1096                 }
1097 
1098                 if (ctx.optlen > max_optlen)
1099                         ctx.optlen = max_optlen;
1100 
1101                 if (copy_from_user(ctx.optval, optval, ctx.optlen) != 0) {
1102                         ret = -EFAULT;
1103                         goto out;
1104                 }
1105         }
1106 
1107         lock_sock(sk);
1108         ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[BPF_CGROUP_GETSOCKOPT],
1109                                  &ctx, BPF_PROG_RUN);
1110         release_sock(sk);
1111 
1112         if (!ret) {
1113                 ret = -EPERM;
1114                 goto out;
1115         }
1116 
1117         if (ctx.optlen > max_optlen) {
1118                 ret = -EFAULT;
1119                 goto out;
1120         }
1121 
1122         /* BPF programs only allowed to set retval to 0, not some
1123          * arbitrary value.
1124          */
1125         if (ctx.retval != 0 && ctx.retval != retval) {
1126                 ret = -EFAULT;
1127                 goto out;
1128         }
1129 
1130         if (copy_to_user(optval, ctx.optval, ctx.optlen) ||
1131             put_user(ctx.optlen, optlen)) {
1132                 ret = -EFAULT;
1133                 goto out;
1134         }
1135 
1136         ret = ctx.retval;
1137 
1138 out:
1139         sockopt_free_buf(&ctx);
1140         return ret;
1141 }
1142 EXPORT_SYMBOL(__cgroup_bpf_run_filter_getsockopt);
1143 #endif
1144 
1145 static ssize_t sysctl_cpy_dir(const struct ctl_dir *dir, char **bufp,
1146                               size_t *lenp)
1147 {
1148         ssize_t tmp_ret = 0, ret;
1149 
1150         if (dir->header.parent) {
1151                 tmp_ret = sysctl_cpy_dir(dir->header.parent, bufp, lenp);
1152                 if (tmp_ret < 0)
1153                         return tmp_ret;
1154         }
1155 
1156         ret = strscpy(*bufp, dir->header.ctl_table[0].procname, *lenp);
1157         if (ret < 0)
1158                 return ret;
1159         *bufp += ret;
1160         *lenp -= ret;
1161         ret += tmp_ret;
1162 
1163         /* Avoid leading slash. */
1164         if (!ret)
1165                 return ret;
1166 
1167         tmp_ret = strscpy(*bufp, "/", *lenp);
1168         if (tmp_ret < 0)
1169                 return tmp_ret;
1170         *bufp += tmp_ret;
1171         *lenp -= tmp_ret;
1172 
1173         return ret + tmp_ret;
1174 }
1175 
1176 BPF_CALL_4(bpf_sysctl_get_name, struct bpf_sysctl_kern *, ctx, char *, buf,
1177            size_t, buf_len, u64, flags)
1178 {
1179         ssize_t tmp_ret = 0, ret;
1180 
1181         if (!buf)
1182                 return -EINVAL;
1183 
1184         if (!(flags & BPF_F_SYSCTL_BASE_NAME)) {
1185                 if (!ctx->head)
1186                         return -EINVAL;
1187                 tmp_ret = sysctl_cpy_dir(ctx->head->parent, &buf, &buf_len);
1188                 if (tmp_ret < 0)
1189                         return tmp_ret;
1190         }
1191 
1192         ret = strscpy(buf, ctx->table->procname, buf_len);
1193 
1194         return ret < 0 ? ret : tmp_ret + ret;
1195 }
1196 
1197 static const struct bpf_func_proto bpf_sysctl_get_name_proto = {
1198         .func           = bpf_sysctl_get_name,
1199         .gpl_only       = false,
1200         .ret_type       = RET_INTEGER,
1201         .arg1_type      = ARG_PTR_TO_CTX,
1202         .arg2_type      = ARG_PTR_TO_MEM,
1203         .arg3_type      = ARG_CONST_SIZE,
1204         .arg4_type      = ARG_ANYTHING,
1205 };
1206 
1207 static int copy_sysctl_value(char *dst, size_t dst_len, char *src,
1208                              size_t src_len)
1209 {
1210         if (!dst)
1211                 return -EINVAL;
1212 
1213         if (!dst_len)
1214                 return -E2BIG;
1215 
1216         if (!src || !src_len) {
1217                 memset(dst, 0, dst_len);
1218                 return -EINVAL;
1219         }
1220 
1221         memcpy(dst, src, min(dst_len, src_len));
1222 
1223         if (dst_len > src_len) {
1224                 memset(dst + src_len, '\0', dst_len - src_len);
1225                 return src_len;
1226         }
1227 
1228         dst[dst_len - 1] = '\0';
1229 
1230         return -E2BIG;
1231 }
1232 
1233 BPF_CALL_3(bpf_sysctl_get_current_value, struct bpf_sysctl_kern *, ctx,
1234            char *, buf, size_t, buf_len)
1235 {
1236         return copy_sysctl_value(buf, buf_len, ctx->cur_val, ctx->cur_len);
1237 }
1238 
1239 static const struct bpf_func_proto bpf_sysctl_get_current_value_proto = {
1240         .func           = bpf_sysctl_get_current_value,
1241         .gpl_only       = false,
1242         .ret_type       = RET_INTEGER,
1243         .arg1_type      = ARG_PTR_TO_CTX,
1244         .arg2_type      = ARG_PTR_TO_UNINIT_MEM,
1245         .arg3_type      = ARG_CONST_SIZE,
1246 };
1247 
1248 BPF_CALL_3(bpf_sysctl_get_new_value, struct bpf_sysctl_kern *, ctx, char *, buf,
1249            size_t, buf_len)
1250 {
1251         if (!ctx->write) {
1252                 if (buf && buf_len)
1253                         memset(buf, '\0', buf_len);
1254                 return -EINVAL;
1255         }
1256         return copy_sysctl_value(buf, buf_len, ctx->new_val, ctx->new_len);
1257 }
1258 
1259 static const struct bpf_func_proto bpf_sysctl_get_new_value_proto = {
1260         .func           = bpf_sysctl_get_new_value,
1261         .gpl_only       = false,
1262         .ret_type       = RET_INTEGER,
1263         .arg1_type      = ARG_PTR_TO_CTX,
1264         .arg2_type      = ARG_PTR_TO_UNINIT_MEM,
1265         .arg3_type      = ARG_CONST_SIZE,
1266 };
1267 
1268 BPF_CALL_3(bpf_sysctl_set_new_value, struct bpf_sysctl_kern *, ctx,
1269            const char *, buf, size_t, buf_len)
1270 {
1271         if (!ctx->write || !ctx->new_val || !ctx->new_len || !buf || !buf_len)
1272                 return -EINVAL;
1273 
1274         if (buf_len > PAGE_SIZE - 1)
1275                 return -E2BIG;
1276 
1277         memcpy(ctx->new_val, buf, buf_len);
1278         ctx->new_len = buf_len;
1279         ctx->new_updated = 1;
1280 
1281         return 0;
1282 }
1283 
1284 static const struct bpf_func_proto bpf_sysctl_set_new_value_proto = {
1285         .func           = bpf_sysctl_set_new_value,
1286         .gpl_only       = false,
1287         .ret_type       = RET_INTEGER,
1288         .arg1_type      = ARG_PTR_TO_CTX,
1289         .arg2_type      = ARG_PTR_TO_MEM,
1290         .arg3_type      = ARG_CONST_SIZE,
1291 };
1292 
1293 static const struct bpf_func_proto *
1294 sysctl_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1295 {
1296         switch (func_id) {
1297         case BPF_FUNC_strtol:
1298                 return &bpf_strtol_proto;
1299         case BPF_FUNC_strtoul:
1300                 return &bpf_strtoul_proto;
1301         case BPF_FUNC_sysctl_get_name:
1302                 return &bpf_sysctl_get_name_proto;
1303         case BPF_FUNC_sysctl_get_current_value:
1304                 return &bpf_sysctl_get_current_value_proto;
1305         case BPF_FUNC_sysctl_get_new_value:
1306                 return &bpf_sysctl_get_new_value_proto;
1307         case BPF_FUNC_sysctl_set_new_value:
1308                 return &bpf_sysctl_set_new_value_proto;
1309         default:
1310                 return cgroup_base_func_proto(func_id, prog);
1311         }
1312 }
1313 
1314 static bool sysctl_is_valid_access(int off, int size, enum bpf_access_type type,
1315                                    const struct bpf_prog *prog,
1316                                    struct bpf_insn_access_aux *info)
1317 {
1318         const int size_default = sizeof(__u32);
1319 
1320         if (off < 0 || off + size > sizeof(struct bpf_sysctl) || off % size)
1321                 return false;
1322 
1323         switch (off) {
1324         case bpf_ctx_range(struct bpf_sysctl, write):
1325                 if (type != BPF_READ)
1326                         return false;
1327                 bpf_ctx_record_field_size(info, size_default);
1328                 return bpf_ctx_narrow_access_ok(off, size, size_default);
1329         case bpf_ctx_range(struct bpf_sysctl, file_pos):
1330                 if (type == BPF_READ) {
1331                         bpf_ctx_record_field_size(info, size_default);
1332                         return bpf_ctx_narrow_access_ok(off, size, size_default);
1333                 } else {
1334                         return size == size_default;
1335                 }
1336         default:
1337                 return false;
1338         }
1339 }
1340 
1341 static u32 sysctl_convert_ctx_access(enum bpf_access_type type,
1342                                      const struct bpf_insn *si,
1343                                      struct bpf_insn *insn_buf,
1344                                      struct bpf_prog *prog, u32 *target_size)
1345 {
1346         struct bpf_insn *insn = insn_buf;
1347         u32 read_size;
1348 
1349         switch (si->off) {
1350         case offsetof(struct bpf_sysctl, write):
1351                 *insn++ = BPF_LDX_MEM(
1352                         BPF_SIZE(si->code), si->dst_reg, si->src_reg,
1353                         bpf_target_off(struct bpf_sysctl_kern, write,
1354                                        sizeof_field(struct bpf_sysctl_kern,
1355                                                     write),
1356                                        target_size));
1357                 break;
1358         case offsetof(struct bpf_sysctl, file_pos):
1359                 /* ppos is a pointer so it should be accessed via indirect
1360                  * loads and stores. Also for stores additional temporary
1361                  * register is used since neither src_reg nor dst_reg can be
1362                  * overridden.
1363                  */
1364                 if (type == BPF_WRITE) {
1365                         int treg = BPF_REG_9;
1366 
1367                         if (si->src_reg == treg || si->dst_reg == treg)
1368                                 --treg;
1369                         if (si->src_reg == treg || si->dst_reg == treg)
1370                                 --treg;
1371                         *insn++ = BPF_STX_MEM(
1372                                 BPF_DW, si->dst_reg, treg,
1373                                 offsetof(struct bpf_sysctl_kern, tmp_reg));
1374                         *insn++ = BPF_LDX_MEM(
1375                                 BPF_FIELD_SIZEOF(struct bpf_sysctl_kern, ppos),
1376                                 treg, si->dst_reg,
1377                                 offsetof(struct bpf_sysctl_kern, ppos));
1378                         *insn++ = BPF_STX_MEM(
1379                                 BPF_SIZEOF(u32), treg, si->src_reg,
1380                                 bpf_ctx_narrow_access_offset(
1381                                         0, sizeof(u32), sizeof(loff_t)));
1382                         *insn++ = BPF_LDX_MEM(
1383                                 BPF_DW, treg, si->dst_reg,
1384                                 offsetof(struct bpf_sysctl_kern, tmp_reg));
1385                 } else {
1386                         *insn++ = BPF_LDX_MEM(
1387                                 BPF_FIELD_SIZEOF(struct bpf_sysctl_kern, ppos),
1388                                 si->dst_reg, si->src_reg,
1389                                 offsetof(struct bpf_sysctl_kern, ppos));
1390                         read_size = bpf_size_to_bytes(BPF_SIZE(si->code));
1391                         *insn++ = BPF_LDX_MEM(
1392                                 BPF_SIZE(si->code), si->dst_reg, si->dst_reg,
1393                                 bpf_ctx_narrow_access_offset(
1394                                         0, read_size, sizeof(loff_t)));
1395                 }
1396                 *target_size = sizeof(u32);
1397                 break;
1398         }
1399 
1400         return insn - insn_buf;
1401 }
1402 
1403 const struct bpf_verifier_ops cg_sysctl_verifier_ops = {
1404         .get_func_proto         = sysctl_func_proto,
1405         .is_valid_access        = sysctl_is_valid_access,
1406         .convert_ctx_access     = sysctl_convert_ctx_access,
1407 };
1408 
1409 const struct bpf_prog_ops cg_sysctl_prog_ops = {
1410 };
1411 
1412 static const struct bpf_func_proto *
1413 cg_sockopt_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1414 {
1415         switch (func_id) {
1416 #ifdef CONFIG_NET
1417         case BPF_FUNC_sk_storage_get:
1418                 return &bpf_sk_storage_get_proto;
1419         case BPF_FUNC_sk_storage_delete:
1420                 return &bpf_sk_storage_delete_proto;
1421 #endif
1422 #ifdef CONFIG_INET
1423         case BPF_FUNC_tcp_sock:
1424                 return &bpf_tcp_sock_proto;
1425 #endif
1426         default:
1427                 return cgroup_base_func_proto(func_id, prog);
1428         }
1429 }
1430 
1431 static bool cg_sockopt_is_valid_access(int off, int size,
1432                                        enum bpf_access_type type,
1433                                        const struct bpf_prog *prog,
1434                                        struct bpf_insn_access_aux *info)
1435 {
1436         const int size_default = sizeof(__u32);
1437 
1438         if (off < 0 || off >= sizeof(struct bpf_sockopt))
1439                 return false;
1440 
1441         if (off % size != 0)
1442                 return false;
1443 
1444         if (type == BPF_WRITE) {
1445                 switch (off) {
1446                 case offsetof(struct bpf_sockopt, retval):
1447                         if (size != size_default)
1448                                 return false;
1449                         return prog->expected_attach_type ==
1450                                 BPF_CGROUP_GETSOCKOPT;
1451                 case offsetof(struct bpf_sockopt, optname):
1452                         /* fallthrough */
1453                 case offsetof(struct bpf_sockopt, level):
1454                         if (size != size_default)
1455                                 return false;
1456                         return prog->expected_attach_type ==
1457                                 BPF_CGROUP_SETSOCKOPT;
1458                 case offsetof(struct bpf_sockopt, optlen):
1459                         return size == size_default;
1460                 default:
1461                         return false;
1462                 }
1463         }
1464 
1465         switch (off) {
1466         case offsetof(struct bpf_sockopt, sk):
1467                 if (size != sizeof(__u64))
1468                         return false;
1469                 info->reg_type = PTR_TO_SOCKET;
1470                 break;
1471         case offsetof(struct bpf_sockopt, optval):
1472                 if (size != sizeof(__u64))
1473                         return false;
1474                 info->reg_type = PTR_TO_PACKET;
1475                 break;
1476         case offsetof(struct bpf_sockopt, optval_end):
1477                 if (size != sizeof(__u64))
1478                         return false;
1479                 info->reg_type = PTR_TO_PACKET_END;
1480                 break;
1481         case offsetof(struct bpf_sockopt, retval):
1482                 if (size != size_default)
1483                         return false;
1484                 return prog->expected_attach_type == BPF_CGROUP_GETSOCKOPT;
1485         default:
1486                 if (size != size_default)
1487                         return false;
1488                 break;
1489         }
1490         return true;
1491 }
1492 
1493 #define CG_SOCKOPT_ACCESS_FIELD(T, F)                                   \
1494         T(BPF_FIELD_SIZEOF(struct bpf_sockopt_kern, F),                 \
1495           si->dst_reg, si->src_reg,                                     \
1496           offsetof(struct bpf_sockopt_kern, F))
1497 
1498 static u32 cg_sockopt_convert_ctx_access(enum bpf_access_type type,
1499                                          const struct bpf_insn *si,
1500                                          struct bpf_insn *insn_buf,
1501                                          struct bpf_prog *prog,
1502                                          u32 *target_size)
1503 {
1504         struct bpf_insn *insn = insn_buf;
1505 
1506         switch (si->off) {
1507         case offsetof(struct bpf_sockopt, sk):
1508                 *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, sk);
1509                 break;
1510         case offsetof(struct bpf_sockopt, level):
1511                 if (type == BPF_WRITE)
1512                         *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, level);
1513                 else
1514                         *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, level);
1515                 break;
1516         case offsetof(struct bpf_sockopt, optname):
1517                 if (type == BPF_WRITE)
1518                         *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, optname);
1519                 else
1520                         *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optname);
1521                 break;
1522         case offsetof(struct bpf_sockopt, optlen):
1523                 if (type == BPF_WRITE)
1524                         *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, optlen);
1525                 else
1526                         *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optlen);
1527                 break;
1528         case offsetof(struct bpf_sockopt, retval):
1529                 if (type == BPF_WRITE)
1530                         *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, retval);
1531                 else
1532                         *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, retval);
1533                 break;
1534         case offsetof(struct bpf_sockopt, optval):
1535                 *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optval);
1536                 break;
1537         case offsetof(struct bpf_sockopt, optval_end):
1538                 *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optval_end);
1539                 break;
1540         }
1541 
1542         return insn - insn_buf;
1543 }
1544 
1545 static int cg_sockopt_get_prologue(struct bpf_insn *insn_buf,
1546                                    bool direct_write,
1547                                    const struct bpf_prog *prog)
1548 {
1549         /* Nothing to do for sockopt argument. The data is kzalloc'ated.
1550          */
1551         return 0;
1552 }
1553 
1554 const struct bpf_verifier_ops cg_sockopt_verifier_ops = {
1555         .get_func_proto         = cg_sockopt_func_proto,
1556         .is_valid_access        = cg_sockopt_is_valid_access,
1557         .convert_ctx_access     = cg_sockopt_convert_ctx_access,
1558         .gen_prologue           = cg_sockopt_get_prologue,
1559 };
1560 
1561 const struct bpf_prog_ops cg_sockopt_prog_ops = {
1562 };
1563 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp