1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 Facebook */
3 #include <linux/rculist.h>
4 #include <linux/list.h>
5 #include <linux/hash.h>
6 #include <linux/types.h>
7 #include <linux/spinlock.h>
8 #include <linux/bpf.h>
9 #include <linux/btf.h>
10 #include <linux/btf_ids.h>
11 #include <linux/bpf_local_storage.h>
12 #include <net/bpf_sk_storage.h>
13 #include <net/sock.h>
14 #include <uapi/linux/sock_diag.h>
15 #include <uapi/linux/btf.h>
16 #include <linux/rcupdate_trace.h>
17
18 DEFINE_BPF_STORAGE_CACHE(sk_cache);
19
20 static struct bpf_local_storage_data *
21 bpf_sk_storage_lookup(struct sock *sk, struct bpf_map *map, bool cacheit_lockit)
22 {
23 struct bpf_local_storage *sk_storage;
24 struct bpf_local_storage_map *smap;
25
26 sk_storage =
27 rcu_dereference_check(sk->sk_bpf_storage, bpf_rcu_lock_held());
28 if (!sk_storage)
29 return NULL;
30
31 smap = (struct bpf_local_storage_map *)map;
32 return bpf_local_storage_lookup(sk_storage, smap, cacheit_lockit);
33 }
34
35 static int bpf_sk_storage_del(struct sock *sk, struct bpf_map *map)
36 {
37 struct bpf_local_storage_data *sdata;
38
39 sdata = bpf_sk_storage_lookup(sk, map, false);
40 if (!sdata)
41 return -ENOENT;
42
43 bpf_selem_unlink(SELEM(sdata), true);
44
45 return 0;
46 }
47
48 /* Called by __sk_destruct() & bpf_sk_storage_clone() */
49 void bpf_sk_storage_free(struct sock *sk)
50 {
51 struct bpf_local_storage_elem *selem;
52 struct bpf_local_storage *sk_storage;
53 bool free_sk_storage = false;
54 struct hlist_node *n;
55
56 rcu_read_lock();
57 sk_storage = rcu_dereference(sk->sk_bpf_storage);
58 if (!sk_storage) {
59 rcu_read_unlock();
60 return;
61 }
62
63 /* Netiher the bpf_prog nor the bpf-map's syscall
64 * could be modifying the sk_storage->list now.
65 * Thus, no elem can be added-to or deleted-from the
66 * sk_storage->list by the bpf_prog or by the bpf-map's syscall.
67 *
68 * It is racing with bpf_local_storage_map_free() alone
69 * when unlinking elem from the sk_storage->list and
70 * the map's bucket->list.
71 */
72 raw_spin_lock_bh(&sk_storage->lock);
73 hlist_for_each_entry_safe(selem, n, &sk_storage->list, snode) {
74 /* Always unlink from map before unlinking from
75 * sk_storage.
76 */
77 bpf_selem_unlink_map(selem);
78 free_sk_storage = bpf_selem_unlink_storage_nolock(
79 sk_storage, selem, true, false);
80 }
81 raw_spin_unlock_bh(&sk_storage->lock);
82 rcu_read_unlock();
83
84 if (free_sk_storage)
85 kfree_rcu(sk_storage, rcu);
86 }
87
88 static void bpf_sk_storage_map_free(struct bpf_map *map)
89 {
90 struct bpf_local_storage_map *smap;
91
92 smap = (struct bpf_local_storage_map *)map;
93 bpf_local_storage_cache_idx_free(&sk_cache, smap->cache_idx);
94 bpf_local_storage_map_free(smap, NULL);
95 }
96
97 static struct bpf_map *bpf_sk_storage_map_alloc(union bpf_attr *attr)
98 {
99 struct bpf_local_storage_map *smap;
100
101 smap = bpf_local_storage_map_alloc(attr);
102 if (IS_ERR(smap))
103 return ERR_CAST(smap);
104
105 smap->cache_idx = bpf_local_storage_cache_idx_get(&sk_cache);
106 return &smap->map;
107 }
108
109 static int notsupp_get_next_key(struct bpf_map *map, void *key,
110 void *next_key)
111 {
112 return -ENOTSUPP;
113 }
114
115 static void *bpf_fd_sk_storage_lookup_elem(struct bpf_map *map, void *key)
116 {
117 struct bpf_local_storage_data *sdata;
118 struct socket *sock;
119 int fd, err;
120
121 fd = *(int *)key;
122 sock = sockfd_lookup(fd, &err);
123 if (sock) {
124 sdata = bpf_sk_storage_lookup(sock->sk, map, true);
125 sockfd_put(sock);
126 return sdata ? sdata->data : NULL;
127 }
128
129 return ERR_PTR(err);
130 }
131
132 static int bpf_fd_sk_storage_update_elem(struct bpf_map *map, void *key,
133 void *value, u64 map_flags)
134 {
135 struct bpf_local_storage_data *sdata;
136 struct socket *sock;
137 int fd, err;
138
139 fd = *(int *)key;
140 sock = sockfd_lookup(fd, &err);
141 if (sock) {
142 sdata = bpf_local_storage_update(
143 sock->sk, (struct bpf_local_storage_map *)map, value,
144 map_flags, GFP_ATOMIC);
145 sockfd_put(sock);
146 return PTR_ERR_OR_ZERO(sdata);
147 }
148
149 return err;
150 }
151
152 static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key)
153 {
154 struct socket *sock;
155 int fd, err;
156
157 fd = *(int *)key;
158 sock = sockfd_lookup(fd, &err);
159 if (sock) {
160 err = bpf_sk_storage_del(sock->sk, map);
161 sockfd_put(sock);
162 return err;
163 }
164
165 return err;
166 }
167
168 static struct bpf_local_storage_elem *
169 bpf_sk_storage_clone_elem(struct sock *newsk,
170 struct bpf_local_storage_map *smap,
171 struct bpf_local_storage_elem *selem)
172 {
173 struct bpf_local_storage_elem *copy_selem;
174
175 copy_selem = bpf_selem_alloc(smap, newsk, NULL, true, GFP_ATOMIC);
176 if (!copy_selem)
177 return NULL;
178
179 if (map_value_has_spin_lock(&smap->map))
180 copy_map_value_locked(&smap->map, SDATA(copy_selem)->data,
181 SDATA(selem)->data, true);
182 else
183 copy_map_value(&smap->map, SDATA(copy_selem)->data,
184 SDATA(selem)->data);
185
186 return copy_selem;
187 }
188
189 int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk)
190 {
191 struct bpf_local_storage *new_sk_storage = NULL;
192 struct bpf_local_storage *sk_storage;
193 struct bpf_local_storage_elem *selem;
194 int ret = 0;
195
196 RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL);
197
198 rcu_read_lock();
199 sk_storage = rcu_dereference(sk->sk_bpf_storage);
200
201 if (!sk_storage || hlist_empty(&sk_storage->list))
202 goto out;
203
204 hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
205 struct bpf_local_storage_elem *copy_selem;
206 struct bpf_local_storage_map *smap;
207 struct bpf_map *map;
208
209 smap = rcu_dereference(SDATA(selem)->smap);
210 if (!(smap->map.map_flags & BPF_F_CLONE))
211 continue;
212
213 /* Note that for lockless listeners adding new element
214 * here can race with cleanup in bpf_local_storage_map_free.
215 * Try to grab map refcnt to make sure that it's still
216 * alive and prevent concurrent removal.
217 */
218 map = bpf_map_inc_not_zero(&smap->map);
219 if (IS_ERR(map))
220 continue;
221
222 copy_selem = bpf_sk_storage_clone_elem(newsk, smap, selem);
223 if (!copy_selem) {
224 ret = -ENOMEM;
225 bpf_map_put(map);
226 goto out;
227 }
228
229 if (new_sk_storage) {
230 bpf_selem_link_map(smap, copy_selem);
231 bpf_selem_link_storage_nolock(new_sk_storage, copy_selem);
232 } else {
233 ret = bpf_local_storage_alloc(newsk, smap, copy_selem, GFP_ATOMIC);
234 if (ret) {
235 kfree(copy_selem);
236 atomic_sub(smap->elem_size,
237 &newsk->sk_omem_alloc);
238 bpf_map_put(map);
239 goto out;
240 }
241
242 new_sk_storage =
243 rcu_dereference(copy_selem->local_storage);
244 }
245 bpf_map_put(map);
246 }
247
248 out:
249 rcu_read_unlock();
250
251 /* In case of an error, don't free anything explicitly here, the
252 * caller is responsible to call bpf_sk_storage_free.
253 */
254
255 return ret;
256 }
257
258 /* *gfp_flags* is a hidden argument provided by the verifier */
259 BPF_CALL_5(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk,
260 void *, value, u64, flags, gfp_t, gfp_flags)
261 {
262 struct bpf_local_storage_data *sdata;
263
264 WARN_ON_ONCE(!bpf_rcu_lock_held());
265 if (!sk || !sk_fullsock(sk) || flags > BPF_SK_STORAGE_GET_F_CREATE)
266 return (unsigned long)NULL;
267
268 sdata = bpf_sk_storage_lookup(sk, map, true);
269 if (sdata)
270 return (unsigned long)sdata->data;
271
272 if (flags == BPF_SK_STORAGE_GET_F_CREATE &&
273 /* Cannot add new elem to a going away sk.
274 * Otherwise, the new elem may become a leak
275 * (and also other memory issues during map
276 * destruction).
277 */
278 refcount_inc_not_zero(&sk->sk_refcnt)) {
279 sdata = bpf_local_storage_update(
280 sk, (struct bpf_local_storage_map *)map, value,
281 BPF_NOEXIST, gfp_flags);
282 /* sk must be a fullsock (guaranteed by verifier),
283 * so sock_gen_put() is unnecessary.
284 */
285 sock_put(sk);
286 return IS_ERR(sdata) ?
287 (unsigned long)NULL : (unsigned long)sdata->data;
288 }
289
290 return (unsigned long)NULL;
291 }
292
293 BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk)
294 {
295 WARN_ON_ONCE(!bpf_rcu_lock_held());
296 if (!sk || !sk_fullsock(sk))
297 return -EINVAL;
298
299 if (refcount_inc_not_zero(&sk->sk_refcnt)) {
300 int err;
301
302 err = bpf_sk_storage_del(sk, map);
303 sock_put(sk);
304 return err;
305 }
306
307 return -ENOENT;
308 }
309
310 static int bpf_sk_storage_charge(struct bpf_local_storage_map *smap,
311 void *owner, u32 size)
312 {
313 int optmem_max = READ_ONCE(sysctl_optmem_max);
314 struct sock *sk = (struct sock *)owner;
315
316 /* same check as in sock_kmalloc() */
317 if (size <= optmem_max &&
318 atomic_read(&sk->sk_omem_alloc) + size < optmem_max) {
319 atomic_add(size, &sk->sk_omem_alloc);
320 return 0;
321 }
322
323 return -ENOMEM;
324 }
325
326 static void bpf_sk_storage_uncharge(struct bpf_local_storage_map *smap,
327 void *owner, u32 size)
328 {
329 struct sock *sk = owner;
330
331 atomic_sub(size, &sk->sk_omem_alloc);
332 }
333
334 static struct bpf_local_storage __rcu **
335 bpf_sk_storage_ptr(void *owner)
336 {
337 struct sock *sk = owner;
338
339 return &sk->sk_bpf_storage;
340 }
341
342 BTF_ID_LIST_SINGLE(sk_storage_map_btf_ids, struct, bpf_local_storage_map)
343 const struct bpf_map_ops sk_storage_map_ops = {
344 .map_meta_equal = bpf_map_meta_equal,
345 .map_alloc_check = bpf_local_storage_map_alloc_check,
346 .map_alloc = bpf_sk_storage_map_alloc,
347 .map_free = bpf_sk_storage_map_free,
348 .map_get_next_key = notsupp_get_next_key,
349 .map_lookup_elem = bpf_fd_sk_storage_lookup_elem,
350 .map_update_elem = bpf_fd_sk_storage_update_elem,
351 .map_delete_elem = bpf_fd_sk_storage_delete_elem,
352 .map_check_btf = bpf_local_storage_map_check_btf,
353 .map_btf_id = &sk_storage_map_btf_ids[0],
354 .map_local_storage_charge = bpf_sk_storage_charge,
355 .map_local_storage_uncharge = bpf_sk_storage_uncharge,
356 .map_owner_storage_ptr = bpf_sk_storage_ptr,
357 };
358
359 const struct bpf_func_proto bpf_sk_storage_get_proto = {
360 .func = bpf_sk_storage_get,
361 .gpl_only = false,
362 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
363 .arg1_type = ARG_CONST_MAP_PTR,
364 .arg2_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
365 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
366 .arg4_type = ARG_ANYTHING,
367 };
368
369 const struct bpf_func_proto bpf_sk_storage_get_cg_sock_proto = {
370 .func = bpf_sk_storage_get,
371 .gpl_only = false,
372 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
373 .arg1_type = ARG_CONST_MAP_PTR,
374 .arg2_type = ARG_PTR_TO_CTX, /* context is 'struct sock' */
375 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
376 .arg4_type = ARG_ANYTHING,
377 };
378
379 const struct bpf_func_proto bpf_sk_storage_delete_proto = {
380 .func = bpf_sk_storage_delete,
381 .gpl_only = false,
382 .ret_type = RET_INTEGER,
383 .arg1_type = ARG_CONST_MAP_PTR,
384 .arg2_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
385 };
386
387 static bool bpf_sk_storage_tracing_allowed(const struct bpf_prog *prog)
388 {
389 const struct btf *btf_vmlinux;
390 const struct btf_type *t;
391 const char *tname;
392 u32 btf_id;
393
394 if (prog->aux->dst_prog)
395 return false;
396
397 /* Ensure the tracing program is not tracing
398 * any bpf_sk_storage*() function and also
399 * use the bpf_sk_storage_(get|delete) helper.
400 */
401 switch (prog->expected_attach_type) {
402 case BPF_TRACE_ITER:
403 case BPF_TRACE_RAW_TP:
404 /* bpf_sk_storage has no trace point */
405 return true;
406 case BPF_TRACE_FENTRY:
407 case BPF_TRACE_FEXIT:
408 btf_vmlinux = bpf_get_btf_vmlinux();
409 if (IS_ERR_OR_NULL(btf_vmlinux))
410 return false;
411 btf_id = prog->aux->attach_btf_id;
412 t = btf_type_by_id(btf_vmlinux, btf_id);
413 tname = btf_name_by_offset(btf_vmlinux, t->name_off);
414 return !!strncmp(tname, "bpf_sk_storage",
415 strlen("bpf_sk_storage"));
416 default:
417 return false;
418 }
419
420 return false;
421 }
422
423 /* *gfp_flags* is a hidden argument provided by the verifier */
424 BPF_CALL_5(bpf_sk_storage_get_tracing, struct bpf_map *, map, struct sock *, sk,
425 void *, value, u64, flags, gfp_t, gfp_flags)
426 {
427 WARN_ON_ONCE(!bpf_rcu_lock_held());
428 if (in_hardirq() || in_nmi())
429 return (unsigned long)NULL;
430
431 return (unsigned long)____bpf_sk_storage_get(map, sk, value, flags,
432 gfp_flags);
433 }
434
435 BPF_CALL_2(bpf_sk_storage_delete_tracing, struct bpf_map *, map,
436 struct sock *, sk)
437 {
438 WARN_ON_ONCE(!bpf_rcu_lock_held());
439 if (in_hardirq() || in_nmi())
440 return -EPERM;
441
442 return ____bpf_sk_storage_delete(map, sk);
443 }
444
445 const struct bpf_func_proto bpf_sk_storage_get_tracing_proto = {
446 .func = bpf_sk_storage_get_tracing,
447 .gpl_only = false,
448 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
449 .arg1_type = ARG_CONST_MAP_PTR,
450 .arg2_type = ARG_PTR_TO_BTF_ID,
451 .arg2_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
452 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
453 .arg4_type = ARG_ANYTHING,
454 .allowed = bpf_sk_storage_tracing_allowed,
455 };
456
457 const struct bpf_func_proto bpf_sk_storage_delete_tracing_proto = {
458 .func = bpf_sk_storage_delete_tracing,
459 .gpl_only = false,
460 .ret_type = RET_INTEGER,
461 .arg1_type = ARG_CONST_MAP_PTR,
462 .arg2_type = ARG_PTR_TO_BTF_ID,
463 .arg2_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
464 .allowed = bpf_sk_storage_tracing_allowed,
465 };
466
467 struct bpf_sk_storage_diag {
468 u32 nr_maps;
469 struct bpf_map *maps[];
470 };
471
472 /* The reply will be like:
473 * INET_DIAG_BPF_SK_STORAGES (nla_nest)
474 * SK_DIAG_BPF_STORAGE (nla_nest)
475 * SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
476 * SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
477 * SK_DIAG_BPF_STORAGE (nla_nest)
478 * SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
479 * SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
480 * ....
481 */
482 static int nla_value_size(u32 value_size)
483 {
484 /* SK_DIAG_BPF_STORAGE (nla_nest)
485 * SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
486 * SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
487 */
488 return nla_total_size(0) + nla_total_size(sizeof(u32)) +
489 nla_total_size_64bit(value_size);
490 }
491
492 void bpf_sk_storage_diag_free(struct bpf_sk_storage_diag *diag)
493 {
494 u32 i;
495
496 if (!diag)
497 return;
498
499 for (i = 0; i < diag->nr_maps; i++)
500 bpf_map_put(diag->maps[i]);
501
502 kfree(diag);
503 }
504 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_free);
505
506 static bool diag_check_dup(const struct bpf_sk_storage_diag *diag,
507 const struct bpf_map *map)
508 {
509 u32 i;
510
511 for (i = 0; i < diag->nr_maps; i++) {
512 if (diag->maps[i] == map)
513 return true;
514 }
515
516 return false;
517 }
518
519 struct bpf_sk_storage_diag *
520 bpf_sk_storage_diag_alloc(const struct nlattr *nla_stgs)
521 {
522 struct bpf_sk_storage_diag *diag;
523 struct nlattr *nla;
524 u32 nr_maps = 0;
525 int rem, err;
526
527 /* bpf_local_storage_map is currently limited to CAP_SYS_ADMIN as
528 * the map_alloc_check() side also does.
529 */
530 if (!bpf_capable())
531 return ERR_PTR(-EPERM);
532
533 nla_for_each_nested(nla, nla_stgs, rem) {
534 if (nla_type(nla) == SK_DIAG_BPF_STORAGE_REQ_MAP_FD)
535 nr_maps++;
536 }
537
538 diag = kzalloc(struct_size(diag, maps, nr_maps), GFP_KERNEL);
539 if (!diag)
540 return ERR_PTR(-ENOMEM);
541
542 nla_for_each_nested(nla, nla_stgs, rem) {
543 struct bpf_map *map;
544 int map_fd;
545
546 if (nla_type(nla) != SK_DIAG_BPF_STORAGE_REQ_MAP_FD)
547 continue;
548
549 map_fd = nla_get_u32(nla);
550 map = bpf_map_get(map_fd);
551 if (IS_ERR(map)) {
552 err = PTR_ERR(map);
553 goto err_free;
554 }
555 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) {
556 bpf_map_put(map);
557 err = -EINVAL;
558 goto err_free;
559 }
560 if (diag_check_dup(diag, map)) {
561 bpf_map_put(map);
562 err = -EEXIST;
563 goto err_free;
564 }
565 diag->maps[diag->nr_maps++] = map;
566 }
567
568 return diag;
569
570 err_free:
571 bpf_sk_storage_diag_free(diag);
572 return ERR_PTR(err);
573 }
574 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_alloc);
575
576 static int diag_get(struct bpf_local_storage_data *sdata, struct sk_buff *skb)
577 {
578 struct nlattr *nla_stg, *nla_value;
579 struct bpf_local_storage_map *smap;
580
581 /* It cannot exceed max nlattr's payload */
582 BUILD_BUG_ON(U16_MAX - NLA_HDRLEN < BPF_LOCAL_STORAGE_MAX_VALUE_SIZE);
583
584 nla_stg = nla_nest_start(skb, SK_DIAG_BPF_STORAGE);
585 if (!nla_stg)
586 return -EMSGSIZE;
587
588 smap = rcu_dereference(sdata->smap);
589 if (nla_put_u32(skb, SK_DIAG_BPF_STORAGE_MAP_ID, smap->map.id))
590 goto errout;
591
592 nla_value = nla_reserve_64bit(skb, SK_DIAG_BPF_STORAGE_MAP_VALUE,
593 smap->map.value_size,
594 SK_DIAG_BPF_STORAGE_PAD);
595 if (!nla_value)
596 goto errout;
597
598 if (map_value_has_spin_lock(&smap->map))
599 copy_map_value_locked(&smap->map, nla_data(nla_value),
600 sdata->data, true);
601 else
602 copy_map_value(&smap->map, nla_data(nla_value), sdata->data);
603
604 nla_nest_end(skb, nla_stg);
605 return 0;
606
607 errout:
608 nla_nest_cancel(skb, nla_stg);
609 return -EMSGSIZE;
610 }
611
612 static int bpf_sk_storage_diag_put_all(struct sock *sk, struct sk_buff *skb,
613 int stg_array_type,
614 unsigned int *res_diag_size)
615 {
616 /* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */
617 unsigned int diag_size = nla_total_size(0);
618 struct bpf_local_storage *sk_storage;
619 struct bpf_local_storage_elem *selem;
620 struct bpf_local_storage_map *smap;
621 struct nlattr *nla_stgs;
622 unsigned int saved_len;
623 int err = 0;
624
625 rcu_read_lock();
626
627 sk_storage = rcu_dereference(sk->sk_bpf_storage);
628 if (!sk_storage || hlist_empty(&sk_storage->list)) {
629 rcu_read_unlock();
630 return 0;
631 }
632
633 nla_stgs = nla_nest_start(skb, stg_array_type);
634 if (!nla_stgs)
635 /* Continue to learn diag_size */
636 err = -EMSGSIZE;
637
638 saved_len = skb->len;
639 hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
640 smap = rcu_dereference(SDATA(selem)->smap);
641 diag_size += nla_value_size(smap->map.value_size);
642
643 if (nla_stgs && diag_get(SDATA(selem), skb))
644 /* Continue to learn diag_size */
645 err = -EMSGSIZE;
646 }
647
648 rcu_read_unlock();
649
650 if (nla_stgs) {
651 if (saved_len == skb->len)
652 nla_nest_cancel(skb, nla_stgs);
653 else
654 nla_nest_end(skb, nla_stgs);
655 }
656
657 if (diag_size == nla_total_size(0)) {
658 *res_diag_size = 0;
659 return 0;
660 }
661
662 *res_diag_size = diag_size;
663 return err;
664 }
665
666 int bpf_sk_storage_diag_put(struct bpf_sk_storage_diag *diag,
667 struct sock *sk, struct sk_buff *skb,
668 int stg_array_type,
669 unsigned int *res_diag_size)
670 {
671 /* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */
672 unsigned int diag_size = nla_total_size(0);
673 struct bpf_local_storage *sk_storage;
674 struct bpf_local_storage_data *sdata;
675 struct nlattr *nla_stgs;
676 unsigned int saved_len;
677 int err = 0;
678 u32 i;
679
680 *res_diag_size = 0;
681
682 /* No map has been specified. Dump all. */
683 if (!diag->nr_maps)
684 return bpf_sk_storage_diag_put_all(sk, skb, stg_array_type,
685 res_diag_size);
686
687 rcu_read_lock();
688 sk_storage = rcu_dereference(sk->sk_bpf_storage);
689 if (!sk_storage || hlist_empty(&sk_storage->list)) {
690 rcu_read_unlock();
691 return 0;
692 }
693
694 nla_stgs = nla_nest_start(skb, stg_array_type);
695 if (!nla_stgs)
696 /* Continue to learn diag_size */
697 err = -EMSGSIZE;
698
699 saved_len = skb->len;
700 for (i = 0; i < diag->nr_maps; i++) {
701 sdata = bpf_local_storage_lookup(sk_storage,
702 (struct bpf_local_storage_map *)diag->maps[i],
703 false);
704
705 if (!sdata)
706 continue;
707
708 diag_size += nla_value_size(diag->maps[i]->value_size);
709
710 if (nla_stgs && diag_get(sdata, skb))
711 /* Continue to learn diag_size */
712 err = -EMSGSIZE;
713 }
714 rcu_read_unlock();
715
716 if (nla_stgs) {
717 if (saved_len == skb->len)
718 nla_nest_cancel(skb, nla_stgs);
719 else
720 nla_nest_end(skb, nla_stgs);
721 }
722
723 if (diag_size == nla_total_size(0)) {
724 *res_diag_size = 0;
725 return 0;
726 }
727
728 *res_diag_size = diag_size;
729 return err;
730 }
731 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_put);
732
733 struct bpf_iter_seq_sk_storage_map_info {
734 struct bpf_map *map;
735 unsigned int bucket_id;
736 unsigned skip_elems;
737 };
738
739 static struct bpf_local_storage_elem *
740 bpf_sk_storage_map_seq_find_next(struct bpf_iter_seq_sk_storage_map_info *info,
741 struct bpf_local_storage_elem *prev_selem)
742 __acquires(RCU) __releases(RCU)
743 {
744 struct bpf_local_storage *sk_storage;
745 struct bpf_local_storage_elem *selem;
746 u32 skip_elems = info->skip_elems;
747 struct bpf_local_storage_map *smap;
748 u32 bucket_id = info->bucket_id;
749 u32 i, count, n_buckets;
750 struct bpf_local_storage_map_bucket *b;
751
752 smap = (struct bpf_local_storage_map *)info->map;
753 n_buckets = 1U << smap->bucket_log;
754 if (bucket_id >= n_buckets)
755 return NULL;
756
757 /* try to find next selem in the same bucket */
758 selem = prev_selem;
759 count = 0;
760 while (selem) {
761 selem = hlist_entry_safe(rcu_dereference(hlist_next_rcu(&selem->map_node)),
762 struct bpf_local_storage_elem, map_node);
763 if (!selem) {
764 /* not found, unlock and go to the next bucket */
765 b = &smap->buckets[bucket_id++];
766 rcu_read_unlock();
767 skip_elems = 0;
768 break;
769 }
770 sk_storage = rcu_dereference(selem->local_storage);
771 if (sk_storage) {
772 info->skip_elems = skip_elems + count;
773 return selem;
774 }
775 count++;
776 }
777
778 for (i = bucket_id; i < (1U << smap->bucket_log); i++) {
779 b = &smap->buckets[i];
780 rcu_read_lock();
781 count = 0;
782 hlist_for_each_entry_rcu(selem, &b->list, map_node) {
783 sk_storage = rcu_dereference(selem->local_storage);
784 if (sk_storage && count >= skip_elems) {
785 info->bucket_id = i;
786 info->skip_elems = count;
787 return selem;
788 }
789 count++;
790 }
791 rcu_read_unlock();
792 skip_elems = 0;
793 }
794
795 info->bucket_id = i;
796 info->skip_elems = 0;
797 return NULL;
798 }
799
800 static void *bpf_sk_storage_map_seq_start(struct seq_file *seq, loff_t *pos)
801 {
802 struct bpf_local_storage_elem *selem;
803
804 selem = bpf_sk_storage_map_seq_find_next(seq->private, NULL);
805 if (!selem)
806 return NULL;
807
808 if (*pos == 0)
809 ++*pos;
810 return selem;
811 }
812
813 static void *bpf_sk_storage_map_seq_next(struct seq_file *seq, void *v,
814 loff_t *pos)
815 {
816 struct bpf_iter_seq_sk_storage_map_info *info = seq->private;
817
818 ++*pos;
819 ++info->skip_elems;
820 return bpf_sk_storage_map_seq_find_next(seq->private, v);
821 }
822
823 struct bpf_iter__bpf_sk_storage_map {
824 __bpf_md_ptr(struct bpf_iter_meta *, meta);
825 __bpf_md_ptr(struct bpf_map *, map);
826 __bpf_md_ptr(struct sock *, sk);
827 __bpf_md_ptr(void *, value);
828 };
829
830 DEFINE_BPF_ITER_FUNC(bpf_sk_storage_map, struct bpf_iter_meta *meta,
831 struct bpf_map *map, struct sock *sk,
832 void *value)
833
834 static int __bpf_sk_storage_map_seq_show(struct seq_file *seq,
835 struct bpf_local_storage_elem *selem)
836 {
837 struct bpf_iter_seq_sk_storage_map_info *info = seq->private;
838 struct bpf_iter__bpf_sk_storage_map ctx = {};
839 struct bpf_local_storage *sk_storage;
840 struct bpf_iter_meta meta;
841 struct bpf_prog *prog;
842 int ret = 0;
843
844 meta.seq = seq;
845 prog = bpf_iter_get_info(&meta, selem == NULL);
846 if (prog) {
847 ctx.meta = &meta;
848 ctx.map = info->map;
849 if (selem) {
850 sk_storage = rcu_dereference(selem->local_storage);
851 ctx.sk = sk_storage->owner;
852 ctx.value = SDATA(selem)->data;
853 }
854 ret = bpf_iter_run_prog(prog, &ctx);
855 }
856
857 return ret;
858 }
859
860 static int bpf_sk_storage_map_seq_show(struct seq_file *seq, void *v)
861 {
862 return __bpf_sk_storage_map_seq_show(seq, v);
863 }
864
865 static void bpf_sk_storage_map_seq_stop(struct seq_file *seq, void *v)
866 __releases(RCU)
867 {
868 if (!v)
869 (void)__bpf_sk_storage_map_seq_show(seq, v);
870 else
871 rcu_read_unlock();
872 }
873
874 static int bpf_iter_init_sk_storage_map(void *priv_data,
875 struct bpf_iter_aux_info *aux)
876 {
877 struct bpf_iter_seq_sk_storage_map_info *seq_info = priv_data;
878
879 bpf_map_inc_with_uref(aux->map);
880 seq_info->map = aux->map;
881 return 0;
882 }
883
884 static void bpf_iter_fini_sk_storage_map(void *priv_data)
885 {
886 struct bpf_iter_seq_sk_storage_map_info *seq_info = priv_data;
887
888 bpf_map_put_with_uref(seq_info->map);
889 }
890
891 static int bpf_iter_attach_map(struct bpf_prog *prog,
892 union bpf_iter_link_info *linfo,
893 struct bpf_iter_aux_info *aux)
894 {
895 struct bpf_map *map;
896 int err = -EINVAL;
897
898 if (!linfo->map.map_fd)
899 return -EBADF;
900
901 map = bpf_map_get_with_uref(linfo->map.map_fd);
902 if (IS_ERR(map))
903 return PTR_ERR(map);
904
905 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE)
906 goto put_map;
907
908 if (prog->aux->max_rdwr_access > map->value_size) {
909 err = -EACCES;
910 goto put_map;
911 }
912
913 aux->map = map;
914 return 0;
915
916 put_map:
917 bpf_map_put_with_uref(map);
918 return err;
919 }
920
921 static void bpf_iter_detach_map(struct bpf_iter_aux_info *aux)
922 {
923 bpf_map_put_with_uref(aux->map);
924 }
925
926 static const struct seq_operations bpf_sk_storage_map_seq_ops = {
927 .start = bpf_sk_storage_map_seq_start,
928 .next = bpf_sk_storage_map_seq_next,
929 .stop = bpf_sk_storage_map_seq_stop,
930 .show = bpf_sk_storage_map_seq_show,
931 };
932
933 static const struct bpf_iter_seq_info iter_seq_info = {
934 .seq_ops = &bpf_sk_storage_map_seq_ops,
935 .init_seq_private = bpf_iter_init_sk_storage_map,
936 .fini_seq_private = bpf_iter_fini_sk_storage_map,
937 .seq_priv_size = sizeof(struct bpf_iter_seq_sk_storage_map_info),
938 };
939
940 static struct bpf_iter_reg bpf_sk_storage_map_reg_info = {
941 .target = "bpf_sk_storage_map",
942 .attach_target = bpf_iter_attach_map,
943 .detach_target = bpf_iter_detach_map,
944 .show_fdinfo = bpf_iter_map_show_fdinfo,
945 .fill_link_info = bpf_iter_map_fill_link_info,
946 .ctx_arg_info_size = 2,
947 .ctx_arg_info = {
948 { offsetof(struct bpf_iter__bpf_sk_storage_map, sk),
949 PTR_TO_BTF_ID_OR_NULL },
950 { offsetof(struct bpf_iter__bpf_sk_storage_map, value),
951 PTR_TO_BUF | PTR_MAYBE_NULL },
952 },
953 .seq_info = &iter_seq_info,
954 };
955
956 static int __init bpf_sk_storage_map_iter_init(void)
957 {
958 bpf_sk_storage_map_reg_info.ctx_arg_info[0].btf_id =
959 btf_sock_ids[BTF_SOCK_TYPE_SOCK];
960 return bpf_iter_reg_target(&bpf_sk_storage_map_reg_info);
961 }
962 late_initcall(bpf_sk_storage_map_iter_init);
963
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.