~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/core/sock_map.c

Version: ~ [ linux-5.8 ] ~ [ linux-5.7.14 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.57 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.138 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.193 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.232 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.232 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */
  3 
  4 #include <linux/bpf.h>
  5 #include <linux/filter.h>
  6 #include <linux/errno.h>
  7 #include <linux/file.h>
  8 #include <linux/net.h>
  9 #include <linux/workqueue.h>
 10 #include <linux/skmsg.h>
 11 #include <linux/list.h>
 12 #include <linux/jhash.h>
 13 
 14 struct bpf_stab {
 15         struct bpf_map map;
 16         struct sock **sks;
 17         struct sk_psock_progs progs;
 18         raw_spinlock_t lock;
 19 };
 20 
 21 #define SOCK_CREATE_FLAG_MASK                           \
 22         (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY)
 23 
 24 static struct bpf_map *sock_map_alloc(union bpf_attr *attr)
 25 {
 26         struct bpf_stab *stab;
 27         u64 cost;
 28         int err;
 29 
 30         if (!capable(CAP_NET_ADMIN))
 31                 return ERR_PTR(-EPERM);
 32         if (attr->max_entries == 0 ||
 33             attr->key_size    != 4 ||
 34             attr->value_size  != 4 ||
 35             attr->map_flags & ~SOCK_CREATE_FLAG_MASK)
 36                 return ERR_PTR(-EINVAL);
 37 
 38         stab = kzalloc(sizeof(*stab), GFP_USER);
 39         if (!stab)
 40                 return ERR_PTR(-ENOMEM);
 41 
 42         bpf_map_init_from_attr(&stab->map, attr);
 43         raw_spin_lock_init(&stab->lock);
 44 
 45         /* Make sure page count doesn't overflow. */
 46         cost = (u64) stab->map.max_entries * sizeof(struct sock *);
 47         if (cost >= U32_MAX - PAGE_SIZE) {
 48                 err = -EINVAL;
 49                 goto free_stab;
 50         }
 51 
 52         stab->map.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
 53         err = bpf_map_precharge_memlock(stab->map.pages);
 54         if (err)
 55                 goto free_stab;
 56 
 57         stab->sks = bpf_map_area_alloc(stab->map.max_entries *
 58                                        sizeof(struct sock *),
 59                                        stab->map.numa_node);
 60         if (stab->sks)
 61                 return &stab->map;
 62         err = -ENOMEM;
 63 free_stab:
 64         kfree(stab);
 65         return ERR_PTR(err);
 66 }
 67 
 68 int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog)
 69 {
 70         u32 ufd = attr->target_fd;
 71         struct bpf_map *map;
 72         struct fd f;
 73         int ret;
 74 
 75         f = fdget(ufd);
 76         map = __bpf_map_get(f);
 77         if (IS_ERR(map))
 78                 return PTR_ERR(map);
 79         ret = sock_map_prog_update(map, prog, attr->attach_type);
 80         fdput(f);
 81         return ret;
 82 }
 83 
 84 static void sock_map_sk_acquire(struct sock *sk)
 85         __acquires(&sk->sk_lock.slock)
 86 {
 87         lock_sock(sk);
 88         preempt_disable();
 89         rcu_read_lock();
 90 }
 91 
 92 static void sock_map_sk_release(struct sock *sk)
 93         __releases(&sk->sk_lock.slock)
 94 {
 95         rcu_read_unlock();
 96         preempt_enable();
 97         release_sock(sk);
 98 }
 99 
100 static void sock_map_add_link(struct sk_psock *psock,
101                               struct sk_psock_link *link,
102                               struct bpf_map *map, void *link_raw)
103 {
104         link->link_raw = link_raw;
105         link->map = map;
106         spin_lock_bh(&psock->link_lock);
107         list_add_tail(&link->list, &psock->link);
108         spin_unlock_bh(&psock->link_lock);
109 }
110 
111 static void sock_map_del_link(struct sock *sk,
112                               struct sk_psock *psock, void *link_raw)
113 {
114         struct sk_psock_link *link, *tmp;
115         bool strp_stop = false;
116 
117         spin_lock_bh(&psock->link_lock);
118         list_for_each_entry_safe(link, tmp, &psock->link, list) {
119                 if (link->link_raw == link_raw) {
120                         struct bpf_map *map = link->map;
121                         struct bpf_stab *stab = container_of(map, struct bpf_stab,
122                                                              map);
123                         if (psock->parser.enabled && stab->progs.skb_parser)
124                                 strp_stop = true;
125                         list_del(&link->list);
126                         sk_psock_free_link(link);
127                 }
128         }
129         spin_unlock_bh(&psock->link_lock);
130         if (strp_stop) {
131                 write_lock_bh(&sk->sk_callback_lock);
132                 sk_psock_stop_strp(sk, psock);
133                 write_unlock_bh(&sk->sk_callback_lock);
134         }
135 }
136 
137 static void sock_map_unref(struct sock *sk, void *link_raw)
138 {
139         struct sk_psock *psock = sk_psock(sk);
140 
141         if (likely(psock)) {
142                 sock_map_del_link(sk, psock, link_raw);
143                 sk_psock_put(sk, psock);
144         }
145 }
146 
147 static int sock_map_link(struct bpf_map *map, struct sk_psock_progs *progs,
148                          struct sock *sk)
149 {
150         struct bpf_prog *msg_parser, *skb_parser, *skb_verdict;
151         bool skb_progs, sk_psock_is_new = false;
152         struct sk_psock *psock;
153         int ret;
154 
155         skb_verdict = READ_ONCE(progs->skb_verdict);
156         skb_parser = READ_ONCE(progs->skb_parser);
157         skb_progs = skb_parser && skb_verdict;
158         if (skb_progs) {
159                 skb_verdict = bpf_prog_inc_not_zero(skb_verdict);
160                 if (IS_ERR(skb_verdict))
161                         return PTR_ERR(skb_verdict);
162                 skb_parser = bpf_prog_inc_not_zero(skb_parser);
163                 if (IS_ERR(skb_parser)) {
164                         bpf_prog_put(skb_verdict);
165                         return PTR_ERR(skb_parser);
166                 }
167         }
168 
169         msg_parser = READ_ONCE(progs->msg_parser);
170         if (msg_parser) {
171                 msg_parser = bpf_prog_inc_not_zero(msg_parser);
172                 if (IS_ERR(msg_parser)) {
173                         ret = PTR_ERR(msg_parser);
174                         goto out;
175                 }
176         }
177 
178         psock = sk_psock_get_checked(sk);
179         if (IS_ERR(psock)) {
180                 ret = PTR_ERR(psock);
181                 goto out_progs;
182         }
183 
184         if (psock) {
185                 if ((msg_parser && READ_ONCE(psock->progs.msg_parser)) ||
186                     (skb_progs  && READ_ONCE(psock->progs.skb_parser))) {
187                         sk_psock_put(sk, psock);
188                         ret = -EBUSY;
189                         goto out_progs;
190                 }
191         } else {
192                 psock = sk_psock_init(sk, map->numa_node);
193                 if (!psock) {
194                         ret = -ENOMEM;
195                         goto out_progs;
196                 }
197                 sk_psock_is_new = true;
198         }
199 
200         if (msg_parser)
201                 psock_set_prog(&psock->progs.msg_parser, msg_parser);
202         if (sk_psock_is_new) {
203                 ret = tcp_bpf_init(sk);
204                 if (ret < 0)
205                         goto out_drop;
206         } else {
207                 tcp_bpf_reinit(sk);
208         }
209 
210         write_lock_bh(&sk->sk_callback_lock);
211         if (skb_progs && !psock->parser.enabled) {
212                 ret = sk_psock_init_strp(sk, psock);
213                 if (ret) {
214                         write_unlock_bh(&sk->sk_callback_lock);
215                         goto out_drop;
216                 }
217                 psock_set_prog(&psock->progs.skb_verdict, skb_verdict);
218                 psock_set_prog(&psock->progs.skb_parser, skb_parser);
219                 sk_psock_start_strp(sk, psock);
220         }
221         write_unlock_bh(&sk->sk_callback_lock);
222         return 0;
223 out_drop:
224         sk_psock_put(sk, psock);
225 out_progs:
226         if (msg_parser)
227                 bpf_prog_put(msg_parser);
228 out:
229         if (skb_progs) {
230                 bpf_prog_put(skb_verdict);
231                 bpf_prog_put(skb_parser);
232         }
233         return ret;
234 }
235 
236 static void sock_map_free(struct bpf_map *map)
237 {
238         struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
239         int i;
240 
241         synchronize_rcu();
242         rcu_read_lock();
243         raw_spin_lock_bh(&stab->lock);
244         for (i = 0; i < stab->map.max_entries; i++) {
245                 struct sock **psk = &stab->sks[i];
246                 struct sock *sk;
247 
248                 sk = xchg(psk, NULL);
249                 if (sk)
250                         sock_map_unref(sk, psk);
251         }
252         raw_spin_unlock_bh(&stab->lock);
253         rcu_read_unlock();
254 
255         synchronize_rcu();
256 
257         bpf_map_area_free(stab->sks);
258         kfree(stab);
259 }
260 
261 static void sock_map_release_progs(struct bpf_map *map)
262 {
263         psock_progs_drop(&container_of(map, struct bpf_stab, map)->progs);
264 }
265 
266 static struct sock *__sock_map_lookup_elem(struct bpf_map *map, u32 key)
267 {
268         struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
269 
270         WARN_ON_ONCE(!rcu_read_lock_held());
271 
272         if (unlikely(key >= map->max_entries))
273                 return NULL;
274         return READ_ONCE(stab->sks[key]);
275 }
276 
277 static void *sock_map_lookup(struct bpf_map *map, void *key)
278 {
279         return ERR_PTR(-EOPNOTSUPP);
280 }
281 
282 static int __sock_map_delete(struct bpf_stab *stab, struct sock *sk_test,
283                              struct sock **psk)
284 {
285         struct sock *sk;
286         int err = 0;
287 
288         raw_spin_lock_bh(&stab->lock);
289         sk = *psk;
290         if (!sk_test || sk_test == sk)
291                 sk = xchg(psk, NULL);
292 
293         if (likely(sk))
294                 sock_map_unref(sk, psk);
295         else
296                 err = -EINVAL;
297 
298         raw_spin_unlock_bh(&stab->lock);
299         return err;
300 }
301 
302 static void sock_map_delete_from_link(struct bpf_map *map, struct sock *sk,
303                                       void *link_raw)
304 {
305         struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
306 
307         __sock_map_delete(stab, sk, link_raw);
308 }
309 
310 static int sock_map_delete_elem(struct bpf_map *map, void *key)
311 {
312         struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
313         u32 i = *(u32 *)key;
314         struct sock **psk;
315 
316         if (unlikely(i >= map->max_entries))
317                 return -EINVAL;
318 
319         psk = &stab->sks[i];
320         return __sock_map_delete(stab, NULL, psk);
321 }
322 
323 static int sock_map_get_next_key(struct bpf_map *map, void *key, void *next)
324 {
325         struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
326         u32 i = key ? *(u32 *)key : U32_MAX;
327         u32 *key_next = next;
328 
329         if (i == stab->map.max_entries - 1)
330                 return -ENOENT;
331         if (i >= stab->map.max_entries)
332                 *key_next = 0;
333         else
334                 *key_next = i + 1;
335         return 0;
336 }
337 
338 static int sock_map_update_common(struct bpf_map *map, u32 idx,
339                                   struct sock *sk, u64 flags)
340 {
341         struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
342         struct inet_connection_sock *icsk = inet_csk(sk);
343         struct sk_psock_link *link;
344         struct sk_psock *psock;
345         struct sock *osk;
346         int ret;
347 
348         WARN_ON_ONCE(!rcu_read_lock_held());
349         if (unlikely(flags > BPF_EXIST))
350                 return -EINVAL;
351         if (unlikely(idx >= map->max_entries))
352                 return -E2BIG;
353         if (unlikely(icsk->icsk_ulp_data))
354                 return -EINVAL;
355 
356         link = sk_psock_init_link();
357         if (!link)
358                 return -ENOMEM;
359 
360         ret = sock_map_link(map, &stab->progs, sk);
361         if (ret < 0)
362                 goto out_free;
363 
364         psock = sk_psock(sk);
365         WARN_ON_ONCE(!psock);
366 
367         raw_spin_lock_bh(&stab->lock);
368         osk = stab->sks[idx];
369         if (osk && flags == BPF_NOEXIST) {
370                 ret = -EEXIST;
371                 goto out_unlock;
372         } else if (!osk && flags == BPF_EXIST) {
373                 ret = -ENOENT;
374                 goto out_unlock;
375         }
376 
377         sock_map_add_link(psock, link, map, &stab->sks[idx]);
378         stab->sks[idx] = sk;
379         if (osk)
380                 sock_map_unref(osk, &stab->sks[idx]);
381         raw_spin_unlock_bh(&stab->lock);
382         return 0;
383 out_unlock:
384         raw_spin_unlock_bh(&stab->lock);
385         if (psock)
386                 sk_psock_put(sk, psock);
387 out_free:
388         sk_psock_free_link(link);
389         return ret;
390 }
391 
392 static bool sock_map_op_okay(const struct bpf_sock_ops_kern *ops)
393 {
394         return ops->op == BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB ||
395                ops->op == BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB;
396 }
397 
398 static bool sock_map_sk_is_suitable(const struct sock *sk)
399 {
400         return sk->sk_type == SOCK_STREAM &&
401                sk->sk_protocol == IPPROTO_TCP;
402 }
403 
404 static int sock_map_update_elem(struct bpf_map *map, void *key,
405                                 void *value, u64 flags)
406 {
407         u32 ufd = *(u32 *)value;
408         u32 idx = *(u32 *)key;
409         struct socket *sock;
410         struct sock *sk;
411         int ret;
412 
413         sock = sockfd_lookup(ufd, &ret);
414         if (!sock)
415                 return ret;
416         sk = sock->sk;
417         if (!sk) {
418                 ret = -EINVAL;
419                 goto out;
420         }
421         if (!sock_map_sk_is_suitable(sk) ||
422             sk->sk_state != TCP_ESTABLISHED) {
423                 ret = -EOPNOTSUPP;
424                 goto out;
425         }
426 
427         sock_map_sk_acquire(sk);
428         ret = sock_map_update_common(map, idx, sk, flags);
429         sock_map_sk_release(sk);
430 out:
431         fput(sock->file);
432         return ret;
433 }
434 
435 BPF_CALL_4(bpf_sock_map_update, struct bpf_sock_ops_kern *, sops,
436            struct bpf_map *, map, void *, key, u64, flags)
437 {
438         WARN_ON_ONCE(!rcu_read_lock_held());
439 
440         if (likely(sock_map_sk_is_suitable(sops->sk) &&
441                    sock_map_op_okay(sops)))
442                 return sock_map_update_common(map, *(u32 *)key, sops->sk,
443                                               flags);
444         return -EOPNOTSUPP;
445 }
446 
447 const struct bpf_func_proto bpf_sock_map_update_proto = {
448         .func           = bpf_sock_map_update,
449         .gpl_only       = false,
450         .pkt_access     = true,
451         .ret_type       = RET_INTEGER,
452         .arg1_type      = ARG_PTR_TO_CTX,
453         .arg2_type      = ARG_CONST_MAP_PTR,
454         .arg3_type      = ARG_PTR_TO_MAP_KEY,
455         .arg4_type      = ARG_ANYTHING,
456 };
457 
458 BPF_CALL_4(bpf_sk_redirect_map, struct sk_buff *, skb,
459            struct bpf_map *, map, u32, key, u64, flags)
460 {
461         struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
462 
463         if (unlikely(flags & ~(BPF_F_INGRESS)))
464                 return SK_DROP;
465         tcb->bpf.flags = flags;
466         tcb->bpf.sk_redir = __sock_map_lookup_elem(map, key);
467         if (!tcb->bpf.sk_redir)
468                 return SK_DROP;
469         return SK_PASS;
470 }
471 
472 const struct bpf_func_proto bpf_sk_redirect_map_proto = {
473         .func           = bpf_sk_redirect_map,
474         .gpl_only       = false,
475         .ret_type       = RET_INTEGER,
476         .arg1_type      = ARG_PTR_TO_CTX,
477         .arg2_type      = ARG_CONST_MAP_PTR,
478         .arg3_type      = ARG_ANYTHING,
479         .arg4_type      = ARG_ANYTHING,
480 };
481 
482 BPF_CALL_4(bpf_msg_redirect_map, struct sk_msg *, msg,
483            struct bpf_map *, map, u32, key, u64, flags)
484 {
485         if (unlikely(flags & ~(BPF_F_INGRESS)))
486                 return SK_DROP;
487         msg->flags = flags;
488         msg->sk_redir = __sock_map_lookup_elem(map, key);
489         if (!msg->sk_redir)
490                 return SK_DROP;
491         return SK_PASS;
492 }
493 
494 const struct bpf_func_proto bpf_msg_redirect_map_proto = {
495         .func           = bpf_msg_redirect_map,
496         .gpl_only       = false,
497         .ret_type       = RET_INTEGER,
498         .arg1_type      = ARG_PTR_TO_CTX,
499         .arg2_type      = ARG_CONST_MAP_PTR,
500         .arg3_type      = ARG_ANYTHING,
501         .arg4_type      = ARG_ANYTHING,
502 };
503 
504 const struct bpf_map_ops sock_map_ops = {
505         .map_alloc              = sock_map_alloc,
506         .map_free               = sock_map_free,
507         .map_get_next_key       = sock_map_get_next_key,
508         .map_update_elem        = sock_map_update_elem,
509         .map_delete_elem        = sock_map_delete_elem,
510         .map_lookup_elem        = sock_map_lookup,
511         .map_release_uref       = sock_map_release_progs,
512         .map_check_btf          = map_check_no_btf,
513 };
514 
515 struct bpf_htab_elem {
516         struct rcu_head rcu;
517         u32 hash;
518         struct sock *sk;
519         struct hlist_node node;
520         u8 key[0];
521 };
522 
523 struct bpf_htab_bucket {
524         struct hlist_head head;
525         raw_spinlock_t lock;
526 };
527 
528 struct bpf_htab {
529         struct bpf_map map;
530         struct bpf_htab_bucket *buckets;
531         u32 buckets_num;
532         u32 elem_size;
533         struct sk_psock_progs progs;
534         atomic_t count;
535 };
536 
537 static inline u32 sock_hash_bucket_hash(const void *key, u32 len)
538 {
539         return jhash(key, len, 0);
540 }
541 
542 static struct bpf_htab_bucket *sock_hash_select_bucket(struct bpf_htab *htab,
543                                                        u32 hash)
544 {
545         return &htab->buckets[hash & (htab->buckets_num - 1)];
546 }
547 
548 static struct bpf_htab_elem *
549 sock_hash_lookup_elem_raw(struct hlist_head *head, u32 hash, void *key,
550                           u32 key_size)
551 {
552         struct bpf_htab_elem *elem;
553 
554         hlist_for_each_entry_rcu(elem, head, node) {
555                 if (elem->hash == hash &&
556                     !memcmp(&elem->key, key, key_size))
557                         return elem;
558         }
559 
560         return NULL;
561 }
562 
563 static struct sock *__sock_hash_lookup_elem(struct bpf_map *map, void *key)
564 {
565         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
566         u32 key_size = map->key_size, hash;
567         struct bpf_htab_bucket *bucket;
568         struct bpf_htab_elem *elem;
569 
570         WARN_ON_ONCE(!rcu_read_lock_held());
571 
572         hash = sock_hash_bucket_hash(key, key_size);
573         bucket = sock_hash_select_bucket(htab, hash);
574         elem = sock_hash_lookup_elem_raw(&bucket->head, hash, key, key_size);
575 
576         return elem ? elem->sk : NULL;
577 }
578 
579 static void sock_hash_free_elem(struct bpf_htab *htab,
580                                 struct bpf_htab_elem *elem)
581 {
582         atomic_dec(&htab->count);
583         kfree_rcu(elem, rcu);
584 }
585 
586 static void sock_hash_delete_from_link(struct bpf_map *map, struct sock *sk,
587                                        void *link_raw)
588 {
589         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
590         struct bpf_htab_elem *elem_probe, *elem = link_raw;
591         struct bpf_htab_bucket *bucket;
592 
593         WARN_ON_ONCE(!rcu_read_lock_held());
594         bucket = sock_hash_select_bucket(htab, elem->hash);
595 
596         /* elem may be deleted in parallel from the map, but access here
597          * is okay since it's going away only after RCU grace period.
598          * However, we need to check whether it's still present.
599          */
600         raw_spin_lock_bh(&bucket->lock);
601         elem_probe = sock_hash_lookup_elem_raw(&bucket->head, elem->hash,
602                                                elem->key, map->key_size);
603         if (elem_probe && elem_probe == elem) {
604                 hlist_del_rcu(&elem->node);
605                 sock_map_unref(elem->sk, elem);
606                 sock_hash_free_elem(htab, elem);
607         }
608         raw_spin_unlock_bh(&bucket->lock);
609 }
610 
611 static int sock_hash_delete_elem(struct bpf_map *map, void *key)
612 {
613         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
614         u32 hash, key_size = map->key_size;
615         struct bpf_htab_bucket *bucket;
616         struct bpf_htab_elem *elem;
617         int ret = -ENOENT;
618 
619         hash = sock_hash_bucket_hash(key, key_size);
620         bucket = sock_hash_select_bucket(htab, hash);
621 
622         raw_spin_lock_bh(&bucket->lock);
623         elem = sock_hash_lookup_elem_raw(&bucket->head, hash, key, key_size);
624         if (elem) {
625                 hlist_del_rcu(&elem->node);
626                 sock_map_unref(elem->sk, elem);
627                 sock_hash_free_elem(htab, elem);
628                 ret = 0;
629         }
630         raw_spin_unlock_bh(&bucket->lock);
631         return ret;
632 }
633 
634 static struct bpf_htab_elem *sock_hash_alloc_elem(struct bpf_htab *htab,
635                                                   void *key, u32 key_size,
636                                                   u32 hash, struct sock *sk,
637                                                   struct bpf_htab_elem *old)
638 {
639         struct bpf_htab_elem *new;
640 
641         if (atomic_inc_return(&htab->count) > htab->map.max_entries) {
642                 if (!old) {
643                         atomic_dec(&htab->count);
644                         return ERR_PTR(-E2BIG);
645                 }
646         }
647 
648         new = kmalloc_node(htab->elem_size, GFP_ATOMIC | __GFP_NOWARN,
649                            htab->map.numa_node);
650         if (!new) {
651                 atomic_dec(&htab->count);
652                 return ERR_PTR(-ENOMEM);
653         }
654         memcpy(new->key, key, key_size);
655         new->sk = sk;
656         new->hash = hash;
657         return new;
658 }
659 
660 static int sock_hash_update_common(struct bpf_map *map, void *key,
661                                    struct sock *sk, u64 flags)
662 {
663         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
664         struct inet_connection_sock *icsk = inet_csk(sk);
665         u32 key_size = map->key_size, hash;
666         struct bpf_htab_elem *elem, *elem_new;
667         struct bpf_htab_bucket *bucket;
668         struct sk_psock_link *link;
669         struct sk_psock *psock;
670         int ret;
671 
672         WARN_ON_ONCE(!rcu_read_lock_held());
673         if (unlikely(flags > BPF_EXIST))
674                 return -EINVAL;
675         if (unlikely(icsk->icsk_ulp_data))
676                 return -EINVAL;
677 
678         link = sk_psock_init_link();
679         if (!link)
680                 return -ENOMEM;
681 
682         ret = sock_map_link(map, &htab->progs, sk);
683         if (ret < 0)
684                 goto out_free;
685 
686         psock = sk_psock(sk);
687         WARN_ON_ONCE(!psock);
688 
689         hash = sock_hash_bucket_hash(key, key_size);
690         bucket = sock_hash_select_bucket(htab, hash);
691 
692         raw_spin_lock_bh(&bucket->lock);
693         elem = sock_hash_lookup_elem_raw(&bucket->head, hash, key, key_size);
694         if (elem && flags == BPF_NOEXIST) {
695                 ret = -EEXIST;
696                 goto out_unlock;
697         } else if (!elem && flags == BPF_EXIST) {
698                 ret = -ENOENT;
699                 goto out_unlock;
700         }
701 
702         elem_new = sock_hash_alloc_elem(htab, key, key_size, hash, sk, elem);
703         if (IS_ERR(elem_new)) {
704                 ret = PTR_ERR(elem_new);
705                 goto out_unlock;
706         }
707 
708         sock_map_add_link(psock, link, map, elem_new);
709         /* Add new element to the head of the list, so that
710          * concurrent search will find it before old elem.
711          */
712         hlist_add_head_rcu(&elem_new->node, &bucket->head);
713         if (elem) {
714                 hlist_del_rcu(&elem->node);
715                 sock_map_unref(elem->sk, elem);
716                 sock_hash_free_elem(htab, elem);
717         }
718         raw_spin_unlock_bh(&bucket->lock);
719         return 0;
720 out_unlock:
721         raw_spin_unlock_bh(&bucket->lock);
722         sk_psock_put(sk, psock);
723 out_free:
724         sk_psock_free_link(link);
725         return ret;
726 }
727 
728 static int sock_hash_update_elem(struct bpf_map *map, void *key,
729                                  void *value, u64 flags)
730 {
731         u32 ufd = *(u32 *)value;
732         struct socket *sock;
733         struct sock *sk;
734         int ret;
735 
736         sock = sockfd_lookup(ufd, &ret);
737         if (!sock)
738                 return ret;
739         sk = sock->sk;
740         if (!sk) {
741                 ret = -EINVAL;
742                 goto out;
743         }
744         if (!sock_map_sk_is_suitable(sk) ||
745             sk->sk_state != TCP_ESTABLISHED) {
746                 ret = -EOPNOTSUPP;
747                 goto out;
748         }
749 
750         sock_map_sk_acquire(sk);
751         ret = sock_hash_update_common(map, key, sk, flags);
752         sock_map_sk_release(sk);
753 out:
754         fput(sock->file);
755         return ret;
756 }
757 
758 static int sock_hash_get_next_key(struct bpf_map *map, void *key,
759                                   void *key_next)
760 {
761         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
762         struct bpf_htab_elem *elem, *elem_next;
763         u32 hash, key_size = map->key_size;
764         struct hlist_head *head;
765         int i = 0;
766 
767         if (!key)
768                 goto find_first_elem;
769         hash = sock_hash_bucket_hash(key, key_size);
770         head = &sock_hash_select_bucket(htab, hash)->head;
771         elem = sock_hash_lookup_elem_raw(head, hash, key, key_size);
772         if (!elem)
773                 goto find_first_elem;
774 
775         elem_next = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(&elem->node)),
776                                      struct bpf_htab_elem, node);
777         if (elem_next) {
778                 memcpy(key_next, elem_next->key, key_size);
779                 return 0;
780         }
781 
782         i = hash & (htab->buckets_num - 1);
783         i++;
784 find_first_elem:
785         for (; i < htab->buckets_num; i++) {
786                 head = &sock_hash_select_bucket(htab, i)->head;
787                 elem_next = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)),
788                                              struct bpf_htab_elem, node);
789                 if (elem_next) {
790                         memcpy(key_next, elem_next->key, key_size);
791                         return 0;
792                 }
793         }
794 
795         return -ENOENT;
796 }
797 
798 static struct bpf_map *sock_hash_alloc(union bpf_attr *attr)
799 {
800         struct bpf_htab *htab;
801         int i, err;
802         u64 cost;
803 
804         if (!capable(CAP_NET_ADMIN))
805                 return ERR_PTR(-EPERM);
806         if (attr->max_entries == 0 ||
807             attr->key_size    == 0 ||
808             attr->value_size  != 4 ||
809             attr->map_flags & ~SOCK_CREATE_FLAG_MASK)
810                 return ERR_PTR(-EINVAL);
811         if (attr->key_size > MAX_BPF_STACK)
812                 return ERR_PTR(-E2BIG);
813 
814         htab = kzalloc(sizeof(*htab), GFP_USER);
815         if (!htab)
816                 return ERR_PTR(-ENOMEM);
817 
818         bpf_map_init_from_attr(&htab->map, attr);
819 
820         htab->buckets_num = roundup_pow_of_two(htab->map.max_entries);
821         htab->elem_size = sizeof(struct bpf_htab_elem) +
822                           round_up(htab->map.key_size, 8);
823         if (htab->buckets_num == 0 ||
824             htab->buckets_num > U32_MAX / sizeof(struct bpf_htab_bucket)) {
825                 err = -EINVAL;
826                 goto free_htab;
827         }
828 
829         cost = (u64) htab->buckets_num * sizeof(struct bpf_htab_bucket) +
830                (u64) htab->elem_size * htab->map.max_entries;
831         if (cost >= U32_MAX - PAGE_SIZE) {
832                 err = -EINVAL;
833                 goto free_htab;
834         }
835 
836         htab->buckets = bpf_map_area_alloc(htab->buckets_num *
837                                            sizeof(struct bpf_htab_bucket),
838                                            htab->map.numa_node);
839         if (!htab->buckets) {
840                 err = -ENOMEM;
841                 goto free_htab;
842         }
843 
844         for (i = 0; i < htab->buckets_num; i++) {
845                 INIT_HLIST_HEAD(&htab->buckets[i].head);
846                 raw_spin_lock_init(&htab->buckets[i].lock);
847         }
848 
849         return &htab->map;
850 free_htab:
851         kfree(htab);
852         return ERR_PTR(err);
853 }
854 
855 static void sock_hash_free(struct bpf_map *map)
856 {
857         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
858         struct bpf_htab_bucket *bucket;
859         struct bpf_htab_elem *elem;
860         struct hlist_node *node;
861         int i;
862 
863         synchronize_rcu();
864         rcu_read_lock();
865         for (i = 0; i < htab->buckets_num; i++) {
866                 bucket = sock_hash_select_bucket(htab, i);
867                 raw_spin_lock_bh(&bucket->lock);
868                 hlist_for_each_entry_safe(elem, node, &bucket->head, node) {
869                         hlist_del_rcu(&elem->node);
870                         sock_map_unref(elem->sk, elem);
871                 }
872                 raw_spin_unlock_bh(&bucket->lock);
873         }
874         rcu_read_unlock();
875 
876         bpf_map_area_free(htab->buckets);
877         kfree(htab);
878 }
879 
880 static void sock_hash_release_progs(struct bpf_map *map)
881 {
882         psock_progs_drop(&container_of(map, struct bpf_htab, map)->progs);
883 }
884 
885 BPF_CALL_4(bpf_sock_hash_update, struct bpf_sock_ops_kern *, sops,
886            struct bpf_map *, map, void *, key, u64, flags)
887 {
888         WARN_ON_ONCE(!rcu_read_lock_held());
889 
890         if (likely(sock_map_sk_is_suitable(sops->sk) &&
891                    sock_map_op_okay(sops)))
892                 return sock_hash_update_common(map, key, sops->sk, flags);
893         return -EOPNOTSUPP;
894 }
895 
896 const struct bpf_func_proto bpf_sock_hash_update_proto = {
897         .func           = bpf_sock_hash_update,
898         .gpl_only       = false,
899         .pkt_access     = true,
900         .ret_type       = RET_INTEGER,
901         .arg1_type      = ARG_PTR_TO_CTX,
902         .arg2_type      = ARG_CONST_MAP_PTR,
903         .arg3_type      = ARG_PTR_TO_MAP_KEY,
904         .arg4_type      = ARG_ANYTHING,
905 };
906 
907 BPF_CALL_4(bpf_sk_redirect_hash, struct sk_buff *, skb,
908            struct bpf_map *, map, void *, key, u64, flags)
909 {
910         struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
911 
912         if (unlikely(flags & ~(BPF_F_INGRESS)))
913                 return SK_DROP;
914         tcb->bpf.flags = flags;
915         tcb->bpf.sk_redir = __sock_hash_lookup_elem(map, key);
916         if (!tcb->bpf.sk_redir)
917                 return SK_DROP;
918         return SK_PASS;
919 }
920 
921 const struct bpf_func_proto bpf_sk_redirect_hash_proto = {
922         .func           = bpf_sk_redirect_hash,
923         .gpl_only       = false,
924         .ret_type       = RET_INTEGER,
925         .arg1_type      = ARG_PTR_TO_CTX,
926         .arg2_type      = ARG_CONST_MAP_PTR,
927         .arg3_type      = ARG_PTR_TO_MAP_KEY,
928         .arg4_type      = ARG_ANYTHING,
929 };
930 
931 BPF_CALL_4(bpf_msg_redirect_hash, struct sk_msg *, msg,
932            struct bpf_map *, map, void *, key, u64, flags)
933 {
934         if (unlikely(flags & ~(BPF_F_INGRESS)))
935                 return SK_DROP;
936         msg->flags = flags;
937         msg->sk_redir = __sock_hash_lookup_elem(map, key);
938         if (!msg->sk_redir)
939                 return SK_DROP;
940         return SK_PASS;
941 }
942 
943 const struct bpf_func_proto bpf_msg_redirect_hash_proto = {
944         .func           = bpf_msg_redirect_hash,
945         .gpl_only       = false,
946         .ret_type       = RET_INTEGER,
947         .arg1_type      = ARG_PTR_TO_CTX,
948         .arg2_type      = ARG_CONST_MAP_PTR,
949         .arg3_type      = ARG_PTR_TO_MAP_KEY,
950         .arg4_type      = ARG_ANYTHING,
951 };
952 
953 const struct bpf_map_ops sock_hash_ops = {
954         .map_alloc              = sock_hash_alloc,
955         .map_free               = sock_hash_free,
956         .map_get_next_key       = sock_hash_get_next_key,
957         .map_update_elem        = sock_hash_update_elem,
958         .map_delete_elem        = sock_hash_delete_elem,
959         .map_lookup_elem        = sock_map_lookup,
960         .map_release_uref       = sock_hash_release_progs,
961         .map_check_btf          = map_check_no_btf,
962 };
963 
964 static struct sk_psock_progs *sock_map_progs(struct bpf_map *map)
965 {
966         switch (map->map_type) {
967         case BPF_MAP_TYPE_SOCKMAP:
968                 return &container_of(map, struct bpf_stab, map)->progs;
969         case BPF_MAP_TYPE_SOCKHASH:
970                 return &container_of(map, struct bpf_htab, map)->progs;
971         default:
972                 break;
973         }
974 
975         return NULL;
976 }
977 
978 int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog,
979                          u32 which)
980 {
981         struct sk_psock_progs *progs = sock_map_progs(map);
982 
983         if (!progs)
984                 return -EOPNOTSUPP;
985 
986         switch (which) {
987         case BPF_SK_MSG_VERDICT:
988                 psock_set_prog(&progs->msg_parser, prog);
989                 break;
990         case BPF_SK_SKB_STREAM_PARSER:
991                 psock_set_prog(&progs->skb_parser, prog);
992                 break;
993         case BPF_SK_SKB_STREAM_VERDICT:
994                 psock_set_prog(&progs->skb_verdict, prog);
995                 break;
996         default:
997                 return -EOPNOTSUPP;
998         }
999 
1000         return 0;
1001 }
1002 
1003 void sk_psock_unlink(struct sock *sk, struct sk_psock_link *link)
1004 {
1005         switch (link->map->map_type) {
1006         case BPF_MAP_TYPE_SOCKMAP:
1007                 return sock_map_delete_from_link(link->map, sk,
1008                                                  link->link_raw);
1009         case BPF_MAP_TYPE_SOCKHASH:
1010                 return sock_hash_delete_from_link(link->map, sk,
1011                                                   link->link_raw);
1012         default:
1013                 break;
1014         }
1015 }
1016 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp