~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/core/sock_map.c

Version: ~ [ linux-5.4-rc3 ] ~ [ linux-5.3.6 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.79 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.149 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.196 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.196 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.75 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */
  3 
  4 #include <linux/bpf.h>
  5 #include <linux/filter.h>
  6 #include <linux/errno.h>
  7 #include <linux/file.h>
  8 #include <linux/net.h>
  9 #include <linux/workqueue.h>
 10 #include <linux/skmsg.h>
 11 #include <linux/list.h>
 12 #include <linux/jhash.h>
 13 
 14 struct bpf_stab {
 15         struct bpf_map map;
 16         struct sock **sks;
 17         struct sk_psock_progs progs;
 18         raw_spinlock_t lock;
 19 };
 20 
 21 #define SOCK_CREATE_FLAG_MASK                           \
 22         (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY)
 23 
 24 static struct bpf_map *sock_map_alloc(union bpf_attr *attr)
 25 {
 26         struct bpf_stab *stab;
 27         u64 cost;
 28         int err;
 29 
 30         if (!capable(CAP_NET_ADMIN))
 31                 return ERR_PTR(-EPERM);
 32         if (attr->max_entries == 0 ||
 33             attr->key_size    != 4 ||
 34             attr->value_size  != 4 ||
 35             attr->map_flags & ~SOCK_CREATE_FLAG_MASK)
 36                 return ERR_PTR(-EINVAL);
 37 
 38         stab = kzalloc(sizeof(*stab), GFP_USER);
 39         if (!stab)
 40                 return ERR_PTR(-ENOMEM);
 41 
 42         bpf_map_init_from_attr(&stab->map, attr);
 43         raw_spin_lock_init(&stab->lock);
 44 
 45         /* Make sure page count doesn't overflow. */
 46         cost = (u64) stab->map.max_entries * sizeof(struct sock *);
 47         if (cost >= U32_MAX - PAGE_SIZE) {
 48                 err = -EINVAL;
 49                 goto free_stab;
 50         }
 51 
 52         stab->map.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
 53         err = bpf_map_precharge_memlock(stab->map.pages);
 54         if (err)
 55                 goto free_stab;
 56 
 57         stab->sks = bpf_map_area_alloc(stab->map.max_entries *
 58                                        sizeof(struct sock *),
 59                                        stab->map.numa_node);
 60         if (stab->sks)
 61                 return &stab->map;
 62         err = -ENOMEM;
 63 free_stab:
 64         kfree(stab);
 65         return ERR_PTR(err);
 66 }
 67 
 68 int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog)
 69 {
 70         u32 ufd = attr->target_fd;
 71         struct bpf_map *map;
 72         struct fd f;
 73         int ret;
 74 
 75         f = fdget(ufd);
 76         map = __bpf_map_get(f);
 77         if (IS_ERR(map))
 78                 return PTR_ERR(map);
 79         ret = sock_map_prog_update(map, prog, attr->attach_type);
 80         fdput(f);
 81         return ret;
 82 }
 83 
 84 static void sock_map_sk_acquire(struct sock *sk)
 85         __acquires(&sk->sk_lock.slock)
 86 {
 87         lock_sock(sk);
 88         preempt_disable();
 89         rcu_read_lock();
 90 }
 91 
 92 static void sock_map_sk_release(struct sock *sk)
 93         __releases(&sk->sk_lock.slock)
 94 {
 95         rcu_read_unlock();
 96         preempt_enable();
 97         release_sock(sk);
 98 }
 99 
100 static void sock_map_add_link(struct sk_psock *psock,
101                               struct sk_psock_link *link,
102                               struct bpf_map *map, void *link_raw)
103 {
104         link->link_raw = link_raw;
105         link->map = map;
106         spin_lock_bh(&psock->link_lock);
107         list_add_tail(&link->list, &psock->link);
108         spin_unlock_bh(&psock->link_lock);
109 }
110 
111 static void sock_map_del_link(struct sock *sk,
112                               struct sk_psock *psock, void *link_raw)
113 {
114         struct sk_psock_link *link, *tmp;
115         bool strp_stop = false;
116 
117         spin_lock_bh(&psock->link_lock);
118         list_for_each_entry_safe(link, tmp, &psock->link, list) {
119                 if (link->link_raw == link_raw) {
120                         struct bpf_map *map = link->map;
121                         struct bpf_stab *stab = container_of(map, struct bpf_stab,
122                                                              map);
123                         if (psock->parser.enabled && stab->progs.skb_parser)
124                                 strp_stop = true;
125                         list_del(&link->list);
126                         sk_psock_free_link(link);
127                 }
128         }
129         spin_unlock_bh(&psock->link_lock);
130         if (strp_stop) {
131                 write_lock_bh(&sk->sk_callback_lock);
132                 sk_psock_stop_strp(sk, psock);
133                 write_unlock_bh(&sk->sk_callback_lock);
134         }
135 }
136 
137 static void sock_map_unref(struct sock *sk, void *link_raw)
138 {
139         struct sk_psock *psock = sk_psock(sk);
140 
141         if (likely(psock)) {
142                 sock_map_del_link(sk, psock, link_raw);
143                 sk_psock_put(sk, psock);
144         }
145 }
146 
147 static int sock_map_link(struct bpf_map *map, struct sk_psock_progs *progs,
148                          struct sock *sk)
149 {
150         struct bpf_prog *msg_parser, *skb_parser, *skb_verdict;
151         bool skb_progs, sk_psock_is_new = false;
152         struct sk_psock *psock;
153         int ret;
154 
155         skb_verdict = READ_ONCE(progs->skb_verdict);
156         skb_parser = READ_ONCE(progs->skb_parser);
157         skb_progs = skb_parser && skb_verdict;
158         if (skb_progs) {
159                 skb_verdict = bpf_prog_inc_not_zero(skb_verdict);
160                 if (IS_ERR(skb_verdict))
161                         return PTR_ERR(skb_verdict);
162                 skb_parser = bpf_prog_inc_not_zero(skb_parser);
163                 if (IS_ERR(skb_parser)) {
164                         bpf_prog_put(skb_verdict);
165                         return PTR_ERR(skb_parser);
166                 }
167         }
168 
169         msg_parser = READ_ONCE(progs->msg_parser);
170         if (msg_parser) {
171                 msg_parser = bpf_prog_inc_not_zero(msg_parser);
172                 if (IS_ERR(msg_parser)) {
173                         ret = PTR_ERR(msg_parser);
174                         goto out;
175                 }
176         }
177 
178         psock = sk_psock_get_checked(sk);
179         if (IS_ERR(psock)) {
180                 ret = PTR_ERR(psock);
181                 goto out_progs;
182         }
183 
184         if (psock) {
185                 if ((msg_parser && READ_ONCE(psock->progs.msg_parser)) ||
186                     (skb_progs  && READ_ONCE(psock->progs.skb_parser))) {
187                         sk_psock_put(sk, psock);
188                         ret = -EBUSY;
189                         goto out_progs;
190                 }
191         } else {
192                 psock = sk_psock_init(sk, map->numa_node);
193                 if (!psock) {
194                         ret = -ENOMEM;
195                         goto out_progs;
196                 }
197                 sk_psock_is_new = true;
198         }
199 
200         if (msg_parser)
201                 psock_set_prog(&psock->progs.msg_parser, msg_parser);
202         if (sk_psock_is_new) {
203                 ret = tcp_bpf_init(sk);
204                 if (ret < 0)
205                         goto out_drop;
206         } else {
207                 tcp_bpf_reinit(sk);
208         }
209 
210         write_lock_bh(&sk->sk_callback_lock);
211         if (skb_progs && !psock->parser.enabled) {
212                 ret = sk_psock_init_strp(sk, psock);
213                 if (ret) {
214                         write_unlock_bh(&sk->sk_callback_lock);
215                         goto out_drop;
216                 }
217                 psock_set_prog(&psock->progs.skb_verdict, skb_verdict);
218                 psock_set_prog(&psock->progs.skb_parser, skb_parser);
219                 sk_psock_start_strp(sk, psock);
220         }
221         write_unlock_bh(&sk->sk_callback_lock);
222         return 0;
223 out_drop:
224         sk_psock_put(sk, psock);
225 out_progs:
226         if (msg_parser)
227                 bpf_prog_put(msg_parser);
228 out:
229         if (skb_progs) {
230                 bpf_prog_put(skb_verdict);
231                 bpf_prog_put(skb_parser);
232         }
233         return ret;
234 }
235 
236 static void sock_map_free(struct bpf_map *map)
237 {
238         struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
239         int i;
240 
241         synchronize_rcu();
242         rcu_read_lock();
243         raw_spin_lock_bh(&stab->lock);
244         for (i = 0; i < stab->map.max_entries; i++) {
245                 struct sock **psk = &stab->sks[i];
246                 struct sock *sk;
247 
248                 sk = xchg(psk, NULL);
249                 if (sk)
250                         sock_map_unref(sk, psk);
251         }
252         raw_spin_unlock_bh(&stab->lock);
253         rcu_read_unlock();
254 
255         bpf_map_area_free(stab->sks);
256         kfree(stab);
257 }
258 
259 static void sock_map_release_progs(struct bpf_map *map)
260 {
261         psock_progs_drop(&container_of(map, struct bpf_stab, map)->progs);
262 }
263 
264 static struct sock *__sock_map_lookup_elem(struct bpf_map *map, u32 key)
265 {
266         struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
267 
268         WARN_ON_ONCE(!rcu_read_lock_held());
269 
270         if (unlikely(key >= map->max_entries))
271                 return NULL;
272         return READ_ONCE(stab->sks[key]);
273 }
274 
275 static void *sock_map_lookup(struct bpf_map *map, void *key)
276 {
277         return ERR_PTR(-EOPNOTSUPP);
278 }
279 
280 static int __sock_map_delete(struct bpf_stab *stab, struct sock *sk_test,
281                              struct sock **psk)
282 {
283         struct sock *sk;
284 
285         raw_spin_lock_bh(&stab->lock);
286         sk = *psk;
287         if (!sk_test || sk_test == sk)
288                 *psk = NULL;
289         raw_spin_unlock_bh(&stab->lock);
290         if (unlikely(!sk))
291                 return -EINVAL;
292         sock_map_unref(sk, psk);
293         return 0;
294 }
295 
296 static void sock_map_delete_from_link(struct bpf_map *map, struct sock *sk,
297                                       void *link_raw)
298 {
299         struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
300 
301         __sock_map_delete(stab, sk, link_raw);
302 }
303 
304 static int sock_map_delete_elem(struct bpf_map *map, void *key)
305 {
306         struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
307         u32 i = *(u32 *)key;
308         struct sock **psk;
309 
310         if (unlikely(i >= map->max_entries))
311                 return -EINVAL;
312 
313         psk = &stab->sks[i];
314         return __sock_map_delete(stab, NULL, psk);
315 }
316 
317 static int sock_map_get_next_key(struct bpf_map *map, void *key, void *next)
318 {
319         struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
320         u32 i = key ? *(u32 *)key : U32_MAX;
321         u32 *key_next = next;
322 
323         if (i == stab->map.max_entries - 1)
324                 return -ENOENT;
325         if (i >= stab->map.max_entries)
326                 *key_next = 0;
327         else
328                 *key_next = i + 1;
329         return 0;
330 }
331 
332 static int sock_map_update_common(struct bpf_map *map, u32 idx,
333                                   struct sock *sk, u64 flags)
334 {
335         struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
336         struct sk_psock_link *link;
337         struct sk_psock *psock;
338         struct sock *osk;
339         int ret;
340 
341         WARN_ON_ONCE(!rcu_read_lock_held());
342         if (unlikely(flags > BPF_EXIST))
343                 return -EINVAL;
344         if (unlikely(idx >= map->max_entries))
345                 return -E2BIG;
346 
347         link = sk_psock_init_link();
348         if (!link)
349                 return -ENOMEM;
350 
351         ret = sock_map_link(map, &stab->progs, sk);
352         if (ret < 0)
353                 goto out_free;
354 
355         psock = sk_psock(sk);
356         WARN_ON_ONCE(!psock);
357 
358         raw_spin_lock_bh(&stab->lock);
359         osk = stab->sks[idx];
360         if (osk && flags == BPF_NOEXIST) {
361                 ret = -EEXIST;
362                 goto out_unlock;
363         } else if (!osk && flags == BPF_EXIST) {
364                 ret = -ENOENT;
365                 goto out_unlock;
366         }
367 
368         sock_map_add_link(psock, link, map, &stab->sks[idx]);
369         stab->sks[idx] = sk;
370         if (osk)
371                 sock_map_unref(osk, &stab->sks[idx]);
372         raw_spin_unlock_bh(&stab->lock);
373         return 0;
374 out_unlock:
375         raw_spin_unlock_bh(&stab->lock);
376         if (psock)
377                 sk_psock_put(sk, psock);
378 out_free:
379         sk_psock_free_link(link);
380         return ret;
381 }
382 
383 static bool sock_map_op_okay(const struct bpf_sock_ops_kern *ops)
384 {
385         return ops->op == BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB ||
386                ops->op == BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB;
387 }
388 
389 static bool sock_map_sk_is_suitable(const struct sock *sk)
390 {
391         return sk->sk_type == SOCK_STREAM &&
392                sk->sk_protocol == IPPROTO_TCP;
393 }
394 
395 static int sock_map_update_elem(struct bpf_map *map, void *key,
396                                 void *value, u64 flags)
397 {
398         u32 ufd = *(u32 *)value;
399         u32 idx = *(u32 *)key;
400         struct socket *sock;
401         struct sock *sk;
402         int ret;
403 
404         sock = sockfd_lookup(ufd, &ret);
405         if (!sock)
406                 return ret;
407         sk = sock->sk;
408         if (!sk) {
409                 ret = -EINVAL;
410                 goto out;
411         }
412         if (!sock_map_sk_is_suitable(sk) ||
413             sk->sk_state != TCP_ESTABLISHED) {
414                 ret = -EOPNOTSUPP;
415                 goto out;
416         }
417 
418         sock_map_sk_acquire(sk);
419         ret = sock_map_update_common(map, idx, sk, flags);
420         sock_map_sk_release(sk);
421 out:
422         fput(sock->file);
423         return ret;
424 }
425 
426 BPF_CALL_4(bpf_sock_map_update, struct bpf_sock_ops_kern *, sops,
427            struct bpf_map *, map, void *, key, u64, flags)
428 {
429         WARN_ON_ONCE(!rcu_read_lock_held());
430 
431         if (likely(sock_map_sk_is_suitable(sops->sk) &&
432                    sock_map_op_okay(sops)))
433                 return sock_map_update_common(map, *(u32 *)key, sops->sk,
434                                               flags);
435         return -EOPNOTSUPP;
436 }
437 
438 const struct bpf_func_proto bpf_sock_map_update_proto = {
439         .func           = bpf_sock_map_update,
440         .gpl_only       = false,
441         .pkt_access     = true,
442         .ret_type       = RET_INTEGER,
443         .arg1_type      = ARG_PTR_TO_CTX,
444         .arg2_type      = ARG_CONST_MAP_PTR,
445         .arg3_type      = ARG_PTR_TO_MAP_KEY,
446         .arg4_type      = ARG_ANYTHING,
447 };
448 
449 BPF_CALL_4(bpf_sk_redirect_map, struct sk_buff *, skb,
450            struct bpf_map *, map, u32, key, u64, flags)
451 {
452         struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
453 
454         if (unlikely(flags & ~(BPF_F_INGRESS)))
455                 return SK_DROP;
456         tcb->bpf.flags = flags;
457         tcb->bpf.sk_redir = __sock_map_lookup_elem(map, key);
458         if (!tcb->bpf.sk_redir)
459                 return SK_DROP;
460         return SK_PASS;
461 }
462 
463 const struct bpf_func_proto bpf_sk_redirect_map_proto = {
464         .func           = bpf_sk_redirect_map,
465         .gpl_only       = false,
466         .ret_type       = RET_INTEGER,
467         .arg1_type      = ARG_PTR_TO_CTX,
468         .arg2_type      = ARG_CONST_MAP_PTR,
469         .arg3_type      = ARG_ANYTHING,
470         .arg4_type      = ARG_ANYTHING,
471 };
472 
473 BPF_CALL_4(bpf_msg_redirect_map, struct sk_msg *, msg,
474            struct bpf_map *, map, u32, key, u64, flags)
475 {
476         if (unlikely(flags & ~(BPF_F_INGRESS)))
477                 return SK_DROP;
478         msg->flags = flags;
479         msg->sk_redir = __sock_map_lookup_elem(map, key);
480         if (!msg->sk_redir)
481                 return SK_DROP;
482         return SK_PASS;
483 }
484 
485 const struct bpf_func_proto bpf_msg_redirect_map_proto = {
486         .func           = bpf_msg_redirect_map,
487         .gpl_only       = false,
488         .ret_type       = RET_INTEGER,
489         .arg1_type      = ARG_PTR_TO_CTX,
490         .arg2_type      = ARG_CONST_MAP_PTR,
491         .arg3_type      = ARG_ANYTHING,
492         .arg4_type      = ARG_ANYTHING,
493 };
494 
495 const struct bpf_map_ops sock_map_ops = {
496         .map_alloc              = sock_map_alloc,
497         .map_free               = sock_map_free,
498         .map_get_next_key       = sock_map_get_next_key,
499         .map_update_elem        = sock_map_update_elem,
500         .map_delete_elem        = sock_map_delete_elem,
501         .map_lookup_elem        = sock_map_lookup,
502         .map_release_uref       = sock_map_release_progs,
503         .map_check_btf          = map_check_no_btf,
504 };
505 
506 struct bpf_htab_elem {
507         struct rcu_head rcu;
508         u32 hash;
509         struct sock *sk;
510         struct hlist_node node;
511         u8 key[0];
512 };
513 
514 struct bpf_htab_bucket {
515         struct hlist_head head;
516         raw_spinlock_t lock;
517 };
518 
519 struct bpf_htab {
520         struct bpf_map map;
521         struct bpf_htab_bucket *buckets;
522         u32 buckets_num;
523         u32 elem_size;
524         struct sk_psock_progs progs;
525         atomic_t count;
526 };
527 
528 static inline u32 sock_hash_bucket_hash(const void *key, u32 len)
529 {
530         return jhash(key, len, 0);
531 }
532 
533 static struct bpf_htab_bucket *sock_hash_select_bucket(struct bpf_htab *htab,
534                                                        u32 hash)
535 {
536         return &htab->buckets[hash & (htab->buckets_num - 1)];
537 }
538 
539 static struct bpf_htab_elem *
540 sock_hash_lookup_elem_raw(struct hlist_head *head, u32 hash, void *key,
541                           u32 key_size)
542 {
543         struct bpf_htab_elem *elem;
544 
545         hlist_for_each_entry_rcu(elem, head, node) {
546                 if (elem->hash == hash &&
547                     !memcmp(&elem->key, key, key_size))
548                         return elem;
549         }
550 
551         return NULL;
552 }
553 
554 static struct sock *__sock_hash_lookup_elem(struct bpf_map *map, void *key)
555 {
556         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
557         u32 key_size = map->key_size, hash;
558         struct bpf_htab_bucket *bucket;
559         struct bpf_htab_elem *elem;
560 
561         WARN_ON_ONCE(!rcu_read_lock_held());
562 
563         hash = sock_hash_bucket_hash(key, key_size);
564         bucket = sock_hash_select_bucket(htab, hash);
565         elem = sock_hash_lookup_elem_raw(&bucket->head, hash, key, key_size);
566 
567         return elem ? elem->sk : NULL;
568 }
569 
570 static void sock_hash_free_elem(struct bpf_htab *htab,
571                                 struct bpf_htab_elem *elem)
572 {
573         atomic_dec(&htab->count);
574         kfree_rcu(elem, rcu);
575 }
576 
577 static void sock_hash_delete_from_link(struct bpf_map *map, struct sock *sk,
578                                        void *link_raw)
579 {
580         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
581         struct bpf_htab_elem *elem_probe, *elem = link_raw;
582         struct bpf_htab_bucket *bucket;
583 
584         WARN_ON_ONCE(!rcu_read_lock_held());
585         bucket = sock_hash_select_bucket(htab, elem->hash);
586 
587         /* elem may be deleted in parallel from the map, but access here
588          * is okay since it's going away only after RCU grace period.
589          * However, we need to check whether it's still present.
590          */
591         raw_spin_lock_bh(&bucket->lock);
592         elem_probe = sock_hash_lookup_elem_raw(&bucket->head, elem->hash,
593                                                elem->key, map->key_size);
594         if (elem_probe && elem_probe == elem) {
595                 hlist_del_rcu(&elem->node);
596                 sock_map_unref(elem->sk, elem);
597                 sock_hash_free_elem(htab, elem);
598         }
599         raw_spin_unlock_bh(&bucket->lock);
600 }
601 
602 static int sock_hash_delete_elem(struct bpf_map *map, void *key)
603 {
604         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
605         u32 hash, key_size = map->key_size;
606         struct bpf_htab_bucket *bucket;
607         struct bpf_htab_elem *elem;
608         int ret = -ENOENT;
609 
610         hash = sock_hash_bucket_hash(key, key_size);
611         bucket = sock_hash_select_bucket(htab, hash);
612 
613         raw_spin_lock_bh(&bucket->lock);
614         elem = sock_hash_lookup_elem_raw(&bucket->head, hash, key, key_size);
615         if (elem) {
616                 hlist_del_rcu(&elem->node);
617                 sock_map_unref(elem->sk, elem);
618                 sock_hash_free_elem(htab, elem);
619                 ret = 0;
620         }
621         raw_spin_unlock_bh(&bucket->lock);
622         return ret;
623 }
624 
625 static struct bpf_htab_elem *sock_hash_alloc_elem(struct bpf_htab *htab,
626                                                   void *key, u32 key_size,
627                                                   u32 hash, struct sock *sk,
628                                                   struct bpf_htab_elem *old)
629 {
630         struct bpf_htab_elem *new;
631 
632         if (atomic_inc_return(&htab->count) > htab->map.max_entries) {
633                 if (!old) {
634                         atomic_dec(&htab->count);
635                         return ERR_PTR(-E2BIG);
636                 }
637         }
638 
639         new = kmalloc_node(htab->elem_size, GFP_ATOMIC | __GFP_NOWARN,
640                            htab->map.numa_node);
641         if (!new) {
642                 atomic_dec(&htab->count);
643                 return ERR_PTR(-ENOMEM);
644         }
645         memcpy(new->key, key, key_size);
646         new->sk = sk;
647         new->hash = hash;
648         return new;
649 }
650 
651 static int sock_hash_update_common(struct bpf_map *map, void *key,
652                                    struct sock *sk, u64 flags)
653 {
654         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
655         u32 key_size = map->key_size, hash;
656         struct bpf_htab_elem *elem, *elem_new;
657         struct bpf_htab_bucket *bucket;
658         struct sk_psock_link *link;
659         struct sk_psock *psock;
660         int ret;
661 
662         WARN_ON_ONCE(!rcu_read_lock_held());
663         if (unlikely(flags > BPF_EXIST))
664                 return -EINVAL;
665 
666         link = sk_psock_init_link();
667         if (!link)
668                 return -ENOMEM;
669 
670         ret = sock_map_link(map, &htab->progs, sk);
671         if (ret < 0)
672                 goto out_free;
673 
674         psock = sk_psock(sk);
675         WARN_ON_ONCE(!psock);
676 
677         hash = sock_hash_bucket_hash(key, key_size);
678         bucket = sock_hash_select_bucket(htab, hash);
679 
680         raw_spin_lock_bh(&bucket->lock);
681         elem = sock_hash_lookup_elem_raw(&bucket->head, hash, key, key_size);
682         if (elem && flags == BPF_NOEXIST) {
683                 ret = -EEXIST;
684                 goto out_unlock;
685         } else if (!elem && flags == BPF_EXIST) {
686                 ret = -ENOENT;
687                 goto out_unlock;
688         }
689 
690         elem_new = sock_hash_alloc_elem(htab, key, key_size, hash, sk, elem);
691         if (IS_ERR(elem_new)) {
692                 ret = PTR_ERR(elem_new);
693                 goto out_unlock;
694         }
695 
696         sock_map_add_link(psock, link, map, elem_new);
697         /* Add new element to the head of the list, so that
698          * concurrent search will find it before old elem.
699          */
700         hlist_add_head_rcu(&elem_new->node, &bucket->head);
701         if (elem) {
702                 hlist_del_rcu(&elem->node);
703                 sock_map_unref(elem->sk, elem);
704                 sock_hash_free_elem(htab, elem);
705         }
706         raw_spin_unlock_bh(&bucket->lock);
707         return 0;
708 out_unlock:
709         raw_spin_unlock_bh(&bucket->lock);
710         sk_psock_put(sk, psock);
711 out_free:
712         sk_psock_free_link(link);
713         return ret;
714 }
715 
716 static int sock_hash_update_elem(struct bpf_map *map, void *key,
717                                  void *value, u64 flags)
718 {
719         u32 ufd = *(u32 *)value;
720         struct socket *sock;
721         struct sock *sk;
722         int ret;
723 
724         sock = sockfd_lookup(ufd, &ret);
725         if (!sock)
726                 return ret;
727         sk = sock->sk;
728         if (!sk) {
729                 ret = -EINVAL;
730                 goto out;
731         }
732         if (!sock_map_sk_is_suitable(sk) ||
733             sk->sk_state != TCP_ESTABLISHED) {
734                 ret = -EOPNOTSUPP;
735                 goto out;
736         }
737 
738         sock_map_sk_acquire(sk);
739         ret = sock_hash_update_common(map, key, sk, flags);
740         sock_map_sk_release(sk);
741 out:
742         fput(sock->file);
743         return ret;
744 }
745 
746 static int sock_hash_get_next_key(struct bpf_map *map, void *key,
747                                   void *key_next)
748 {
749         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
750         struct bpf_htab_elem *elem, *elem_next;
751         u32 hash, key_size = map->key_size;
752         struct hlist_head *head;
753         int i = 0;
754 
755         if (!key)
756                 goto find_first_elem;
757         hash = sock_hash_bucket_hash(key, key_size);
758         head = &sock_hash_select_bucket(htab, hash)->head;
759         elem = sock_hash_lookup_elem_raw(head, hash, key, key_size);
760         if (!elem)
761                 goto find_first_elem;
762 
763         elem_next = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(&elem->node)),
764                                      struct bpf_htab_elem, node);
765         if (elem_next) {
766                 memcpy(key_next, elem_next->key, key_size);
767                 return 0;
768         }
769 
770         i = hash & (htab->buckets_num - 1);
771         i++;
772 find_first_elem:
773         for (; i < htab->buckets_num; i++) {
774                 head = &sock_hash_select_bucket(htab, i)->head;
775                 elem_next = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)),
776                                              struct bpf_htab_elem, node);
777                 if (elem_next) {
778                         memcpy(key_next, elem_next->key, key_size);
779                         return 0;
780                 }
781         }
782 
783         return -ENOENT;
784 }
785 
786 static struct bpf_map *sock_hash_alloc(union bpf_attr *attr)
787 {
788         struct bpf_htab *htab;
789         int i, err;
790         u64 cost;
791 
792         if (!capable(CAP_NET_ADMIN))
793                 return ERR_PTR(-EPERM);
794         if (attr->max_entries == 0 ||
795             attr->key_size    == 0 ||
796             attr->value_size  != 4 ||
797             attr->map_flags & ~SOCK_CREATE_FLAG_MASK)
798                 return ERR_PTR(-EINVAL);
799         if (attr->key_size > MAX_BPF_STACK)
800                 return ERR_PTR(-E2BIG);
801 
802         htab = kzalloc(sizeof(*htab), GFP_USER);
803         if (!htab)
804                 return ERR_PTR(-ENOMEM);
805 
806         bpf_map_init_from_attr(&htab->map, attr);
807 
808         htab->buckets_num = roundup_pow_of_two(htab->map.max_entries);
809         htab->elem_size = sizeof(struct bpf_htab_elem) +
810                           round_up(htab->map.key_size, 8);
811         if (htab->buckets_num == 0 ||
812             htab->buckets_num > U32_MAX / sizeof(struct bpf_htab_bucket)) {
813                 err = -EINVAL;
814                 goto free_htab;
815         }
816 
817         cost = (u64) htab->buckets_num * sizeof(struct bpf_htab_bucket) +
818                (u64) htab->elem_size * htab->map.max_entries;
819         if (cost >= U32_MAX - PAGE_SIZE) {
820                 err = -EINVAL;
821                 goto free_htab;
822         }
823 
824         htab->buckets = bpf_map_area_alloc(htab->buckets_num *
825                                            sizeof(struct bpf_htab_bucket),
826                                            htab->map.numa_node);
827         if (!htab->buckets) {
828                 err = -ENOMEM;
829                 goto free_htab;
830         }
831 
832         for (i = 0; i < htab->buckets_num; i++) {
833                 INIT_HLIST_HEAD(&htab->buckets[i].head);
834                 raw_spin_lock_init(&htab->buckets[i].lock);
835         }
836 
837         return &htab->map;
838 free_htab:
839         kfree(htab);
840         return ERR_PTR(err);
841 }
842 
843 static void sock_hash_free(struct bpf_map *map)
844 {
845         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
846         struct bpf_htab_bucket *bucket;
847         struct bpf_htab_elem *elem;
848         struct hlist_node *node;
849         int i;
850 
851         synchronize_rcu();
852         rcu_read_lock();
853         for (i = 0; i < htab->buckets_num; i++) {
854                 bucket = sock_hash_select_bucket(htab, i);
855                 raw_spin_lock_bh(&bucket->lock);
856                 hlist_for_each_entry_safe(elem, node, &bucket->head, node) {
857                         hlist_del_rcu(&elem->node);
858                         sock_map_unref(elem->sk, elem);
859                 }
860                 raw_spin_unlock_bh(&bucket->lock);
861         }
862         rcu_read_unlock();
863 
864         bpf_map_area_free(htab->buckets);
865         kfree(htab);
866 }
867 
868 static void sock_hash_release_progs(struct bpf_map *map)
869 {
870         psock_progs_drop(&container_of(map, struct bpf_htab, map)->progs);
871 }
872 
873 BPF_CALL_4(bpf_sock_hash_update, struct bpf_sock_ops_kern *, sops,
874            struct bpf_map *, map, void *, key, u64, flags)
875 {
876         WARN_ON_ONCE(!rcu_read_lock_held());
877 
878         if (likely(sock_map_sk_is_suitable(sops->sk) &&
879                    sock_map_op_okay(sops)))
880                 return sock_hash_update_common(map, key, sops->sk, flags);
881         return -EOPNOTSUPP;
882 }
883 
884 const struct bpf_func_proto bpf_sock_hash_update_proto = {
885         .func           = bpf_sock_hash_update,
886         .gpl_only       = false,
887         .pkt_access     = true,
888         .ret_type       = RET_INTEGER,
889         .arg1_type      = ARG_PTR_TO_CTX,
890         .arg2_type      = ARG_CONST_MAP_PTR,
891         .arg3_type      = ARG_PTR_TO_MAP_KEY,
892         .arg4_type      = ARG_ANYTHING,
893 };
894 
895 BPF_CALL_4(bpf_sk_redirect_hash, struct sk_buff *, skb,
896            struct bpf_map *, map, void *, key, u64, flags)
897 {
898         struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
899 
900         if (unlikely(flags & ~(BPF_F_INGRESS)))
901                 return SK_DROP;
902         tcb->bpf.flags = flags;
903         tcb->bpf.sk_redir = __sock_hash_lookup_elem(map, key);
904         if (!tcb->bpf.sk_redir)
905                 return SK_DROP;
906         return SK_PASS;
907 }
908 
909 const struct bpf_func_proto bpf_sk_redirect_hash_proto = {
910         .func           = bpf_sk_redirect_hash,
911         .gpl_only       = false,
912         .ret_type       = RET_INTEGER,
913         .arg1_type      = ARG_PTR_TO_CTX,
914         .arg2_type      = ARG_CONST_MAP_PTR,
915         .arg3_type      = ARG_PTR_TO_MAP_KEY,
916         .arg4_type      = ARG_ANYTHING,
917 };
918 
919 BPF_CALL_4(bpf_msg_redirect_hash, struct sk_msg *, msg,
920            struct bpf_map *, map, void *, key, u64, flags)
921 {
922         if (unlikely(flags & ~(BPF_F_INGRESS)))
923                 return SK_DROP;
924         msg->flags = flags;
925         msg->sk_redir = __sock_hash_lookup_elem(map, key);
926         if (!msg->sk_redir)
927                 return SK_DROP;
928         return SK_PASS;
929 }
930 
931 const struct bpf_func_proto bpf_msg_redirect_hash_proto = {
932         .func           = bpf_msg_redirect_hash,
933         .gpl_only       = false,
934         .ret_type       = RET_INTEGER,
935         .arg1_type      = ARG_PTR_TO_CTX,
936         .arg2_type      = ARG_CONST_MAP_PTR,
937         .arg3_type      = ARG_PTR_TO_MAP_KEY,
938         .arg4_type      = ARG_ANYTHING,
939 };
940 
941 const struct bpf_map_ops sock_hash_ops = {
942         .map_alloc              = sock_hash_alloc,
943         .map_free               = sock_hash_free,
944         .map_get_next_key       = sock_hash_get_next_key,
945         .map_update_elem        = sock_hash_update_elem,
946         .map_delete_elem        = sock_hash_delete_elem,
947         .map_lookup_elem        = sock_map_lookup,
948         .map_release_uref       = sock_hash_release_progs,
949         .map_check_btf          = map_check_no_btf,
950 };
951 
952 static struct sk_psock_progs *sock_map_progs(struct bpf_map *map)
953 {
954         switch (map->map_type) {
955         case BPF_MAP_TYPE_SOCKMAP:
956                 return &container_of(map, struct bpf_stab, map)->progs;
957         case BPF_MAP_TYPE_SOCKHASH:
958                 return &container_of(map, struct bpf_htab, map)->progs;
959         default:
960                 break;
961         }
962 
963         return NULL;
964 }
965 
966 int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog,
967                          u32 which)
968 {
969         struct sk_psock_progs *progs = sock_map_progs(map);
970 
971         if (!progs)
972                 return -EOPNOTSUPP;
973 
974         switch (which) {
975         case BPF_SK_MSG_VERDICT:
976                 psock_set_prog(&progs->msg_parser, prog);
977                 break;
978         case BPF_SK_SKB_STREAM_PARSER:
979                 psock_set_prog(&progs->skb_parser, prog);
980                 break;
981         case BPF_SK_SKB_STREAM_VERDICT:
982                 psock_set_prog(&progs->skb_verdict, prog);
983                 break;
984         default:
985                 return -EOPNOTSUPP;
986         }
987 
988         return 0;
989 }
990 
991 void sk_psock_unlink(struct sock *sk, struct sk_psock_link *link)
992 {
993         switch (link->map->map_type) {
994         case BPF_MAP_TYPE_SOCKMAP:
995                 return sock_map_delete_from_link(link->map, sk,
996                                                  link->link_raw);
997         case BPF_MAP_TYPE_SOCKHASH:
998                 return sock_hash_delete_from_link(link->map, sk,
999                                                   link->link_raw);
1000         default:
1001                 break;
1002         }
1003 }
1004 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp