1 /*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Generic socket support routines. Memory allocators, socket lock/release
7 * handler for protocols to use and generic option handler.
8 *
9 *
10 * Authors: Ross Biro
11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Florian La Roche, <flla@stud.uni-sb.de>
13 * Alan Cox, <A.Cox@swansea.ac.uk>
14 *
15 * Fixes:
16 * Alan Cox : Numerous verify_area() problems
17 * Alan Cox : Connecting on a connecting socket
18 * now returns an error for tcp.
19 * Alan Cox : sock->protocol is set correctly.
20 * and is not sometimes left as 0.
21 * Alan Cox : connect handles icmp errors on a
22 * connect properly. Unfortunately there
23 * is a restart syscall nasty there. I
24 * can't match BSD without hacking the C
25 * library. Ideas urgently sought!
26 * Alan Cox : Disallow bind() to addresses that are
27 * not ours - especially broadcast ones!!
28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost)
29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets,
30 * instead they leave that for the DESTROY timer.
31 * Alan Cox : Clean up error flag in accept
32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer
33 * was buggy. Put a remove_sock() in the handler
34 * for memory when we hit 0. Also altered the timer
35 * code. The ACK stuff can wait and needs major
36 * TCP layer surgery.
37 * Alan Cox : Fixed TCP ack bug, removed remove sock
38 * and fixed timer/inet_bh race.
39 * Alan Cox : Added zapped flag for TCP
40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code
41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb
42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources
43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing.
44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so...
45 * Rick Sladkey : Relaxed UDP rules for matching packets.
46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support
47 * Pauline Middelink : identd support
48 * Alan Cox : Fixed connect() taking signals I think.
49 * Alan Cox : SO_LINGER supported
50 * Alan Cox : Error reporting fixes
51 * Anonymous : inet_create tidied up (sk->reuse setting)
52 * Alan Cox : inet sockets don't set sk->type!
53 * Alan Cox : Split socket option code
54 * Alan Cox : Callbacks
55 * Alan Cox : Nagle flag for Charles & Johannes stuff
56 * Alex : Removed restriction on inet fioctl
57 * Alan Cox : Splitting INET from NET core
58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt()
59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code
60 * Alan Cox : Split IP from generic code
61 * Alan Cox : New kfree_skbmem()
62 * Alan Cox : Make SO_DEBUG superuser only.
63 * Alan Cox : Allow anyone to clear SO_DEBUG
64 * (compatibility fix)
65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput.
66 * Alan Cox : Allocator for a socket is settable.
67 * Alan Cox : SO_ERROR includes soft errors.
68 * Alan Cox : Allow NULL arguments on some SO_ opts
69 * Alan Cox : Generic socket allocation to make hooks
70 * easier (suggested by Craig Metz).
71 * Michael Pall : SO_ERROR returns positive errno again
72 * Steve Whitehouse: Added default destructor to free
73 * protocol private data.
74 * Steve Whitehouse: Added various other default routines
75 * common to several socket families.
76 * Chris Evans : Call suser() check last on F_SETOWN
77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER.
78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s()
79 * Andi Kleen : Fix write_space callback
80 * Chris Evans : Security fixes - signedness again
81 * Arnaldo C. Melo : cleanups, use skb_queue_purge
82 *
83 * To Fix:
84 *
85 *
86 * This program is free software; you can redistribute it and/or
87 * modify it under the terms of the GNU General Public License
88 * as published by the Free Software Foundation; either version
89 * 2 of the License, or (at your option) any later version.
90 */
91
92 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
93
94 #include <linux/capability.h>
95 #include <linux/errno.h>
96 #include <linux/errqueue.h>
97 #include <linux/types.h>
98 #include <linux/socket.h>
99 #include <linux/in.h>
100 #include <linux/kernel.h>
101 #include <linux/module.h>
102 #include <linux/proc_fs.h>
103 #include <linux/seq_file.h>
104 #include <linux/sched.h>
105 #include <linux/timer.h>
106 #include <linux/string.h>
107 #include <linux/sockios.h>
108 #include <linux/net.h>
109 #include <linux/mm.h>
110 #include <linux/slab.h>
111 #include <linux/interrupt.h>
112 #include <linux/poll.h>
113 #include <linux/tcp.h>
114 #include <linux/init.h>
115 #include <linux/highmem.h>
116 #include <linux/user_namespace.h>
117 #include <linux/static_key.h>
118 #include <linux/memcontrol.h>
119 #include <linux/prefetch.h>
120
121 #include <asm/uaccess.h>
122
123 #include <linux/netdevice.h>
124 #include <net/protocol.h>
125 #include <linux/skbuff.h>
126 #include <net/net_namespace.h>
127 #include <net/request_sock.h>
128 #include <net/sock.h>
129 #include <linux/net_tstamp.h>
130 #include <net/xfrm.h>
131 #include <linux/ipsec.h>
132 #include <net/cls_cgroup.h>
133 #include <net/netprio_cgroup.h>
134 #include <linux/sock_diag.h>
135
136 #include <linux/filter.h>
137 #include <net/sock_reuseport.h>
138
139 #include <trace/events/sock.h>
140
141 #ifdef CONFIG_INET
142 #include <net/tcp.h>
143 #endif
144
145 #include <net/busy_poll.h>
146
147 static DEFINE_MUTEX(proto_list_mutex);
148 static LIST_HEAD(proto_list);
149
150 /**
151 * sk_ns_capable - General socket capability test
152 * @sk: Socket to use a capability on or through
153 * @user_ns: The user namespace of the capability to use
154 * @cap: The capability to use
155 *
156 * Test to see if the opener of the socket had when the socket was
157 * created and the current process has the capability @cap in the user
158 * namespace @user_ns.
159 */
160 bool sk_ns_capable(const struct sock *sk,
161 struct user_namespace *user_ns, int cap)
162 {
163 return file_ns_capable(sk->sk_socket->file, user_ns, cap) &&
164 ns_capable(user_ns, cap);
165 }
166 EXPORT_SYMBOL(sk_ns_capable);
167
168 /**
169 * sk_capable - Socket global capability test
170 * @sk: Socket to use a capability on or through
171 * @cap: The global capability to use
172 *
173 * Test to see if the opener of the socket had when the socket was
174 * created and the current process has the capability @cap in all user
175 * namespaces.
176 */
177 bool sk_capable(const struct sock *sk, int cap)
178 {
179 return sk_ns_capable(sk, &init_user_ns, cap);
180 }
181 EXPORT_SYMBOL(sk_capable);
182
183 /**
184 * sk_net_capable - Network namespace socket capability test
185 * @sk: Socket to use a capability on or through
186 * @cap: The capability to use
187 *
188 * Test to see if the opener of the socket had when the socket was created
189 * and the current process has the capability @cap over the network namespace
190 * the socket is a member of.
191 */
192 bool sk_net_capable(const struct sock *sk, int cap)
193 {
194 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap);
195 }
196 EXPORT_SYMBOL(sk_net_capable);
197
198 /*
199 * Each address family might have different locking rules, so we have
200 * one slock key per address family:
201 */
202 static struct lock_class_key af_family_keys[AF_MAX];
203 static struct lock_class_key af_family_slock_keys[AF_MAX];
204
205 /*
206 * Make lock validator output more readable. (we pre-construct these
207 * strings build-time, so that runtime initialization of socket
208 * locks is fast):
209 */
210 static const char *const af_family_key_strings[AF_MAX+1] = {
211 "sk_lock-AF_UNSPEC", "sk_lock-AF_UNIX" , "sk_lock-AF_INET" ,
212 "sk_lock-AF_AX25" , "sk_lock-AF_IPX" , "sk_lock-AF_APPLETALK",
213 "sk_lock-AF_NETROM", "sk_lock-AF_BRIDGE" , "sk_lock-AF_ATMPVC" ,
214 "sk_lock-AF_X25" , "sk_lock-AF_INET6" , "sk_lock-AF_ROSE" ,
215 "sk_lock-AF_DECnet", "sk_lock-AF_NETBEUI" , "sk_lock-AF_SECURITY" ,
216 "sk_lock-AF_KEY" , "sk_lock-AF_NETLINK" , "sk_lock-AF_PACKET" ,
217 "sk_lock-AF_ASH" , "sk_lock-AF_ECONET" , "sk_lock-AF_ATMSVC" ,
218 "sk_lock-AF_RDS" , "sk_lock-AF_SNA" , "sk_lock-AF_IRDA" ,
219 "sk_lock-AF_PPPOX" , "sk_lock-AF_WANPIPE" , "sk_lock-AF_LLC" ,
220 "sk_lock-27" , "sk_lock-28" , "sk_lock-AF_CAN" ,
221 "sk_lock-AF_TIPC" , "sk_lock-AF_BLUETOOTH", "sk_lock-IUCV" ,
222 "sk_lock-AF_RXRPC" , "sk_lock-AF_ISDN" , "sk_lock-AF_PHONET" ,
223 "sk_lock-AF_IEEE802154", "sk_lock-AF_CAIF" , "sk_lock-AF_ALG" ,
224 "sk_lock-AF_NFC" , "sk_lock-AF_VSOCK" , "sk_lock-AF_KCM" ,
225 "sk_lock-AF_MAX"
226 };
227 static const char *const af_family_slock_key_strings[AF_MAX+1] = {
228 "slock-AF_UNSPEC", "slock-AF_UNIX" , "slock-AF_INET" ,
229 "slock-AF_AX25" , "slock-AF_IPX" , "slock-AF_APPLETALK",
230 "slock-AF_NETROM", "slock-AF_BRIDGE" , "slock-AF_ATMPVC" ,
231 "slock-AF_X25" , "slock-AF_INET6" , "slock-AF_ROSE" ,
232 "slock-AF_DECnet", "slock-AF_NETBEUI" , "slock-AF_SECURITY" ,
233 "slock-AF_KEY" , "slock-AF_NETLINK" , "slock-AF_PACKET" ,
234 "slock-AF_ASH" , "slock-AF_ECONET" , "slock-AF_ATMSVC" ,
235 "slock-AF_RDS" , "slock-AF_SNA" , "slock-AF_IRDA" ,
236 "slock-AF_PPPOX" , "slock-AF_WANPIPE" , "slock-AF_LLC" ,
237 "slock-27" , "slock-28" , "slock-AF_CAN" ,
238 "slock-AF_TIPC" , "slock-AF_BLUETOOTH", "slock-AF_IUCV" ,
239 "slock-AF_RXRPC" , "slock-AF_ISDN" , "slock-AF_PHONET" ,
240 "slock-AF_IEEE802154", "slock-AF_CAIF" , "slock-AF_ALG" ,
241 "slock-AF_NFC" , "slock-AF_VSOCK" ,"slock-AF_KCM" ,
242 "slock-AF_MAX"
243 };
244 static const char *const af_family_clock_key_strings[AF_MAX+1] = {
245 "clock-AF_UNSPEC", "clock-AF_UNIX" , "clock-AF_INET" ,
246 "clock-AF_AX25" , "clock-AF_IPX" , "clock-AF_APPLETALK",
247 "clock-AF_NETROM", "clock-AF_BRIDGE" , "clock-AF_ATMPVC" ,
248 "clock-AF_X25" , "clock-AF_INET6" , "clock-AF_ROSE" ,
249 "clock-AF_DECnet", "clock-AF_NETBEUI" , "clock-AF_SECURITY" ,
250 "clock-AF_KEY" , "clock-AF_NETLINK" , "clock-AF_PACKET" ,
251 "clock-AF_ASH" , "clock-AF_ECONET" , "clock-AF_ATMSVC" ,
252 "clock-AF_RDS" , "clock-AF_SNA" , "clock-AF_IRDA" ,
253 "clock-AF_PPPOX" , "clock-AF_WANPIPE" , "clock-AF_LLC" ,
254 "clock-27" , "clock-28" , "clock-AF_CAN" ,
255 "clock-AF_TIPC" , "clock-AF_BLUETOOTH", "clock-AF_IUCV" ,
256 "clock-AF_RXRPC" , "clock-AF_ISDN" , "clock-AF_PHONET" ,
257 "clock-AF_IEEE802154", "clock-AF_CAIF" , "clock-AF_ALG" ,
258 "clock-AF_NFC" , "clock-AF_VSOCK" , "clock-AF_KCM" ,
259 "clock-AF_MAX"
260 };
261
262 /*
263 * sk_callback_lock locking rules are per-address-family,
264 * so split the lock classes by using a per-AF key:
265 */
266 static struct lock_class_key af_callback_keys[AF_MAX];
267
268 /* Take into consideration the size of the struct sk_buff overhead in the
269 * determination of these values, since that is non-constant across
270 * platforms. This makes socket queueing behavior and performance
271 * not depend upon such differences.
272 */
273 #define _SK_MEM_PACKETS 256
274 #define _SK_MEM_OVERHEAD SKB_TRUESIZE(256)
275 #define SK_WMEM_MAX (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
276 #define SK_RMEM_MAX (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
277
278 /* Run time adjustable parameters. */
279 __u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX;
280 EXPORT_SYMBOL(sysctl_wmem_max);
281 __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX;
282 EXPORT_SYMBOL(sysctl_rmem_max);
283 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX;
284 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX;
285
286 /* Maximal space eaten by iovec or ancillary data plus some space */
287 int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512);
288 EXPORT_SYMBOL(sysctl_optmem_max);
289
290 int sysctl_tstamp_allow_data __read_mostly = 1;
291
292 struct static_key memalloc_socks = STATIC_KEY_INIT_FALSE;
293 EXPORT_SYMBOL_GPL(memalloc_socks);
294
295 /**
296 * sk_set_memalloc - sets %SOCK_MEMALLOC
297 * @sk: socket to set it on
298 *
299 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves.
300 * It's the responsibility of the admin to adjust min_free_kbytes
301 * to meet the requirements
302 */
303 void sk_set_memalloc(struct sock *sk)
304 {
305 sock_set_flag(sk, SOCK_MEMALLOC);
306 sk->sk_allocation |= __GFP_MEMALLOC;
307 static_key_slow_inc(&memalloc_socks);
308 }
309 EXPORT_SYMBOL_GPL(sk_set_memalloc);
310
311 void sk_clear_memalloc(struct sock *sk)
312 {
313 sock_reset_flag(sk, SOCK_MEMALLOC);
314 sk->sk_allocation &= ~__GFP_MEMALLOC;
315 static_key_slow_dec(&memalloc_socks);
316
317 /*
318 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward
319 * progress of swapping. SOCK_MEMALLOC may be cleared while
320 * it has rmem allocations due to the last swapfile being deactivated
321 * but there is a risk that the socket is unusable due to exceeding
322 * the rmem limits. Reclaim the reserves and obey rmem limits again.
323 */
324 sk_mem_reclaim(sk);
325 }
326 EXPORT_SYMBOL_GPL(sk_clear_memalloc);
327
328 int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
329 {
330 int ret;
331 unsigned long pflags = current->flags;
332
333 /* these should have been dropped before queueing */
334 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC));
335
336 current->flags |= PF_MEMALLOC;
337 ret = sk->sk_backlog_rcv(sk, skb);
338 tsk_restore_flags(current, pflags, PF_MEMALLOC);
339
340 return ret;
341 }
342 EXPORT_SYMBOL(__sk_backlog_rcv);
343
344 static int sock_set_timeout(long *timeo_p, char __user *optval, int optlen)
345 {
346 struct timeval tv;
347
348 if (optlen < sizeof(tv))
349 return -EINVAL;
350 if (copy_from_user(&tv, optval, sizeof(tv)))
351 return -EFAULT;
352 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC)
353 return -EDOM;
354
355 if (tv.tv_sec < 0) {
356 static int warned __read_mostly;
357
358 *timeo_p = 0;
359 if (warned < 10 && net_ratelimit()) {
360 warned++;
361 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n",
362 __func__, current->comm, task_pid_nr(current));
363 }
364 return 0;
365 }
366 *timeo_p = MAX_SCHEDULE_TIMEOUT;
367 if (tv.tv_sec == 0 && tv.tv_usec == 0)
368 return 0;
369 if (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT/HZ - 1))
370 *timeo_p = tv.tv_sec*HZ + (tv.tv_usec+(1000000/HZ-1))/(1000000/HZ);
371 return 0;
372 }
373
374 static void sock_warn_obsolete_bsdism(const char *name)
375 {
376 static int warned;
377 static char warncomm[TASK_COMM_LEN];
378 if (strcmp(warncomm, current->comm) && warned < 5) {
379 strcpy(warncomm, current->comm);
380 pr_warn("process `%s' is using obsolete %s SO_BSDCOMPAT\n",
381 warncomm, name);
382 warned++;
383 }
384 }
385
386 static bool sock_needs_netstamp(const struct sock *sk)
387 {
388 switch (sk->sk_family) {
389 case AF_UNSPEC:
390 case AF_UNIX:
391 return false;
392 default:
393 return true;
394 }
395 }
396
397 static void sock_disable_timestamp(struct sock *sk, unsigned long flags)
398 {
399 if (sk->sk_flags & flags) {
400 sk->sk_flags &= ~flags;
401 if (sock_needs_netstamp(sk) &&
402 !(sk->sk_flags & SK_FLAGS_TIMESTAMP))
403 net_disable_timestamp();
404 }
405 }
406
407
408 int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
409 {
410 unsigned long flags;
411 struct sk_buff_head *list = &sk->sk_receive_queue;
412
413 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) {
414 atomic_inc(&sk->sk_drops);
415 trace_sock_rcvqueue_full(sk, skb);
416 return -ENOMEM;
417 }
418
419 if (!sk_rmem_schedule(sk, skb, skb->truesize)) {
420 atomic_inc(&sk->sk_drops);
421 return -ENOBUFS;
422 }
423
424 skb->dev = NULL;
425 skb_set_owner_r(skb, sk);
426
427 /* we escape from rcu protected region, make sure we dont leak
428 * a norefcounted dst
429 */
430 skb_dst_force(skb);
431
432 spin_lock_irqsave(&list->lock, flags);
433 sock_skb_set_dropcount(sk, skb);
434 __skb_queue_tail(list, skb);
435 spin_unlock_irqrestore(&list->lock, flags);
436
437 if (!sock_flag(sk, SOCK_DEAD))
438 sk->sk_data_ready(sk);
439 return 0;
440 }
441 EXPORT_SYMBOL(__sock_queue_rcv_skb);
442
443 int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
444 {
445 int err;
446
447 err = sk_filter(sk, skb);
448 if (err)
449 return err;
450
451 return __sock_queue_rcv_skb(sk, skb);
452 }
453 EXPORT_SYMBOL(sock_queue_rcv_skb);
454
455 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb,
456 const int nested, unsigned int trim_cap, bool refcounted)
457 {
458 int rc = NET_RX_SUCCESS;
459
460 if (sk_filter_trim_cap(sk, skb, trim_cap))
461 goto discard_and_relse;
462
463 skb->dev = NULL;
464
465 if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) {
466 atomic_inc(&sk->sk_drops);
467 goto discard_and_relse;
468 }
469 if (nested)
470 bh_lock_sock_nested(sk);
471 else
472 bh_lock_sock(sk);
473 if (!sock_owned_by_user(sk)) {
474 /*
475 * trylock + unlock semantics:
476 */
477 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_);
478
479 rc = sk_backlog_rcv(sk, skb);
480
481 mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_);
482 } else if (sk_add_backlog(sk, skb, sk->sk_rcvbuf)) {
483 bh_unlock_sock(sk);
484 atomic_inc(&sk->sk_drops);
485 goto discard_and_relse;
486 }
487
488 bh_unlock_sock(sk);
489 out:
490 if (refcounted)
491 sock_put(sk);
492 return rc;
493 discard_and_relse:
494 kfree_skb(skb);
495 goto out;
496 }
497 EXPORT_SYMBOL(__sk_receive_skb);
498
499 struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie)
500 {
501 struct dst_entry *dst = __sk_dst_get(sk);
502
503 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
504 sk_tx_queue_clear(sk);
505 RCU_INIT_POINTER(sk->sk_dst_cache, NULL);
506 dst_release(dst);
507 return NULL;
508 }
509
510 return dst;
511 }
512 EXPORT_SYMBOL(__sk_dst_check);
513
514 struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie)
515 {
516 struct dst_entry *dst = sk_dst_get(sk);
517
518 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
519 sk_dst_reset(sk);
520 dst_release(dst);
521 return NULL;
522 }
523
524 return dst;
525 }
526 EXPORT_SYMBOL(sk_dst_check);
527
528 static int sock_setbindtodevice(struct sock *sk, char __user *optval,
529 int optlen)
530 {
531 int ret = -ENOPROTOOPT;
532 #ifdef CONFIG_NETDEVICES
533 struct net *net = sock_net(sk);
534 char devname[IFNAMSIZ];
535 int index;
536
537 /* Sorry... */
538 ret = -EPERM;
539 if (!ns_capable(net->user_ns, CAP_NET_RAW))
540 goto out;
541
542 ret = -EINVAL;
543 if (optlen < 0)
544 goto out;
545
546 /* Bind this socket to a particular device like "eth0",
547 * as specified in the passed interface name. If the
548 * name is "" or the option length is zero the socket
549 * is not bound.
550 */
551 if (optlen > IFNAMSIZ - 1)
552 optlen = IFNAMSIZ - 1;
553 memset(devname, 0, sizeof(devname));
554
555 ret = -EFAULT;
556 if (copy_from_user(devname, optval, optlen))
557 goto out;
558
559 index = 0;
560 if (devname[0] != '\0') {
561 struct net_device *dev;
562
563 rcu_read_lock();
564 dev = dev_get_by_name_rcu(net, devname);
565 if (dev)
566 index = dev->ifindex;
567 rcu_read_unlock();
568 ret = -ENODEV;
569 if (!dev)
570 goto out;
571 }
572
573 lock_sock(sk);
574 sk->sk_bound_dev_if = index;
575 sk_dst_reset(sk);
576 release_sock(sk);
577
578 ret = 0;
579
580 out:
581 #endif
582
583 return ret;
584 }
585
586 static int sock_getbindtodevice(struct sock *sk, char __user *optval,
587 int __user *optlen, int len)
588 {
589 int ret = -ENOPROTOOPT;
590 #ifdef CONFIG_NETDEVICES
591 struct net *net = sock_net(sk);
592 char devname[IFNAMSIZ];
593
594 if (sk->sk_bound_dev_if == 0) {
595 len = 0;
596 goto zero;
597 }
598
599 ret = -EINVAL;
600 if (len < IFNAMSIZ)
601 goto out;
602
603 ret = netdev_get_name(net, devname, sk->sk_bound_dev_if);
604 if (ret)
605 goto out;
606
607 len = strlen(devname) + 1;
608
609 ret = -EFAULT;
610 if (copy_to_user(optval, devname, len))
611 goto out;
612
613 zero:
614 ret = -EFAULT;
615 if (put_user(len, optlen))
616 goto out;
617
618 ret = 0;
619
620 out:
621 #endif
622
623 return ret;
624 }
625
626 static inline void sock_valbool_flag(struct sock *sk, int bit, int valbool)
627 {
628 if (valbool)
629 sock_set_flag(sk, bit);
630 else
631 sock_reset_flag(sk, bit);
632 }
633
634 bool sk_mc_loop(struct sock *sk)
635 {
636 if (dev_recursion_level())
637 return false;
638 if (!sk)
639 return true;
640 switch (sk->sk_family) {
641 case AF_INET:
642 return inet_sk(sk)->mc_loop;
643 #if IS_ENABLED(CONFIG_IPV6)
644 case AF_INET6:
645 return inet6_sk(sk)->mc_loop;
646 #endif
647 }
648 WARN_ON(1);
649 return true;
650 }
651 EXPORT_SYMBOL(sk_mc_loop);
652
653 /*
654 * This is meant for all protocols to use and covers goings on
655 * at the socket level. Everything here is generic.
656 */
657
658 int sock_setsockopt(struct socket *sock, int level, int optname,
659 char __user *optval, unsigned int optlen)
660 {
661 struct sock *sk = sock->sk;
662 int val;
663 int valbool;
664 struct linger ling;
665 int ret = 0;
666
667 /*
668 * Options without arguments
669 */
670
671 if (optname == SO_BINDTODEVICE)
672 return sock_setbindtodevice(sk, optval, optlen);
673
674 if (optlen < sizeof(int))
675 return -EINVAL;
676
677 if (get_user(val, (int __user *)optval))
678 return -EFAULT;
679
680 valbool = val ? 1 : 0;
681
682 lock_sock(sk);
683
684 switch (optname) {
685 case SO_DEBUG:
686 if (val && !capable(CAP_NET_ADMIN))
687 ret = -EACCES;
688 else
689 sock_valbool_flag(sk, SOCK_DBG, valbool);
690 break;
691 case SO_REUSEADDR:
692 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE);
693 break;
694 case SO_REUSEPORT:
695 sk->sk_reuseport = valbool;
696 break;
697 case SO_TYPE:
698 case SO_PROTOCOL:
699 case SO_DOMAIN:
700 case SO_ERROR:
701 ret = -ENOPROTOOPT;
702 break;
703 case SO_DONTROUTE:
704 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool);
705 break;
706 case SO_BROADCAST:
707 sock_valbool_flag(sk, SOCK_BROADCAST, valbool);
708 break;
709 case SO_SNDBUF:
710 /* Don't error on this BSD doesn't and if you think
711 * about it this is right. Otherwise apps have to
712 * play 'guess the biggest size' games. RCVBUF/SNDBUF
713 * are treated in BSD as hints
714 */
715 val = min_t(u32, val, sysctl_wmem_max);
716 set_sndbuf:
717 sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
718 sk->sk_sndbuf = max_t(int, val * 2, SOCK_MIN_SNDBUF);
719 /* Wake up sending tasks if we upped the value. */
720 sk->sk_write_space(sk);
721 break;
722
723 case SO_SNDBUFFORCE:
724 if (!capable(CAP_NET_ADMIN)) {
725 ret = -EPERM;
726 break;
727 }
728 goto set_sndbuf;
729
730 case SO_RCVBUF:
731 /* Don't error on this BSD doesn't and if you think
732 * about it this is right. Otherwise apps have to
733 * play 'guess the biggest size' games. RCVBUF/SNDBUF
734 * are treated in BSD as hints
735 */
736 val = min_t(u32, val, sysctl_rmem_max);
737 set_rcvbuf:
738 sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
739 /*
740 * We double it on the way in to account for
741 * "struct sk_buff" etc. overhead. Applications
742 * assume that the SO_RCVBUF setting they make will
743 * allow that much actual data to be received on that
744 * socket.
745 *
746 * Applications are unaware that "struct sk_buff" and
747 * other overheads allocate from the receive buffer
748 * during socket buffer allocation.
749 *
750 * And after considering the possible alternatives,
751 * returning the value we actually used in getsockopt
752 * is the most desirable behavior.
753 */
754 sk->sk_rcvbuf = max_t(int, val * 2, SOCK_MIN_RCVBUF);
755 break;
756
757 case SO_RCVBUFFORCE:
758 if (!capable(CAP_NET_ADMIN)) {
759 ret = -EPERM;
760 break;
761 }
762 goto set_rcvbuf;
763
764 case SO_KEEPALIVE:
765 #ifdef CONFIG_INET
766 if (sk->sk_protocol == IPPROTO_TCP &&
767 sk->sk_type == SOCK_STREAM)
768 tcp_set_keepalive(sk, valbool);
769 #endif
770 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool);
771 break;
772
773 case SO_OOBINLINE:
774 sock_valbool_flag(sk, SOCK_URGINLINE, valbool);
775 break;
776
777 case SO_NO_CHECK:
778 sk->sk_no_check_tx = valbool;
779 break;
780
781 case SO_PRIORITY:
782 if ((val >= 0 && val <= 6) ||
783 ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
784 sk->sk_priority = val;
785 else
786 ret = -EPERM;
787 break;
788
789 case SO_LINGER:
790 if (optlen < sizeof(ling)) {
791 ret = -EINVAL; /* 1003.1g */
792 break;
793 }
794 if (copy_from_user(&ling, optval, sizeof(ling))) {
795 ret = -EFAULT;
796 break;
797 }
798 if (!ling.l_onoff)
799 sock_reset_flag(sk, SOCK_LINGER);
800 else {
801 #if (BITS_PER_LONG == 32)
802 if ((unsigned int)ling.l_linger >= MAX_SCHEDULE_TIMEOUT/HZ)
803 sk->sk_lingertime = MAX_SCHEDULE_TIMEOUT;
804 else
805 #endif
806 sk->sk_lingertime = (unsigned int)ling.l_linger * HZ;
807 sock_set_flag(sk, SOCK_LINGER);
808 }
809 break;
810
811 case SO_BSDCOMPAT:
812 sock_warn_obsolete_bsdism("setsockopt");
813 break;
814
815 case SO_PASSCRED:
816 if (valbool)
817 set_bit(SOCK_PASSCRED, &sock->flags);
818 else
819 clear_bit(SOCK_PASSCRED, &sock->flags);
820 break;
821
822 case SO_TIMESTAMP:
823 case SO_TIMESTAMPNS:
824 if (valbool) {
825 if (optname == SO_TIMESTAMP)
826 sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
827 else
828 sock_set_flag(sk, SOCK_RCVTSTAMPNS);
829 sock_set_flag(sk, SOCK_RCVTSTAMP);
830 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
831 } else {
832 sock_reset_flag(sk, SOCK_RCVTSTAMP);
833 sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
834 }
835 break;
836
837 case SO_TIMESTAMPING:
838 if (val & ~SOF_TIMESTAMPING_MASK) {
839 ret = -EINVAL;
840 break;
841 }
842
843 if (val & SOF_TIMESTAMPING_OPT_ID &&
844 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) {
845 if (sk->sk_protocol == IPPROTO_TCP &&
846 sk->sk_type == SOCK_STREAM) {
847 if ((1 << sk->sk_state) &
848 (TCPF_CLOSE | TCPF_LISTEN)) {
849 ret = -EINVAL;
850 break;
851 }
852 sk->sk_tskey = tcp_sk(sk)->snd_una;
853 } else {
854 sk->sk_tskey = 0;
855 }
856 }
857 sk->sk_tsflags = val;
858 if (val & SOF_TIMESTAMPING_RX_SOFTWARE)
859 sock_enable_timestamp(sk,
860 SOCK_TIMESTAMPING_RX_SOFTWARE);
861 else
862 sock_disable_timestamp(sk,
863 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE));
864 break;
865
866 case SO_RCVLOWAT:
867 if (val < 0)
868 val = INT_MAX;
869 sk->sk_rcvlowat = val ? : 1;
870 break;
871
872 case SO_RCVTIMEO:
873 ret = sock_set_timeout(&sk->sk_rcvtimeo, optval, optlen);
874 break;
875
876 case SO_SNDTIMEO:
877 ret = sock_set_timeout(&sk->sk_sndtimeo, optval, optlen);
878 break;
879
880 case SO_ATTACH_FILTER:
881 ret = -EINVAL;
882 if (optlen == sizeof(struct sock_fprog)) {
883 struct sock_fprog fprog;
884
885 ret = -EFAULT;
886 if (copy_from_user(&fprog, optval, sizeof(fprog)))
887 break;
888
889 ret = sk_attach_filter(&fprog, sk);
890 }
891 break;
892
893 case SO_ATTACH_BPF:
894 ret = -EINVAL;
895 if (optlen == sizeof(u32)) {
896 u32 ufd;
897
898 ret = -EFAULT;
899 if (copy_from_user(&ufd, optval, sizeof(ufd)))
900 break;
901
902 ret = sk_attach_bpf(ufd, sk);
903 }
904 break;
905
906 case SO_ATTACH_REUSEPORT_CBPF:
907 ret = -EINVAL;
908 if (optlen == sizeof(struct sock_fprog)) {
909 struct sock_fprog fprog;
910
911 ret = -EFAULT;
912 if (copy_from_user(&fprog, optval, sizeof(fprog)))
913 break;
914
915 ret = sk_reuseport_attach_filter(&fprog, sk);
916 }
917 break;
918
919 case SO_ATTACH_REUSEPORT_EBPF:
920 ret = -EINVAL;
921 if (optlen == sizeof(u32)) {
922 u32 ufd;
923
924 ret = -EFAULT;
925 if (copy_from_user(&ufd, optval, sizeof(ufd)))
926 break;
927
928 ret = sk_reuseport_attach_bpf(ufd, sk);
929 }
930 break;
931
932 case SO_DETACH_FILTER:
933 ret = sk_detach_filter(sk);
934 break;
935
936 case SO_LOCK_FILTER:
937 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool)
938 ret = -EPERM;
939 else
940 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool);
941 break;
942
943 case SO_PASSSEC:
944 if (valbool)
945 set_bit(SOCK_PASSSEC, &sock->flags);
946 else
947 clear_bit(SOCK_PASSSEC, &sock->flags);
948 break;
949 case SO_MARK:
950 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
951 ret = -EPERM;
952 else
953 sk->sk_mark = val;
954 break;
955
956 case SO_RXQ_OVFL:
957 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool);
958 break;
959
960 case SO_WIFI_STATUS:
961 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool);
962 break;
963
964 case SO_PEEK_OFF:
965 if (sock->ops->set_peek_off)
966 ret = sock->ops->set_peek_off(sk, val);
967 else
968 ret = -EOPNOTSUPP;
969 break;
970
971 case SO_NOFCS:
972 sock_valbool_flag(sk, SOCK_NOFCS, valbool);
973 break;
974
975 case SO_SELECT_ERR_QUEUE:
976 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool);
977 break;
978
979 #ifdef CONFIG_NET_RX_BUSY_POLL
980 case SO_BUSY_POLL:
981 /* allow unprivileged users to decrease the value */
982 if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN))
983 ret = -EPERM;
984 else {
985 if (val < 0)
986 ret = -EINVAL;
987 else
988 sk->sk_ll_usec = val;
989 }
990 break;
991 #endif
992
993 case SO_MAX_PACING_RATE:
994 sk->sk_max_pacing_rate = val;
995 sk->sk_pacing_rate = min(sk->sk_pacing_rate,
996 sk->sk_max_pacing_rate);
997 break;
998
999 case SO_INCOMING_CPU:
1000 sk->sk_incoming_cpu = val;
1001 break;
1002
1003 case SO_CNX_ADVICE:
1004 if (val == 1)
1005 dst_negative_advice(sk);
1006 break;
1007 default:
1008 ret = -ENOPROTOOPT;
1009 break;
1010 }
1011 release_sock(sk);
1012 return ret;
1013 }
1014 EXPORT_SYMBOL(sock_setsockopt);
1015
1016
1017 static void cred_to_ucred(struct pid *pid, const struct cred *cred,
1018 struct ucred *ucred)
1019 {
1020 ucred->pid = pid_vnr(pid);
1021 ucred->uid = ucred->gid = -1;
1022 if (cred) {
1023 struct user_namespace *current_ns = current_user_ns();
1024
1025 ucred->uid = from_kuid_munged(current_ns, cred->euid);
1026 ucred->gid = from_kgid_munged(current_ns, cred->egid);
1027 }
1028 }
1029
1030 int sock_getsockopt(struct socket *sock, int level, int optname,
1031 char __user *optval, int __user *optlen)
1032 {
1033 struct sock *sk = sock->sk;
1034
1035 union {
1036 int val;
1037 struct linger ling;
1038 struct timeval tm;
1039 } v;
1040
1041 int lv = sizeof(int);
1042 int len;
1043
1044 if (get_user(len, optlen))
1045 return -EFAULT;
1046 if (len < 0)
1047 return -EINVAL;
1048
1049 memset(&v, 0, sizeof(v));
1050
1051 switch (optname) {
1052 case SO_DEBUG:
1053 v.val = sock_flag(sk, SOCK_DBG);
1054 break;
1055
1056 case SO_DONTROUTE:
1057 v.val = sock_flag(sk, SOCK_LOCALROUTE);
1058 break;
1059
1060 case SO_BROADCAST:
1061 v.val = sock_flag(sk, SOCK_BROADCAST);
1062 break;
1063
1064 case SO_SNDBUF:
1065 v.val = sk->sk_sndbuf;
1066 break;
1067
1068 case SO_RCVBUF:
1069 v.val = sk->sk_rcvbuf;
1070 break;
1071
1072 case SO_REUSEADDR:
1073 v.val = sk->sk_reuse;
1074 break;
1075
1076 case SO_REUSEPORT:
1077 v.val = sk->sk_reuseport;
1078 break;
1079
1080 case SO_KEEPALIVE:
1081 v.val = sock_flag(sk, SOCK_KEEPOPEN);
1082 break;
1083
1084 case SO_TYPE:
1085 v.val = sk->sk_type;
1086 break;
1087
1088 case SO_PROTOCOL:
1089 v.val = sk->sk_protocol;
1090 break;
1091
1092 case SO_DOMAIN:
1093 v.val = sk->sk_family;
1094 break;
1095
1096 case SO_ERROR:
1097 v.val = -sock_error(sk);
1098 if (v.val == 0)
1099 v.val = xchg(&sk->sk_err_soft, 0);
1100 break;
1101
1102 case SO_OOBINLINE:
1103 v.val = sock_flag(sk, SOCK_URGINLINE);
1104 break;
1105
1106 case SO_NO_CHECK:
1107 v.val = sk->sk_no_check_tx;
1108 break;
1109
1110 case SO_PRIORITY:
1111 v.val = sk->sk_priority;
1112 break;
1113
1114 case SO_LINGER:
1115 lv = sizeof(v.ling);
1116 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER);
1117 v.ling.l_linger = sk->sk_lingertime / HZ;
1118 break;
1119
1120 case SO_BSDCOMPAT:
1121 sock_warn_obsolete_bsdism("getsockopt");
1122 break;
1123
1124 case SO_TIMESTAMP:
1125 v.val = sock_flag(sk, SOCK_RCVTSTAMP) &&
1126 !sock_flag(sk, SOCK_RCVTSTAMPNS);
1127 break;
1128
1129 case SO_TIMESTAMPNS:
1130 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS);
1131 break;
1132
1133 case SO_TIMESTAMPING:
1134 v.val = sk->sk_tsflags;
1135 break;
1136
1137 case SO_RCVTIMEO:
1138 lv = sizeof(struct timeval);
1139 if (sk->sk_rcvtimeo == MAX_SCHEDULE_TIMEOUT) {
1140 v.tm.tv_sec = 0;
1141 v.tm.tv_usec = 0;
1142 } else {
1143 v.tm.tv_sec = sk->sk_rcvtimeo / HZ;
1144 v.tm.tv_usec = ((sk->sk_rcvtimeo % HZ) * 1000000) / HZ;
1145 }
1146 break;
1147
1148 case SO_SNDTIMEO:
1149 lv = sizeof(struct timeval);
1150 if (sk->sk_sndtimeo == MAX_SCHEDULE_TIMEOUT) {
1151 v.tm.tv_sec = 0;
1152 v.tm.tv_usec = 0;
1153 } else {
1154 v.tm.tv_sec = sk->sk_sndtimeo / HZ;
1155 v.tm.tv_usec = ((sk->sk_sndtimeo % HZ) * 1000000) / HZ;
1156 }
1157 break;
1158
1159 case SO_RCVLOWAT:
1160 v.val = sk->sk_rcvlowat;
1161 break;
1162
1163 case SO_SNDLOWAT:
1164 v.val = 1;
1165 break;
1166
1167 case SO_PASSCRED:
1168 v.val = !!test_bit(SOCK_PASSCRED, &sock->flags);
1169 break;
1170
1171 case SO_PEERCRED:
1172 {
1173 struct ucred peercred;
1174 if (len > sizeof(peercred))
1175 len = sizeof(peercred);
1176 cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred);
1177 if (copy_to_user(optval, &peercred, len))
1178 return -EFAULT;
1179 goto lenout;
1180 }
1181
1182 case SO_PEERNAME:
1183 {
1184 char address[128];
1185
1186 if (sock->ops->getname(sock, (struct sockaddr *)address, &lv, 2))
1187 return -ENOTCONN;
1188 if (lv < len)
1189 return -EINVAL;
1190 if (copy_to_user(optval, address, len))
1191 return -EFAULT;
1192 goto lenout;
1193 }
1194
1195 /* Dubious BSD thing... Probably nobody even uses it, but
1196 * the UNIX standard wants it for whatever reason... -DaveM
1197 */
1198 case SO_ACCEPTCONN:
1199 v.val = sk->sk_state == TCP_LISTEN;
1200 break;
1201
1202 case SO_PASSSEC:
1203 v.val = !!test_bit(SOCK_PASSSEC, &sock->flags);
1204 break;
1205
1206 case SO_PEERSEC:
1207 return security_socket_getpeersec_stream(sock, optval, optlen, len);
1208
1209 case SO_MARK:
1210 v.val = sk->sk_mark;
1211 break;
1212
1213 case SO_RXQ_OVFL:
1214 v.val = sock_flag(sk, SOCK_RXQ_OVFL);
1215 break;
1216
1217 case SO_WIFI_STATUS:
1218 v.val = sock_flag(sk, SOCK_WIFI_STATUS);
1219 break;
1220
1221 case SO_PEEK_OFF:
1222 if (!sock->ops->set_peek_off)
1223 return -EOPNOTSUPP;
1224
1225 v.val = sk->sk_peek_off;
1226 break;
1227 case SO_NOFCS:
1228 v.val = sock_flag(sk, SOCK_NOFCS);
1229 break;
1230
1231 case SO_BINDTODEVICE:
1232 return sock_getbindtodevice(sk, optval, optlen, len);
1233
1234 case SO_GET_FILTER:
1235 len = sk_get_filter(sk, (struct sock_filter __user *)optval, len);
1236 if (len < 0)
1237 return len;
1238
1239 goto lenout;
1240
1241 case SO_LOCK_FILTER:
1242 v.val = sock_flag(sk, SOCK_FILTER_LOCKED);
1243 break;
1244
1245 case SO_BPF_EXTENSIONS:
1246 v.val = bpf_tell_extensions();
1247 break;
1248
1249 case SO_SELECT_ERR_QUEUE:
1250 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE);
1251 break;
1252
1253 #ifdef CONFIG_NET_RX_BUSY_POLL
1254 case SO_BUSY_POLL:
1255 v.val = sk->sk_ll_usec;
1256 break;
1257 #endif
1258
1259 case SO_MAX_PACING_RATE:
1260 v.val = sk->sk_max_pacing_rate;
1261 break;
1262
1263 case SO_INCOMING_CPU:
1264 v.val = sk->sk_incoming_cpu;
1265 break;
1266
1267 default:
1268 /* We implement the SO_SNDLOWAT etc to not be settable
1269 * (1003.1g 7).
1270 */
1271 return -ENOPROTOOPT;
1272 }
1273
1274 if (len > lv)
1275 len = lv;
1276 if (copy_to_user(optval, &v, len))
1277 return -EFAULT;
1278 lenout:
1279 if (put_user(len, optlen))
1280 return -EFAULT;
1281 return 0;
1282 }
1283
1284 /*
1285 * Initialize an sk_lock.
1286 *
1287 * (We also register the sk_lock with the lock validator.)
1288 */
1289 static inline void sock_lock_init(struct sock *sk)
1290 {
1291 sock_lock_init_class_and_name(sk,
1292 af_family_slock_key_strings[sk->sk_family],
1293 af_family_slock_keys + sk->sk_family,
1294 af_family_key_strings[sk->sk_family],
1295 af_family_keys + sk->sk_family);
1296 }
1297
1298 /*
1299 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet,
1300 * even temporarly, because of RCU lookups. sk_node should also be left as is.
1301 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end
1302 */
1303 static void sock_copy(struct sock *nsk, const struct sock *osk)
1304 {
1305 #ifdef CONFIG_SECURITY_NETWORK
1306 void *sptr = nsk->sk_security;
1307 #endif
1308 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin));
1309
1310 memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end,
1311 osk->sk_prot->obj_size - offsetof(struct sock, sk_dontcopy_end));
1312
1313 #ifdef CONFIG_SECURITY_NETWORK
1314 nsk->sk_security = sptr;
1315 security_sk_clone(osk, nsk);
1316 #endif
1317 }
1318
1319 void sk_prot_clear_portaddr_nulls(struct sock *sk, int size)
1320 {
1321 unsigned long nulls1, nulls2;
1322
1323 nulls1 = offsetof(struct sock, __sk_common.skc_node.next);
1324 nulls2 = offsetof(struct sock, __sk_common.skc_portaddr_node.next);
1325 if (nulls1 > nulls2)
1326 swap(nulls1, nulls2);
1327
1328 if (nulls1 != 0)
1329 memset((char *)sk, 0, nulls1);
1330 memset((char *)sk + nulls1 + sizeof(void *), 0,
1331 nulls2 - nulls1 - sizeof(void *));
1332 memset((char *)sk + nulls2 + sizeof(void *), 0,
1333 size - nulls2 - sizeof(void *));
1334 }
1335 EXPORT_SYMBOL(sk_prot_clear_portaddr_nulls);
1336
1337 static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority,
1338 int family)
1339 {
1340 struct sock *sk;
1341 struct kmem_cache *slab;
1342
1343 slab = prot->slab;
1344 if (slab != NULL) {
1345 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO);
1346 if (!sk)
1347 return sk;
1348 if (priority & __GFP_ZERO) {
1349 if (prot->clear_sk)
1350 prot->clear_sk(sk, prot->obj_size);
1351 else
1352 sk_prot_clear_nulls(sk, prot->obj_size);
1353 }
1354 } else
1355 sk = kmalloc(prot->obj_size, priority);
1356
1357 if (sk != NULL) {
1358 kmemcheck_annotate_bitfield(sk, flags);
1359
1360 if (security_sk_alloc(sk, family, priority))
1361 goto out_free;
1362
1363 if (!try_module_get(prot->owner))
1364 goto out_free_sec;
1365 sk_tx_queue_clear(sk);
1366 }
1367
1368 return sk;
1369
1370 out_free_sec:
1371 security_sk_free(sk);
1372 out_free:
1373 if (slab != NULL)
1374 kmem_cache_free(slab, sk);
1375 else
1376 kfree(sk);
1377 return NULL;
1378 }
1379
1380 static void sk_prot_free(struct proto *prot, struct sock *sk)
1381 {
1382 struct kmem_cache *slab;
1383 struct module *owner;
1384
1385 owner = prot->owner;
1386 slab = prot->slab;
1387
1388 cgroup_sk_free(&sk->sk_cgrp_data);
1389 security_sk_free(sk);
1390 if (slab != NULL)
1391 kmem_cache_free(slab, sk);
1392 else
1393 kfree(sk);
1394 module_put(owner);
1395 }
1396
1397 /**
1398 * sk_alloc - All socket objects are allocated here
1399 * @net: the applicable net namespace
1400 * @family: protocol family
1401 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
1402 * @prot: struct proto associated with this new sock instance
1403 * @kern: is this to be a kernel socket?
1404 */
1405 struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
1406 struct proto *prot, int kern)
1407 {
1408 struct sock *sk;
1409
1410 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family);
1411 if (sk) {
1412 sk->sk_family = family;
1413 /*
1414 * See comment in struct sock definition to understand
1415 * why we need sk_prot_creator -acme
1416 */
1417 sk->sk_prot = sk->sk_prot_creator = prot;
1418 sock_lock_init(sk);
1419 sk->sk_net_refcnt = kern ? 0 : 1;
1420 if (likely(sk->sk_net_refcnt))
1421 get_net(net);
1422 sock_net_set(sk, net);
1423 atomic_set(&sk->sk_wmem_alloc, 1);
1424
1425 cgroup_sk_alloc(&sk->sk_cgrp_data);
1426 sock_update_classid(&sk->sk_cgrp_data);
1427 sock_update_netprioidx(&sk->sk_cgrp_data);
1428 }
1429
1430 return sk;
1431 }
1432 EXPORT_SYMBOL(sk_alloc);
1433
1434 /* Sockets having SOCK_RCU_FREE will call this function after one RCU
1435 * grace period. This is the case for UDP sockets and TCP listeners.
1436 */
1437 static void __sk_destruct(struct rcu_head *head)
1438 {
1439 struct sock *sk = container_of(head, struct sock, sk_rcu);
1440 struct sk_filter *filter;
1441
1442 if (sk->sk_destruct)
1443 sk->sk_destruct(sk);
1444
1445 filter = rcu_dereference_check(sk->sk_filter,
1446 atomic_read(&sk->sk_wmem_alloc) == 0);
1447 if (filter) {
1448 sk_filter_uncharge(sk, filter);
1449 RCU_INIT_POINTER(sk->sk_filter, NULL);
1450 }
1451 if (rcu_access_pointer(sk->sk_reuseport_cb))
1452 reuseport_detach_sock(sk);
1453
1454 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP);
1455
1456 if (atomic_read(&sk->sk_omem_alloc))
1457 pr_debug("%s: optmem leakage (%d bytes) detected\n",
1458 __func__, atomic_read(&sk->sk_omem_alloc));
1459
1460 if (sk->sk_peer_cred)
1461 put_cred(sk->sk_peer_cred);
1462 put_pid(sk->sk_peer_pid);
1463 if (likely(sk->sk_net_refcnt))
1464 put_net(sock_net(sk));
1465 sk_prot_free(sk->sk_prot_creator, sk);
1466 }
1467
1468 void sk_destruct(struct sock *sk)
1469 {
1470 if (sock_flag(sk, SOCK_RCU_FREE))
1471 call_rcu(&sk->sk_rcu, __sk_destruct);
1472 else
1473 __sk_destruct(&sk->sk_rcu);
1474 }
1475
1476 static void __sk_free(struct sock *sk)
1477 {
1478 if (unlikely(sock_diag_has_destroy_listeners(sk) && sk->sk_net_refcnt))
1479 sock_diag_broadcast_destroy(sk);
1480 else
1481 sk_destruct(sk);
1482 }
1483
1484 void sk_free(struct sock *sk)
1485 {
1486 /*
1487 * We subtract one from sk_wmem_alloc and can know if
1488 * some packets are still in some tx queue.
1489 * If not null, sock_wfree() will call __sk_free(sk) later
1490 */
1491 if (atomic_dec_and_test(&sk->sk_wmem_alloc))
1492 __sk_free(sk);
1493 }
1494 EXPORT_SYMBOL(sk_free);
1495
1496 /**
1497 * sk_clone_lock - clone a socket, and lock its clone
1498 * @sk: the socket to clone
1499 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
1500 *
1501 * Caller must unlock socket even in error path (bh_unlock_sock(newsk))
1502 */
1503 struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
1504 {
1505 struct sock *newsk;
1506 bool is_charged = true;
1507
1508 newsk = sk_prot_alloc(sk->sk_prot, priority, sk->sk_family);
1509 if (newsk != NULL) {
1510 struct sk_filter *filter;
1511
1512 sock_copy(newsk, sk);
1513
1514 /* SANITY */
1515 if (likely(newsk->sk_net_refcnt))
1516 get_net(sock_net(newsk));
1517 sk_node_init(&newsk->sk_node);
1518 sock_lock_init(newsk);
1519 bh_lock_sock(newsk);
1520 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL;
1521 newsk->sk_backlog.len = 0;
1522
1523 atomic_set(&newsk->sk_rmem_alloc, 0);
1524 /*
1525 * sk_wmem_alloc set to one (see sk_free() and sock_wfree())
1526 */
1527 atomic_set(&newsk->sk_wmem_alloc, 1);
1528 atomic_set(&newsk->sk_omem_alloc, 0);
1529 skb_queue_head_init(&newsk->sk_receive_queue);
1530 skb_queue_head_init(&newsk->sk_write_queue);
1531
1532 rwlock_init(&newsk->sk_callback_lock);
1533 lockdep_set_class_and_name(&newsk->sk_callback_lock,
1534 af_callback_keys + newsk->sk_family,
1535 af_family_clock_key_strings[newsk->sk_family]);
1536
1537 newsk->sk_dst_cache = NULL;
1538 newsk->sk_wmem_queued = 0;
1539 newsk->sk_forward_alloc = 0;
1540 atomic_set(&newsk->sk_drops, 0);
1541 newsk->sk_send_head = NULL;
1542 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK;
1543
1544 sock_reset_flag(newsk, SOCK_DONE);
1545 skb_queue_head_init(&newsk->sk_error_queue);
1546
1547 filter = rcu_dereference_protected(newsk->sk_filter, 1);
1548 if (filter != NULL)
1549 /* though it's an empty new sock, the charging may fail
1550 * if sysctl_optmem_max was changed between creation of
1551 * original socket and cloning
1552 */
1553 is_charged = sk_filter_charge(newsk, filter);
1554
1555 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) {
1556 /* It is still raw copy of parent, so invalidate
1557 * destructor and make plain sk_free() */
1558 newsk->sk_destruct = NULL;
1559 bh_unlock_sock(newsk);
1560 sk_free(newsk);
1561 newsk = NULL;
1562 goto out;
1563 }
1564 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL);
1565
1566 newsk->sk_err = 0;
1567 newsk->sk_err_soft = 0;
1568 newsk->sk_priority = 0;
1569 newsk->sk_incoming_cpu = raw_smp_processor_id();
1570 atomic64_set(&newsk->sk_cookie, 0);
1571
1572 cgroup_sk_alloc(&newsk->sk_cgrp_data);
1573
1574 /*
1575 * Before updating sk_refcnt, we must commit prior changes to memory
1576 * (Documentation/RCU/rculist_nulls.txt for details)
1577 */
1578 smp_wmb();
1579 atomic_set(&newsk->sk_refcnt, 2);
1580
1581 /*
1582 * Increment the counter in the same struct proto as the master
1583 * sock (sk_refcnt_debug_inc uses newsk->sk_prot->socks, that
1584 * is the same as sk->sk_prot->socks, as this field was copied
1585 * with memcpy).
1586 *
1587 * This _changes_ the previous behaviour, where
1588 * tcp_create_openreq_child always was incrementing the
1589 * equivalent to tcp_prot->socks (inet_sock_nr), so this have
1590 * to be taken into account in all callers. -acme
1591 */
1592 sk_refcnt_debug_inc(newsk);
1593 sk_set_socket(newsk, NULL);
1594 newsk->sk_wq = NULL;
1595
1596 if (mem_cgroup_sockets_enabled && sk->sk_memcg)
1597 sock_update_memcg(newsk);
1598
1599 if (newsk->sk_prot->sockets_allocated)
1600 sk_sockets_allocated_inc(newsk);
1601
1602 if (sock_needs_netstamp(sk) &&
1603 newsk->sk_flags & SK_FLAGS_TIMESTAMP)
1604 net_enable_timestamp();
1605 }
1606 out:
1607 return newsk;
1608 }
1609 EXPORT_SYMBOL_GPL(sk_clone_lock);
1610
1611 void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
1612 {
1613 u32 max_segs = 1;
1614
1615 sk_dst_set(sk, dst);
1616 sk->sk_route_caps = dst->dev->features;
1617 if (sk->sk_route_caps & NETIF_F_GSO)
1618 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE;
1619 sk->sk_route_caps &= ~sk->sk_route_nocaps;
1620 if (sk_can_gso(sk)) {
1621 if (dst->header_len) {
1622 sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
1623 } else {
1624 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;
1625 sk->sk_gso_max_size = dst->dev->gso_max_size;
1626 max_segs = max_t(u32, dst->dev->gso_max_segs, 1);
1627 }
1628 }
1629 sk->sk_gso_max_segs = max_segs;
1630 }
1631 EXPORT_SYMBOL_GPL(sk_setup_caps);
1632
1633 /*
1634 * Simple resource managers for sockets.
1635 */
1636
1637
1638 /*
1639 * Write buffer destructor automatically called from kfree_skb.
1640 */
1641 void sock_wfree(struct sk_buff *skb)
1642 {
1643 struct sock *sk = skb->sk;
1644 unsigned int len = skb->truesize;
1645
1646 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) {
1647 /*
1648 * Keep a reference on sk_wmem_alloc, this will be released
1649 * after sk_write_space() call
1650 */
1651 atomic_sub(len - 1, &sk->sk_wmem_alloc);
1652 sk->sk_write_space(sk);
1653 len = 1;
1654 }
1655 /*
1656 * if sk_wmem_alloc reaches 0, we must finish what sk_free()
1657 * could not do because of in-flight packets
1658 */
1659 if (atomic_sub_and_test(len, &sk->sk_wmem_alloc))
1660 __sk_free(sk);
1661 }
1662 EXPORT_SYMBOL(sock_wfree);
1663
1664 /* This variant of sock_wfree() is used by TCP,
1665 * since it sets SOCK_USE_WRITE_QUEUE.
1666 */
1667 void __sock_wfree(struct sk_buff *skb)
1668 {
1669 struct sock *sk = skb->sk;
1670
1671 if (atomic_sub_and_test(skb->truesize, &sk->sk_wmem_alloc))
1672 __sk_free(sk);
1673 }
1674
1675 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
1676 {
1677 skb_orphan(skb);
1678 skb->sk = sk;
1679 #ifdef CONFIG_INET
1680 if (unlikely(!sk_fullsock(sk))) {
1681 skb->destructor = sock_edemux;
1682 sock_hold(sk);
1683 return;
1684 }
1685 #endif
1686 skb->destructor = sock_wfree;
1687 skb_set_hash_from_sk(skb, sk);
1688 /*
1689 * We used to take a refcount on sk, but following operation
1690 * is enough to guarantee sk_free() wont free this sock until
1691 * all in-flight packets are completed
1692 */
1693 atomic_add(skb->truesize, &sk->sk_wmem_alloc);
1694 }
1695 EXPORT_SYMBOL(skb_set_owner_w);
1696
1697 /* This helper is used by netem, as it can hold packets in its
1698 * delay queue. We want to allow the owner socket to send more
1699 * packets, as if they were already TX completed by a typical driver.
1700 * But we also want to keep skb->sk set because some packet schedulers
1701 * rely on it (sch_fq for example). So we set skb->truesize to a small
1702 * amount (1) and decrease sk_wmem_alloc accordingly.
1703 */
1704 void skb_orphan_partial(struct sk_buff *skb)
1705 {
1706 /* If this skb is a TCP pure ACK or already went here,
1707 * we have nothing to do. 2 is already a very small truesize.
1708 */
1709 if (skb->truesize <= 2)
1710 return;
1711
1712 /* TCP stack sets skb->ooo_okay based on sk_wmem_alloc,
1713 * so we do not completely orphan skb, but transfert all
1714 * accounted bytes but one, to avoid unexpected reorders.
1715 */
1716 if (skb->destructor == sock_wfree
1717 #ifdef CONFIG_INET
1718 || skb->destructor == tcp_wfree
1719 #endif
1720 ) {
1721 atomic_sub(skb->truesize - 1, &skb->sk->sk_wmem_alloc);
1722 skb->truesize = 1;
1723 } else {
1724 skb_orphan(skb);
1725 }
1726 }
1727 EXPORT_SYMBOL(skb_orphan_partial);
1728
1729 /*
1730 * Read buffer destructor automatically called from kfree_skb.
1731 */
1732 void sock_rfree(struct sk_buff *skb)
1733 {
1734 struct sock *sk = skb->sk;
1735 unsigned int len = skb->truesize;
1736
1737 atomic_sub(len, &sk->sk_rmem_alloc);
1738 sk_mem_uncharge(sk, len);
1739 }
1740 EXPORT_SYMBOL(sock_rfree);
1741
1742 /*
1743 * Buffer destructor for skbs that are not used directly in read or write
1744 * path, e.g. for error handler skbs. Automatically called from kfree_skb.
1745 */
1746 void sock_efree(struct sk_buff *skb)
1747 {
1748 sock_put(skb->sk);
1749 }
1750 EXPORT_SYMBOL(sock_efree);
1751
1752 kuid_t sock_i_uid(struct sock *sk)
1753 {
1754 kuid_t uid;
1755
1756 read_lock_bh(&sk->sk_callback_lock);
1757 uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID;
1758 read_unlock_bh(&sk->sk_callback_lock);
1759 return uid;
1760 }
1761 EXPORT_SYMBOL(sock_i_uid);
1762
1763 unsigned long sock_i_ino(struct sock *sk)
1764 {
1765 unsigned long ino;
1766
1767 read_lock_bh(&sk->sk_callback_lock);
1768 ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0;
1769 read_unlock_bh(&sk->sk_callback_lock);
1770 return ino;
1771 }
1772 EXPORT_SYMBOL(sock_i_ino);
1773
1774 /*
1775 * Allocate a skb from the socket's send buffer.
1776 */
1777 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
1778 gfp_t priority)
1779 {
1780 if (force || atomic_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf) {
1781 struct sk_buff *skb = alloc_skb(size, priority);
1782 if (skb) {
1783 skb_set_owner_w(skb, sk);
1784 return skb;
1785 }
1786 }
1787 return NULL;
1788 }
1789 EXPORT_SYMBOL(sock_wmalloc);
1790
1791 /*
1792 * Allocate a memory block from the socket's option memory buffer.
1793 */
1794 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority)
1795 {
1796 if ((unsigned int)size <= sysctl_optmem_max &&
1797 atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) {
1798 void *mem;
1799 /* First do the add, to avoid the race if kmalloc
1800 * might sleep.
1801 */
1802 atomic_add(size, &sk->sk_omem_alloc);
1803 mem = kmalloc(size, priority);
1804 if (mem)
1805 return mem;
1806 atomic_sub(size, &sk->sk_omem_alloc);
1807 }
1808 return NULL;
1809 }
1810 EXPORT_SYMBOL(sock_kmalloc);
1811
1812 /* Free an option memory block. Note, we actually want the inline
1813 * here as this allows gcc to detect the nullify and fold away the
1814 * condition entirely.
1815 */
1816 static inline void __sock_kfree_s(struct sock *sk, void *mem, int size,
1817 const bool nullify)
1818 {
1819 if (WARN_ON_ONCE(!mem))
1820 return;
1821 if (nullify)
1822 kzfree(mem);
1823 else
1824 kfree(mem);
1825 atomic_sub(size, &sk->sk_omem_alloc);
1826 }
1827
1828 void sock_kfree_s(struct sock *sk, void *mem, int size)
1829 {
1830 __sock_kfree_s(sk, mem, size, false);
1831 }
1832 EXPORT_SYMBOL(sock_kfree_s);
1833
1834 void sock_kzfree_s(struct sock *sk, void *mem, int size)
1835 {
1836 __sock_kfree_s(sk, mem, size, true);
1837 }
1838 EXPORT_SYMBOL(sock_kzfree_s);
1839
1840 /* It is almost wait_for_tcp_memory minus release_sock/lock_sock.
1841 I think, these locks should be removed for datagram sockets.
1842 */
1843 static long sock_wait_for_wmem(struct sock *sk, long timeo)
1844 {
1845 DEFINE_WAIT(wait);
1846
1847 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1848 for (;;) {
1849 if (!timeo)
1850 break;
1851 if (signal_pending(current))
1852 break;
1853 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1854 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
1855 if (atomic_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf)
1856 break;
1857 if (sk->sk_shutdown & SEND_SHUTDOWN)
1858 break;
1859 if (sk->sk_err)
1860 break;
1861 timeo = schedule_timeout(timeo);
1862 }
1863 finish_wait(sk_sleep(sk), &wait);
1864 return timeo;
1865 }
1866
1867
1868 /*
1869 * Generic send/receive buffer handlers
1870 */
1871
1872 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
1873 unsigned long data_len, int noblock,
1874 int *errcode, int max_page_order)
1875 {
1876 struct sk_buff *skb;
1877 long timeo;
1878 int err;
1879
1880 timeo = sock_sndtimeo(sk, noblock);
1881 for (;;) {
1882 err = sock_error(sk);
1883 if (err != 0)
1884 goto failure;
1885
1886 err = -EPIPE;
1887 if (sk->sk_shutdown & SEND_SHUTDOWN)
1888 goto failure;
1889
1890 if (sk_wmem_alloc_get(sk) < sk->sk_sndbuf)
1891 break;
1892
1893 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1894 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1895 err = -EAGAIN;
1896 if (!timeo)
1897 goto failure;
1898 if (signal_pending(current))
1899 goto interrupted;
1900 timeo = sock_wait_for_wmem(sk, timeo);
1901 }
1902 skb = alloc_skb_with_frags(header_len, data_len, max_page_order,
1903 errcode, sk->sk_allocation);
1904 if (skb)
1905 skb_set_owner_w(skb, sk);
1906 return skb;
1907
1908 interrupted:
1909 err = sock_intr_errno(timeo);
1910 failure:
1911 *errcode = err;
1912 return NULL;
1913 }
1914 EXPORT_SYMBOL(sock_alloc_send_pskb);
1915
1916 struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size,
1917 int noblock, int *errcode)
1918 {
1919 return sock_alloc_send_pskb(sk, size, 0, noblock, errcode, 0);
1920 }
1921 EXPORT_SYMBOL(sock_alloc_send_skb);
1922
1923 int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg,
1924 struct sockcm_cookie *sockc)
1925 {
1926 u32 tsflags;
1927
1928 switch (cmsg->cmsg_type) {
1929 case SO_MARK:
1930 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
1931 return -EPERM;
1932 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
1933 return -EINVAL;
1934 sockc->mark = *(u32 *)CMSG_DATA(cmsg);
1935 break;
1936 case SO_TIMESTAMPING:
1937 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
1938 return -EINVAL;
1939
1940 tsflags = *(u32 *)CMSG_DATA(cmsg);
1941 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK)
1942 return -EINVAL;
1943
1944 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK;
1945 sockc->tsflags |= tsflags;
1946 break;
1947 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */
1948 case SCM_RIGHTS:
1949 case SCM_CREDENTIALS:
1950 break;
1951 default:
1952 return -EINVAL;
1953 }
1954 return 0;
1955 }
1956 EXPORT_SYMBOL(__sock_cmsg_send);
1957
1958 int sock_cmsg_send(struct sock *sk, struct msghdr *msg,
1959 struct sockcm_cookie *sockc)
1960 {
1961 struct cmsghdr *cmsg;
1962 int ret;
1963
1964 for_each_cmsghdr(cmsg, msg) {
1965 if (!CMSG_OK(msg, cmsg))
1966 return -EINVAL;
1967 if (cmsg->cmsg_level != SOL_SOCKET)
1968 continue;
1969 ret = __sock_cmsg_send(sk, msg, cmsg, sockc);
1970 if (ret)
1971 return ret;
1972 }
1973 return 0;
1974 }
1975 EXPORT_SYMBOL(sock_cmsg_send);
1976
1977 /* On 32bit arches, an skb frag is limited to 2^15 */
1978 #define SKB_FRAG_PAGE_ORDER get_order(32768)
1979
1980 /**
1981 * skb_page_frag_refill - check that a page_frag contains enough room
1982 * @sz: minimum size of the fragment we want to get
1983 * @pfrag: pointer to page_frag
1984 * @gfp: priority for memory allocation
1985 *
1986 * Note: While this allocator tries to use high order pages, there is
1987 * no guarantee that allocations succeed. Therefore, @sz MUST be
1988 * less or equal than PAGE_SIZE.
1989 */
1990 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp)
1991 {
1992 if (pfrag->page) {
1993 if (page_ref_count(pfrag->page) == 1) {
1994 pfrag->offset = 0;
1995 return true;
1996 }
1997 if (pfrag->offset + sz <= pfrag->size)
1998 return true;
1999 put_page(pfrag->page);
2000 }
2001
2002 pfrag->offset = 0;
2003 if (SKB_FRAG_PAGE_ORDER) {
2004 /* Avoid direct reclaim but allow kswapd to wake */
2005 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) |
2006 __GFP_COMP | __GFP_NOWARN |
2007 __GFP_NORETRY,
2008 SKB_FRAG_PAGE_ORDER);
2009 if (likely(pfrag->page)) {
2010 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER;
2011 return true;
2012 }
2013 }
2014 pfrag->page = alloc_page(gfp);
2015 if (likely(pfrag->page)) {
2016 pfrag->size = PAGE_SIZE;
2017 return true;
2018 }
2019 return false;
2020 }
2021 EXPORT_SYMBOL(skb_page_frag_refill);
2022
2023 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
2024 {
2025 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation)))
2026 return true;
2027
2028 sk_enter_memory_pressure(sk);
2029 sk_stream_moderate_sndbuf(sk);
2030 return false;
2031 }
2032 EXPORT_SYMBOL(sk_page_frag_refill);
2033
2034 static void __lock_sock(struct sock *sk)
2035 __releases(&sk->sk_lock.slock)
2036 __acquires(&sk->sk_lock.slock)
2037 {
2038 DEFINE_WAIT(wait);
2039
2040 for (;;) {
2041 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait,
2042 TASK_UNINTERRUPTIBLE);
2043 spin_unlock_bh(&sk->sk_lock.slock);
2044 schedule();
2045 spin_lock_bh(&sk->sk_lock.slock);
2046 if (!sock_owned_by_user(sk))
2047 break;
2048 }
2049 finish_wait(&sk->sk_lock.wq, &wait);
2050 }
2051
2052 static void __release_sock(struct sock *sk)
2053 __releases(&sk->sk_lock.slock)
2054 __acquires(&sk->sk_lock.slock)
2055 {
2056 struct sk_buff *skb, *next;
2057
2058 while ((skb = sk->sk_backlog.head) != NULL) {
2059 sk->sk_backlog.head = sk->sk_backlog.tail = NULL;
2060
2061 spin_unlock_bh(&sk->sk_lock.slock);
2062
2063 do {
2064 next = skb->next;
2065 prefetch(next);
2066 WARN_ON_ONCE(skb_dst_is_noref(skb));
2067 skb->next = NULL;
2068 sk_backlog_rcv(sk, skb);
2069
2070 cond_resched();
2071
2072 skb = next;
2073 } while (skb != NULL);
2074
2075 spin_lock_bh(&sk->sk_lock.slock);
2076 }
2077
2078 /*
2079 * Doing the zeroing here guarantee we can not loop forever
2080 * while a wild producer attempts to flood us.
2081 */
2082 sk->sk_backlog.len = 0;
2083 }
2084
2085 void __sk_flush_backlog(struct sock *sk)
2086 {
2087 spin_lock_bh(&sk->sk_lock.slock);
2088 __release_sock(sk);
2089 spin_unlock_bh(&sk->sk_lock.slock);
2090 }
2091
2092 /**
2093 * sk_wait_data - wait for data to arrive at sk_receive_queue
2094 * @sk: sock to wait on
2095 * @timeo: for how long
2096 * @skb: last skb seen on sk_receive_queue
2097 *
2098 * Now socket state including sk->sk_err is changed only under lock,
2099 * hence we may omit checks after joining wait queue.
2100 * We check receive queue before schedule() only as optimization;
2101 * it is very likely that release_sock() added new data.
2102 */
2103 int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb)
2104 {
2105 int rc;
2106 DEFINE_WAIT(wait);
2107
2108 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
2109 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
2110 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb);
2111 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
2112 finish_wait(sk_sleep(sk), &wait);
2113 return rc;
2114 }
2115 EXPORT_SYMBOL(sk_wait_data);
2116
2117 /**
2118 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated
2119 * @sk: socket
2120 * @size: memory size to allocate
2121 * @kind: allocation type
2122 *
2123 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means
2124 * rmem allocation. This function assumes that protocols which have
2125 * memory_pressure use sk_wmem_queued as write buffer accounting.
2126 */
2127 int __sk_mem_schedule(struct sock *sk, int size, int kind)
2128 {
2129 struct proto *prot = sk->sk_prot;
2130 int amt = sk_mem_pages(size);
2131 long allocated;
2132
2133 sk->sk_forward_alloc += amt * SK_MEM_QUANTUM;
2134
2135 allocated = sk_memory_allocated_add(sk, amt);
2136
2137 if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
2138 !mem_cgroup_charge_skmem(sk->sk_memcg, amt))
2139 goto suppress_allocation;
2140
2141 /* Under limit. */
2142 if (allocated <= sk_prot_mem_limits(sk, 0)) {
2143 sk_leave_memory_pressure(sk);
2144 return 1;
2145 }
2146
2147 /* Under pressure. */
2148 if (allocated > sk_prot_mem_limits(sk, 1))
2149 sk_enter_memory_pressure(sk);
2150
2151 /* Over hard limit. */
2152 if (allocated > sk_prot_mem_limits(sk, 2))
2153 goto suppress_allocation;
2154
2155 /* guarantee minimum buffer size under pressure */
2156 if (kind == SK_MEM_RECV) {
2157 if (atomic_read(&sk->sk_rmem_alloc) < prot->sysctl_rmem[0])
2158 return 1;
2159
2160 } else { /* SK_MEM_SEND */
2161 if (sk->sk_type == SOCK_STREAM) {
2162 if (sk->sk_wmem_queued < prot->sysctl_wmem[0])
2163 return 1;
2164 } else if (atomic_read(&sk->sk_wmem_alloc) <
2165 prot->sysctl_wmem[0])
2166 return 1;
2167 }
2168
2169 if (sk_has_memory_pressure(sk)) {
2170 int alloc;
2171
2172 if (!sk_under_memory_pressure(sk))
2173 return 1;
2174 alloc = sk_sockets_allocated_read_positive(sk);
2175 if (sk_prot_mem_limits(sk, 2) > alloc *
2176 sk_mem_pages(sk->sk_wmem_queued +
2177 atomic_read(&sk->sk_rmem_alloc) +
2178 sk->sk_forward_alloc))
2179 return 1;
2180 }
2181
2182 suppress_allocation:
2183
2184 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) {
2185 sk_stream_moderate_sndbuf(sk);
2186
2187 /* Fail only if socket is _under_ its sndbuf.
2188 * In this case we cannot block, so that we have to fail.
2189 */
2190 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf)
2191 return 1;
2192 }
2193
2194 trace_sock_exceed_buf_limit(sk, prot, allocated);
2195
2196 /* Alas. Undo changes. */
2197 sk->sk_forward_alloc -= amt * SK_MEM_QUANTUM;
2198
2199 sk_memory_allocated_sub(sk, amt);
2200
2201 if (mem_cgroup_sockets_enabled && sk->sk_memcg)
2202 mem_cgroup_uncharge_skmem(sk->sk_memcg, amt);
2203
2204 return 0;
2205 }
2206 EXPORT_SYMBOL(__sk_mem_schedule);
2207
2208 /**
2209 * __sk_mem_reclaim - reclaim memory_allocated
2210 * @sk: socket
2211 * @amount: number of bytes (rounded down to a SK_MEM_QUANTUM multiple)
2212 */
2213 void __sk_mem_reclaim(struct sock *sk, int amount)
2214 {
2215 amount >>= SK_MEM_QUANTUM_SHIFT;
2216 sk_memory_allocated_sub(sk, amount);
2217 sk->sk_forward_alloc -= amount << SK_MEM_QUANTUM_SHIFT;
2218
2219 if (mem_cgroup_sockets_enabled && sk->sk_memcg)
2220 mem_cgroup_uncharge_skmem(sk->sk_memcg, amount);
2221
2222 if (sk_under_memory_pressure(sk) &&
2223 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)))
2224 sk_leave_memory_pressure(sk);
2225 }
2226 EXPORT_SYMBOL(__sk_mem_reclaim);
2227
2228 int sk_set_peek_off(struct sock *sk, int val)
2229 {
2230 if (val < 0)
2231 return -EINVAL;
2232
2233 sk->sk_peek_off = val;
2234 return 0;
2235 }
2236 EXPORT_SYMBOL_GPL(sk_set_peek_off);
2237
2238 /*
2239 * Set of default routines for initialising struct proto_ops when
2240 * the protocol does not support a particular function. In certain
2241 * cases where it makes no sense for a protocol to have a "do nothing"
2242 * function, some default processing is provided.
2243 */
2244
2245 int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len)
2246 {
2247 return -EOPNOTSUPP;
2248 }
2249 EXPORT_SYMBOL(sock_no_bind);
2250
2251 int sock_no_connect(struct socket *sock, struct sockaddr *saddr,
2252 int len, int flags)
2253 {
2254 return -EOPNOTSUPP;
2255 }
2256 EXPORT_SYMBOL(sock_no_connect);
2257
2258 int sock_no_socketpair(struct socket *sock1, struct socket *sock2)
2259 {
2260 return -EOPNOTSUPP;
2261 }
2262 EXPORT_SYMBOL(sock_no_socketpair);
2263
2264 int sock_no_accept(struct socket *sock, struct socket *newsock, int flags)
2265 {
2266 return -EOPNOTSUPP;
2267 }
2268 EXPORT_SYMBOL(sock_no_accept);
2269
2270 int sock_no_getname(struct socket *sock, struct sockaddr *saddr,
2271 int *len, int peer)
2272 {
2273 return -EOPNOTSUPP;
2274 }
2275 EXPORT_SYMBOL(sock_no_getname);
2276
2277 unsigned int sock_no_poll(struct file *file, struct socket *sock, poll_table *pt)
2278 {
2279 return 0;
2280 }
2281 EXPORT_SYMBOL(sock_no_poll);
2282
2283 int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
2284 {
2285 return -EOPNOTSUPP;
2286 }
2287 EXPORT_SYMBOL(sock_no_ioctl);
2288
2289 int sock_no_listen(struct socket *sock, int backlog)
2290 {
2291 return -EOPNOTSUPP;
2292 }
2293 EXPORT_SYMBOL(sock_no_listen);
2294
2295 int sock_no_shutdown(struct socket *sock, int how)
2296 {
2297 return -EOPNOTSUPP;
2298 }
2299 EXPORT_SYMBOL(sock_no_shutdown);
2300
2301 int sock_no_setsockopt(struct socket *sock, int level, int optname,
2302 char __user *optval, unsigned int optlen)
2303 {
2304 return -EOPNOTSUPP;
2305 }
2306 EXPORT_SYMBOL(sock_no_setsockopt);
2307
2308 int sock_no_getsockopt(struct socket *sock, int level, int optname,
2309 char __user *optval, int __user *optlen)
2310 {
2311 return -EOPNOTSUPP;
2312 }
2313 EXPORT_SYMBOL(sock_no_getsockopt);
2314
2315 int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len)
2316 {
2317 return -EOPNOTSUPP;
2318 }
2319 EXPORT_SYMBOL(sock_no_sendmsg);
2320
2321 int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len,
2322 int flags)
2323 {
2324 return -EOPNOTSUPP;
2325 }
2326 EXPORT_SYMBOL(sock_no_recvmsg);
2327
2328 int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma)
2329 {
2330 /* Mirror missing mmap method error code */
2331 return -ENODEV;
2332 }
2333 EXPORT_SYMBOL(sock_no_mmap);
2334
2335 ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags)
2336 {
2337 ssize_t res;
2338 struct msghdr msg = {.msg_flags = flags};
2339 struct kvec iov;
2340 char *kaddr = kmap(page);
2341 iov.iov_base = kaddr + offset;
2342 iov.iov_len = size;
2343 res = kernel_sendmsg(sock, &msg, &iov, 1, size);
2344 kunmap(page);
2345 return res;
2346 }
2347 EXPORT_SYMBOL(sock_no_sendpage);
2348
2349 /*
2350 * Default Socket Callbacks
2351 */
2352
2353 static void sock_def_wakeup(struct sock *sk)
2354 {
2355 struct socket_wq *wq;
2356
2357 rcu_read_lock();
2358 wq = rcu_dereference(sk->sk_wq);
2359 if (skwq_has_sleeper(wq))
2360 wake_up_interruptible_all(&wq->wait);
2361 rcu_read_unlock();
2362 }
2363
2364 static void sock_def_error_report(struct sock *sk)
2365 {
2366 struct socket_wq *wq;
2367
2368 rcu_read_lock();
2369 wq = rcu_dereference(sk->sk_wq);
2370 if (skwq_has_sleeper(wq))
2371 wake_up_interruptible_poll(&wq->wait, POLLERR);
2372 sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR);
2373 rcu_read_unlock();
2374 }
2375
2376 static void sock_def_readable(struct sock *sk)
2377 {
2378 struct socket_wq *wq;
2379
2380 rcu_read_lock();
2381 wq = rcu_dereference(sk->sk_wq);
2382 if (skwq_has_sleeper(wq))
2383 wake_up_interruptible_sync_poll(&wq->wait, POLLIN | POLLPRI |
2384 POLLRDNORM | POLLRDBAND);
2385 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
2386 rcu_read_unlock();
2387 }
2388
2389 static void sock_def_write_space(struct sock *sk)
2390 {
2391 struct socket_wq *wq;
2392
2393 rcu_read_lock();
2394
2395 /* Do not wake up a writer until he can make "significant"
2396 * progress. --DaveM
2397 */
2398 if ((atomic_read(&sk->sk_wmem_alloc) << 1) <= sk->sk_sndbuf) {
2399 wq = rcu_dereference(sk->sk_wq);
2400 if (skwq_has_sleeper(wq))
2401 wake_up_interruptible_sync_poll(&wq->wait, POLLOUT |
2402 POLLWRNORM | POLLWRBAND);
2403
2404 /* Should agree with poll, otherwise some programs break */
2405 if (sock_writeable(sk))
2406 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
2407 }
2408
2409 rcu_read_unlock();
2410 }
2411
2412 static void sock_def_destruct(struct sock *sk)
2413 {
2414 }
2415
2416 void sk_send_sigurg(struct sock *sk)
2417 {
2418 if (sk->sk_socket && sk->sk_socket->file)
2419 if (send_sigurg(&sk->sk_socket->file->f_owner))
2420 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI);
2421 }
2422 EXPORT_SYMBOL(sk_send_sigurg);
2423
2424 void sk_reset_timer(struct sock *sk, struct timer_list* timer,
2425 unsigned long expires)
2426 {
2427 if (!mod_timer(timer, expires))
2428 sock_hold(sk);
2429 }
2430 EXPORT_SYMBOL(sk_reset_timer);
2431
2432 void sk_stop_timer(struct sock *sk, struct timer_list* timer)
2433 {
2434 if (del_timer(timer))
2435 __sock_put(sk);
2436 }
2437 EXPORT_SYMBOL(sk_stop_timer);
2438
2439 void sock_init_data(struct socket *sock, struct sock *sk)
2440 {
2441 skb_queue_head_init(&sk->sk_receive_queue);
2442 skb_queue_head_init(&sk->sk_write_queue);
2443 skb_queue_head_init(&sk->sk_error_queue);
2444
2445 sk->sk_send_head = NULL;
2446
2447 init_timer(&sk->sk_timer);
2448
2449 sk->sk_allocation = GFP_KERNEL;
2450 sk->sk_rcvbuf = sysctl_rmem_default;
2451 sk->sk_sndbuf = sysctl_wmem_default;
2452 sk->sk_state = TCP_CLOSE;
2453 sk_set_socket(sk, sock);
2454
2455 sock_set_flag(sk, SOCK_ZAPPED);
2456
2457 if (sock) {
2458 sk->sk_type = sock->type;
2459 sk->sk_wq = sock->wq;
2460 sock->sk = sk;
2461 } else
2462 sk->sk_wq = NULL;
2463
2464 rwlock_init(&sk->sk_callback_lock);
2465 lockdep_set_class_and_name(&sk->sk_callback_lock,
2466 af_callback_keys + sk->sk_family,
2467 af_family_clock_key_strings[sk->sk_family]);
2468
2469 sk->sk_state_change = sock_def_wakeup;
2470 sk->sk_data_ready = sock_def_readable;
2471 sk->sk_write_space = sock_def_write_space;
2472 sk->sk_error_report = sock_def_error_report;
2473 sk->sk_destruct = sock_def_destruct;
2474
2475 sk->sk_frag.page = NULL;
2476 sk->sk_frag.offset = 0;
2477 sk->sk_peek_off = -1;
2478
2479 sk->sk_peer_pid = NULL;
2480 sk->sk_peer_cred = NULL;
2481 sk->sk_write_pending = 0;
2482 sk->sk_rcvlowat = 1;
2483 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT;
2484 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT;
2485
2486 sk->sk_stamp = ktime_set(-1L, 0);
2487
2488 #ifdef CONFIG_NET_RX_BUSY_POLL
2489 sk->sk_napi_id = 0;
2490 sk->sk_ll_usec = sysctl_net_busy_read;
2491 #endif
2492
2493 sk->sk_max_pacing_rate = ~0U;
2494 sk->sk_pacing_rate = ~0U;
2495 sk->sk_incoming_cpu = -1;
2496 /*
2497 * Before updating sk_refcnt, we must commit prior changes to memory
2498 * (Documentation/RCU/rculist_nulls.txt for details)
2499 */
2500 smp_wmb();
2501 atomic_set(&sk->sk_refcnt, 1);
2502 atomic_set(&sk->sk_drops, 0);
2503 }
2504 EXPORT_SYMBOL(sock_init_data);
2505
2506 void lock_sock_nested(struct sock *sk, int subclass)
2507 {
2508 might_sleep();
2509 spin_lock_bh(&sk->sk_lock.slock);
2510 if (sk->sk_lock.owned)
2511 __lock_sock(sk);
2512 sk->sk_lock.owned = 1;
2513 spin_unlock(&sk->sk_lock.slock);
2514 /*
2515 * The sk_lock has mutex_lock() semantics here:
2516 */
2517 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_);
2518 local_bh_enable();
2519 }
2520 EXPORT_SYMBOL(lock_sock_nested);
2521
2522 void release_sock(struct sock *sk)
2523 {
2524 spin_lock_bh(&sk->sk_lock.slock);
2525 if (sk->sk_backlog.tail)
2526 __release_sock(sk);
2527
2528 /* Warning : release_cb() might need to release sk ownership,
2529 * ie call sock_release_ownership(sk) before us.
2530 */
2531 if (sk->sk_prot->release_cb)
2532 sk->sk_prot->release_cb(sk);
2533
2534 sock_release_ownership(sk);
2535 if (waitqueue_active(&sk->sk_lock.wq))
2536 wake_up(&sk->sk_lock.wq);
2537 spin_unlock_bh(&sk->sk_lock.slock);
2538 }
2539 EXPORT_SYMBOL(release_sock);
2540
2541 /**
2542 * lock_sock_fast - fast version of lock_sock
2543 * @sk: socket
2544 *
2545 * This version should be used for very small section, where process wont block
2546 * return false if fast path is taken
2547 * sk_lock.slock locked, owned = 0, BH disabled
2548 * return true if slow path is taken
2549 * sk_lock.slock unlocked, owned = 1, BH enabled
2550 */
2551 bool lock_sock_fast(struct sock *sk)
2552 {
2553 might_sleep();
2554 spin_lock_bh(&sk->sk_lock.slock);
2555
2556 if (!sk->sk_lock.owned)
2557 /*
2558 * Note : We must disable BH
2559 */
2560 return false;
2561
2562 __lock_sock(sk);
2563 sk->sk_lock.owned = 1;
2564 spin_unlock(&sk->sk_lock.slock);
2565 /*
2566 * The sk_lock has mutex_lock() semantics here:
2567 */
2568 mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_);
2569 local_bh_enable();
2570 return true;
2571 }
2572 EXPORT_SYMBOL(lock_sock_fast);
2573
2574 int sock_get_timestamp(struct sock *sk, struct timeval __user *userstamp)
2575 {
2576 struct timeval tv;
2577 if (!sock_flag(sk, SOCK_TIMESTAMP))
2578 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
2579 tv = ktime_to_timeval(sk->sk_stamp);
2580 if (tv.tv_sec == -1)
2581 return -ENOENT;
2582 if (tv.tv_sec == 0) {
2583 sk->sk_stamp = ktime_get_real();
2584 tv = ktime_to_timeval(sk->sk_stamp);
2585 }
2586 return copy_to_user(userstamp, &tv, sizeof(tv)) ? -EFAULT : 0;
2587 }
2588 EXPORT_SYMBOL(sock_get_timestamp);
2589
2590 int sock_get_timestampns(struct sock *sk, struct timespec __user *userstamp)
2591 {
2592 struct timespec ts;
2593 if (!sock_flag(sk, SOCK_TIMESTAMP))
2594 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
2595 ts = ktime_to_timespec(sk->sk_stamp);
2596 if (ts.tv_sec == -1)
2597 return -ENOENT;
2598 if (ts.tv_sec == 0) {
2599 sk->sk_stamp = ktime_get_real();
2600 ts = ktime_to_timespec(sk->sk_stamp);
2601 }
2602 return copy_to_user(userstamp, &ts, sizeof(ts)) ? -EFAULT : 0;
2603 }
2604 EXPORT_SYMBOL(sock_get_timestampns);
2605
2606 void sock_enable_timestamp(struct sock *sk, int flag)
2607 {
2608 if (!sock_flag(sk, flag)) {
2609 unsigned long previous_flags = sk->sk_flags;
2610
2611 sock_set_flag(sk, flag);
2612 /*
2613 * we just set one of the two flags which require net
2614 * time stamping, but time stamping might have been on
2615 * already because of the other one
2616 */
2617 if (sock_needs_netstamp(sk) &&
2618 !(previous_flags & SK_FLAGS_TIMESTAMP))
2619 net_enable_timestamp();
2620 }
2621 }
2622
2623 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len,
2624 int level, int type)
2625 {
2626 struct sock_exterr_skb *serr;
2627 struct sk_buff *skb;
2628 int copied, err;
2629
2630 err = -EAGAIN;
2631 skb = sock_dequeue_err_skb(sk);
2632 if (skb == NULL)
2633 goto out;
2634
2635 copied = skb->len;
2636 if (copied > len) {
2637 msg->msg_flags |= MSG_TRUNC;
2638 copied = len;
2639 }
2640 err = skb_copy_datagram_msg(skb, 0, msg, copied);
2641 if (err)
2642 goto out_free_skb;
2643
2644 sock_recv_timestamp(msg, sk, skb);
2645
2646 serr = SKB_EXT_ERR(skb);
2647 put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee);
2648
2649 msg->msg_flags |= MSG_ERRQUEUE;
2650 err = copied;
2651
2652 out_free_skb:
2653 kfree_skb(skb);
2654 out:
2655 return err;
2656 }
2657 EXPORT_SYMBOL(sock_recv_errqueue);
2658
2659 /*
2660 * Get a socket option on an socket.
2661 *
2662 * FIX: POSIX 1003.1g is very ambiguous here. It states that
2663 * asynchronous errors should be reported by getsockopt. We assume
2664 * this means if you specify SO_ERROR (otherwise whats the point of it).
2665 */
2666 int sock_common_getsockopt(struct socket *sock, int level, int optname,
2667 char __user *optval, int __user *optlen)
2668 {
2669 struct sock *sk = sock->sk;
2670
2671 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen);
2672 }
2673 EXPORT_SYMBOL(sock_common_getsockopt);
2674
2675 #ifdef CONFIG_COMPAT
2676 int compat_sock_common_getsockopt(struct socket *sock, int level, int optname,
2677 char __user *optval, int __user *optlen)
2678 {
2679 struct sock *sk = sock->sk;
2680
2681 if (sk->sk_prot->compat_getsockopt != NULL)
2682 return sk->sk_prot->compat_getsockopt(sk, level, optname,
2683 optval, optlen);
2684 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen);
2685 }
2686 EXPORT_SYMBOL(compat_sock_common_getsockopt);
2687 #endif
2688
2689 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
2690 int flags)
2691 {
2692 struct sock *sk = sock->sk;
2693 int addr_len = 0;
2694 int err;
2695
2696 err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT,
2697 flags & ~MSG_DONTWAIT, &addr_len);
2698 if (err >= 0)
2699 msg->msg_namelen = addr_len;
2700 return err;
2701 }
2702 EXPORT_SYMBOL(sock_common_recvmsg);
2703
2704 /*
2705 * Set socket options on an inet socket.
2706 */
2707 int sock_common_setsockopt(struct socket *sock, int level, int optname,
2708 char __user *optval, unsigned int optlen)
2709 {
2710 struct sock *sk = sock->sk;
2711
2712 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen);
2713 }
2714 EXPORT_SYMBOL(sock_common_setsockopt);
2715
2716 #ifdef CONFIG_COMPAT
2717 int compat_sock_common_setsockopt(struct socket *sock, int level, int optname,
2718 char __user *optval, unsigned int optlen)
2719 {
2720 struct sock *sk = sock->sk;
2721
2722 if (sk->sk_prot->compat_setsockopt != NULL)
2723 return sk->sk_prot->compat_setsockopt(sk, level, optname,
2724 optval, optlen);
2725 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen);
2726 }
2727 EXPORT_SYMBOL(compat_sock_common_setsockopt);
2728 #endif
2729
2730 void sk_common_release(struct sock *sk)
2731 {
2732 if (sk->sk_prot->destroy)
2733 sk->sk_prot->destroy(sk);
2734
2735 /*
2736 * Observation: when sock_common_release is called, processes have
2737 * no access to socket. But net still has.
2738 * Step one, detach it from networking:
2739 *
2740 * A. Remove from hash tables.
2741 */
2742
2743 sk->sk_prot->unhash(sk);
2744
2745 /*
2746 * In this point socket cannot receive new packets, but it is possible
2747 * that some packets are in flight because some CPU runs receiver and
2748 * did hash table lookup before we unhashed socket. They will achieve
2749 * receive queue and will be purged by socket destructor.
2750 *
2751 * Also we still have packets pending on receive queue and probably,
2752 * our own packets waiting in device queues. sock_destroy will drain
2753 * receive queue, but transmitted packets will delay socket destruction
2754 * until the last reference will be released.
2755 */
2756
2757 sock_orphan(sk);
2758
2759 xfrm_sk_free_policy(sk);
2760
2761 sk_refcnt_debug_release(sk);
2762
2763 if (sk->sk_frag.page) {
2764 put_page(sk->sk_frag.page);
2765 sk->sk_frag.page = NULL;
2766 }
2767
2768 sock_put(sk);
2769 }
2770 EXPORT_SYMBOL(sk_common_release);
2771
2772 #ifdef CONFIG_PROC_FS
2773 #define PROTO_INUSE_NR 64 /* should be enough for the first time */
2774 struct prot_inuse {
2775 int val[PROTO_INUSE_NR];
2776 };
2777
2778 static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR);
2779
2780 #ifdef CONFIG_NET_NS
2781 void sock_prot_inuse_add(struct net *net, struct proto *prot, int val)
2782 {
2783 __this_cpu_add(net->core.inuse->val[prot->inuse_idx], val);
2784 }
2785 EXPORT_SYMBOL_GPL(sock_prot_inuse_add);
2786
2787 int sock_prot_inuse_get(struct net *net, struct proto *prot)
2788 {
2789 int cpu, idx = prot->inuse_idx;
2790 int res = 0;
2791
2792 for_each_possible_cpu(cpu)
2793 res += per_cpu_ptr(net->core.inuse, cpu)->val[idx];
2794
2795 return res >= 0 ? res : 0;
2796 }
2797 EXPORT_SYMBOL_GPL(sock_prot_inuse_get);
2798
2799 static int __net_init sock_inuse_init_net(struct net *net)
2800 {
2801 net->core.inuse = alloc_percpu(struct prot_inuse);
2802 return net->core.inuse ? 0 : -ENOMEM;
2803 }
2804
2805 static void __net_exit sock_inuse_exit_net(struct net *net)
2806 {
2807 free_percpu(net->core.inuse);
2808 }
2809
2810 static struct pernet_operations net_inuse_ops = {
2811 .init = sock_inuse_init_net,
2812 .exit = sock_inuse_exit_net,
2813 };
2814
2815 static __init int net_inuse_init(void)
2816 {
2817 if (register_pernet_subsys(&net_inuse_ops))
2818 panic("Cannot initialize net inuse counters");
2819
2820 return 0;
2821 }
2822
2823 core_initcall(net_inuse_init);
2824 #else
2825 static DEFINE_PER_CPU(struct prot_inuse, prot_inuse);
2826
2827 void sock_prot_inuse_add(struct net *net, struct proto *prot, int val)
2828 {
2829 __this_cpu_add(prot_inuse.val[prot->inuse_idx], val);
2830 }
2831 EXPORT_SYMBOL_GPL(sock_prot_inuse_add);
2832
2833 int sock_prot_inuse_get(struct net *net, struct proto *prot)
2834 {
2835 int cpu, idx = prot->inuse_idx;
2836 int res = 0;
2837
2838 for_each_possible_cpu(cpu)
2839 res += per_cpu(prot_inuse, cpu).val[idx];
2840
2841 return res >= 0 ? res : 0;
2842 }
2843 EXPORT_SYMBOL_GPL(sock_prot_inuse_get);
2844 #endif
2845
2846 static void assign_proto_idx(struct proto *prot)
2847 {
2848 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR);
2849
2850 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) {
2851 pr_err("PROTO_INUSE_NR exhausted\n");
2852 return;
2853 }
2854
2855 set_bit(prot->inuse_idx, proto_inuse_idx);
2856 }
2857
2858 static void release_proto_idx(struct proto *prot)
2859 {
2860 if (prot->inuse_idx != PROTO_INUSE_NR - 1)
2861 clear_bit(prot->inuse_idx, proto_inuse_idx);
2862 }
2863 #else
2864 static inline void assign_proto_idx(struct proto *prot)
2865 {
2866 }
2867
2868 static inline void release_proto_idx(struct proto *prot)
2869 {
2870 }
2871 #endif
2872
2873 static void req_prot_cleanup(struct request_sock_ops *rsk_prot)
2874 {
2875 if (!rsk_prot)
2876 return;
2877 kfree(rsk_prot->slab_name);
2878 rsk_prot->slab_name = NULL;
2879 kmem_cache_destroy(rsk_prot->slab);
2880 rsk_prot->slab = NULL;
2881 }
2882
2883 static int req_prot_init(const struct proto *prot)
2884 {
2885 struct request_sock_ops *rsk_prot = prot->rsk_prot;
2886
2887 if (!rsk_prot)
2888 return 0;
2889
2890 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s",
2891 prot->name);
2892 if (!rsk_prot->slab_name)
2893 return -ENOMEM;
2894
2895 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name,
2896 rsk_prot->obj_size, 0,
2897 prot->slab_flags, NULL);
2898
2899 if (!rsk_prot->slab) {
2900 pr_crit("%s: Can't create request sock SLAB cache!\n",
2901 prot->name);
2902 return -ENOMEM;
2903 }
2904 return 0;
2905 }
2906
2907 int proto_register(struct proto *prot, int alloc_slab)
2908 {
2909 if (alloc_slab) {
2910 prot->slab = kmem_cache_create(prot->name, prot->obj_size, 0,
2911 SLAB_HWCACHE_ALIGN | prot->slab_flags,
2912 NULL);
2913
2914 if (prot->slab == NULL) {
2915 pr_crit("%s: Can't create sock SLAB cache!\n",
2916 prot->name);
2917 goto out;
2918 }
2919
2920 if (req_prot_init(prot))
2921 goto out_free_request_sock_slab;
2922
2923 if (prot->twsk_prot != NULL) {
2924 prot->twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", prot->name);
2925
2926 if (prot->twsk_prot->twsk_slab_name == NULL)
2927 goto out_free_request_sock_slab;
2928
2929 prot->twsk_prot->twsk_slab =
2930 kmem_cache_create(prot->twsk_prot->twsk_slab_name,
2931 prot->twsk_prot->twsk_obj_size,
2932 0,
2933 prot->slab_flags,
2934 NULL);
2935 if (prot->twsk_prot->twsk_slab == NULL)
2936 goto out_free_timewait_sock_slab_name;
2937 }
2938 }
2939
2940 mutex_lock(&proto_list_mutex);
2941 list_add(&prot->node, &proto_list);
2942 assign_proto_idx(prot);
2943 mutex_unlock(&proto_list_mutex);
2944 return 0;
2945
2946 out_free_timewait_sock_slab_name:
2947 kfree(prot->twsk_prot->twsk_slab_name);
2948 out_free_request_sock_slab:
2949 req_prot_cleanup(prot->rsk_prot);
2950
2951 kmem_cache_destroy(prot->slab);
2952 prot->slab = NULL;
2953 out:
2954 return -ENOBUFS;
2955 }
2956 EXPORT_SYMBOL(proto_register);
2957
2958 void proto_unregister(struct proto *prot)
2959 {
2960 mutex_lock(&proto_list_mutex);
2961 release_proto_idx(prot);
2962 list_del(&prot->node);
2963 mutex_unlock(&proto_list_mutex);
2964
2965 kmem_cache_destroy(prot->slab);
2966 prot->slab = NULL;
2967
2968 req_prot_cleanup(prot->rsk_prot);
2969
2970 if (prot->twsk_prot != NULL && prot->twsk_prot->twsk_slab != NULL) {
2971 kmem_cache_destroy(prot->twsk_prot->twsk_slab);
2972 kfree(prot->twsk_prot->twsk_slab_name);
2973 prot->twsk_prot->twsk_slab = NULL;
2974 }
2975 }
2976 EXPORT_SYMBOL(proto_unregister);
2977
2978 #ifdef CONFIG_PROC_FS
2979 static void *proto_seq_start(struct seq_file *seq, loff_t *pos)
2980 __acquires(proto_list_mutex)
2981 {
2982 mutex_lock(&proto_list_mutex);
2983 return seq_list_start_head(&proto_list, *pos);
2984 }
2985
2986 static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2987 {
2988 return seq_list_next(v, &proto_list, pos);
2989 }
2990
2991 static void proto_seq_stop(struct seq_file *seq, void *v)
2992 __releases(proto_list_mutex)
2993 {
2994 mutex_unlock(&proto_list_mutex);
2995 }
2996
2997 static char proto_method_implemented(const void *method)
2998 {
2999 return method == NULL ? 'n' : 'y';
3000 }
3001 static long sock_prot_memory_allocated(struct proto *proto)
3002 {
3003 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L;
3004 }
3005
3006 static char *sock_prot_memory_pressure(struct proto *proto)
3007 {
3008 return proto->memory_pressure != NULL ?
3009 proto_memory_pressure(proto) ? "yes" : "no" : "NI";
3010 }
3011
3012 static void proto_seq_printf(struct seq_file *seq, struct proto *proto)
3013 {
3014
3015 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s "
3016 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n",
3017 proto->name,
3018 proto->obj_size,
3019 sock_prot_inuse_get(seq_file_net(seq), proto),
3020 sock_prot_memory_allocated(proto),
3021 sock_prot_memory_pressure(proto),
3022 proto->max_header,
3023 proto->slab == NULL ? "no" : "yes",
3024 module_name(proto->owner),
3025 proto_method_implemented(proto->close),
3026 proto_method_implemented(proto->connect),
3027 proto_method_implemented(proto->disconnect),
3028 proto_method_implemented(proto->accept),
3029 proto_method_implemented(proto->ioctl),
3030 proto_method_implemented(proto->init),
3031 proto_method_implemented(proto->destroy),
3032 proto_method_implemented(proto->shutdown),
3033 proto_method_implemented(proto->setsockopt),
3034 proto_method_implemented(proto->getsockopt),
3035 proto_method_implemented(proto->sendmsg),
3036 proto_method_implemented(proto->recvmsg),
3037 proto_method_implemented(proto->sendpage),
3038 proto_method_implemented(proto->bind),
3039 proto_method_implemented(proto->backlog_rcv),
3040 proto_method_implemented(proto->hash),
3041 proto_method_implemented(proto->unhash),
3042 proto_method_implemented(proto->get_port),
3043 proto_method_implemented(proto->enter_memory_pressure));
3044 }
3045
3046 static int proto_seq_show(struct seq_file *seq, void *v)
3047 {
3048 if (v == &proto_list)
3049 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s",
3050 "protocol",
3051 "size",
3052 "sockets",
3053 "memory",
3054 "press",
3055 "maxhdr",
3056 "slab",
3057 "module",
3058 "cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n");
3059 else
3060 proto_seq_printf(seq, list_entry(v, struct proto, node));
3061 return 0;
3062 }
3063
3064 static const struct seq_operations proto_seq_ops = {
3065 .start = proto_seq_start,
3066 .next = proto_seq_next,
3067 .stop = proto_seq_stop,
3068 .show = proto_seq_show,
3069 };
3070
3071 static int proto_seq_open(struct inode *inode, struct file *file)
3072 {
3073 return seq_open_net(inode, file, &proto_seq_ops,
3074 sizeof(struct seq_net_private));
3075 }
3076
3077 static const struct file_operations proto_seq_fops = {
3078 .owner = THIS_MODULE,
3079 .open = proto_seq_open,
3080 .read = seq_read,
3081 .llseek = seq_lseek,
3082 .release = seq_release_net,
3083 };
3084
3085 static __net_init int proto_init_net(struct net *net)
3086 {
3087 if (!proc_create("protocols", S_IRUGO, net->proc_net, &proto_seq_fops))
3088 return -ENOMEM;
3089
3090 return 0;
3091 }
3092
3093 static __net_exit void proto_exit_net(struct net *net)
3094 {
3095 remove_proc_entry("protocols", net->proc_net);
3096 }
3097
3098
3099 static __net_initdata struct pernet_operations proto_net_ops = {
3100 .init = proto_init_net,
3101 .exit = proto_exit_net,
3102 };
3103
3104 static int __init proto_init(void)
3105 {
3106 return register_pernet_subsys(&proto_net_ops);
3107 }
3108
3109 subsys_initcall(proto_init);
3110
3111 #endif /* PROC_FS */
3112
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.