1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * NET3 Protocol independent device support routines.
4 *
5 * Derived from the non IP parts of dev.c 1.0.19
6 * Authors: Ross Biro
7 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
8 * Mark Evans, <evansmp@uhura.aston.ac.uk>
9 *
10 * Additional Authors:
11 * Florian la Roche <rzsfl@rz.uni-sb.de>
12 * Alan Cox <gw4pts@gw4pts.ampr.org>
13 * David Hinds <dahinds@users.sourceforge.net>
14 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
15 * Adam Sulmicki <adam@cfar.umd.edu>
16 * Pekka Riikonen <priikone@poesidon.pspt.fi>
17 *
18 * Changes:
19 * D.J. Barrow : Fixed bug where dev->refcnt gets set
20 * to 2 if register_netdev gets called
21 * before net_dev_init & also removed a
22 * few lines of code in the process.
23 * Alan Cox : device private ioctl copies fields back.
24 * Alan Cox : Transmit queue code does relevant
25 * stunts to keep the queue safe.
26 * Alan Cox : Fixed double lock.
27 * Alan Cox : Fixed promisc NULL pointer trap
28 * ???????? : Support the full private ioctl range
29 * Alan Cox : Moved ioctl permission check into
30 * drivers
31 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
32 * Alan Cox : 100 backlog just doesn't cut it when
33 * you start doing multicast video 8)
34 * Alan Cox : Rewrote net_bh and list manager.
35 * Alan Cox : Fix ETH_P_ALL echoback lengths.
36 * Alan Cox : Took out transmit every packet pass
37 * Saved a few bytes in the ioctl handler
38 * Alan Cox : Network driver sets packet type before
39 * calling netif_rx. Saves a function
40 * call a packet.
41 * Alan Cox : Hashed net_bh()
42 * Richard Kooijman: Timestamp fixes.
43 * Alan Cox : Wrong field in SIOCGIFDSTADDR
44 * Alan Cox : Device lock protection.
45 * Alan Cox : Fixed nasty side effect of device close
46 * changes.
47 * Rudi Cilibrasi : Pass the right thing to
48 * set_mac_address()
49 * Dave Miller : 32bit quantity for the device lock to
50 * make it work out on a Sparc.
51 * Bjorn Ekwall : Added KERNELD hack.
52 * Alan Cox : Cleaned up the backlog initialise.
53 * Craig Metz : SIOCGIFCONF fix if space for under
54 * 1 device.
55 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
56 * is no device open function.
57 * Andi Kleen : Fix error reporting for SIOCGIFCONF
58 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
59 * Cyrus Durgin : Cleaned for KMOD
60 * Adam Sulmicki : Bug Fix : Network Device Unload
61 * A network device unload needs to purge
62 * the backlog queue.
63 * Paul Rusty Russell : SIOCSIFNAME
64 * Pekka Riikonen : Netdev boot-time settings code
65 * Andrew Morton : Make unregister_netdevice wait
66 * indefinitely on dev->refcnt
67 * J Hadi Salim : - Backlog queue sampling
68 * - netif_rx() feedback
69 */
70
71 #include <linux/uaccess.h>
72 #include <linux/bitops.h>
73 #include <linux/capability.h>
74 #include <linux/cpu.h>
75 #include <linux/types.h>
76 #include <linux/kernel.h>
77 #include <linux/hash.h>
78 #include <linux/slab.h>
79 #include <linux/sched.h>
80 #include <linux/sched/mm.h>
81 #include <linux/mutex.h>
82 #include <linux/rwsem.h>
83 #include <linux/string.h>
84 #include <linux/mm.h>
85 #include <linux/socket.h>
86 #include <linux/sockios.h>
87 #include <linux/errno.h>
88 #include <linux/interrupt.h>
89 #include <linux/if_ether.h>
90 #include <linux/netdevice.h>
91 #include <linux/etherdevice.h>
92 #include <linux/ethtool.h>
93 #include <linux/skbuff.h>
94 #include <linux/kthread.h>
95 #include <linux/bpf.h>
96 #include <linux/bpf_trace.h>
97 #include <net/net_namespace.h>
98 #include <net/sock.h>
99 #include <net/busy_poll.h>
100 #include <linux/rtnetlink.h>
101 #include <linux/stat.h>
102 #include <net/dsa.h>
103 #include <net/dst.h>
104 #include <net/dst_metadata.h>
105 #include <net/gro.h>
106 #include <net/pkt_sched.h>
107 #include <net/pkt_cls.h>
108 #include <net/checksum.h>
109 #include <net/xfrm.h>
110 #include <linux/highmem.h>
111 #include <linux/init.h>
112 #include <linux/module.h>
113 #include <linux/netpoll.h>
114 #include <linux/rcupdate.h>
115 #include <linux/delay.h>
116 #include <net/iw_handler.h>
117 #include <asm/current.h>
118 #include <linux/audit.h>
119 #include <linux/dmaengine.h>
120 #include <linux/err.h>
121 #include <linux/ctype.h>
122 #include <linux/if_arp.h>
123 #include <linux/if_vlan.h>
124 #include <linux/ip.h>
125 #include <net/ip.h>
126 #include <net/mpls.h>
127 #include <linux/ipv6.h>
128 #include <linux/in.h>
129 #include <linux/jhash.h>
130 #include <linux/random.h>
131 #include <trace/events/napi.h>
132 #include <trace/events/net.h>
133 #include <trace/events/skb.h>
134 #include <trace/events/qdisc.h>
135 #include <linux/inetdevice.h>
136 #include <linux/cpu_rmap.h>
137 #include <linux/static_key.h>
138 #include <linux/hashtable.h>
139 #include <linux/vmalloc.h>
140 #include <linux/if_macvlan.h>
141 #include <linux/errqueue.h>
142 #include <linux/hrtimer.h>
143 #include <linux/netfilter_netdev.h>
144 #include <linux/crash_dump.h>
145 #include <linux/sctp.h>
146 #include <net/udp_tunnel.h>
147 #include <linux/net_namespace.h>
148 #include <linux/indirect_call_wrapper.h>
149 #include <net/devlink.h>
150 #include <linux/pm_runtime.h>
151 #include <linux/prandom.h>
152 #include <linux/once_lite.h>
153
154 #include "dev.h"
155 #include "net-sysfs.h"
156
157
158 static DEFINE_SPINLOCK(ptype_lock);
159 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
160 struct list_head ptype_all __read_mostly; /* Taps */
161
162 static int netif_rx_internal(struct sk_buff *skb);
163 static int call_netdevice_notifiers_info(unsigned long val,
164 struct netdev_notifier_info *info);
165 static int call_netdevice_notifiers_extack(unsigned long val,
166 struct net_device *dev,
167 struct netlink_ext_ack *extack);
168 static struct napi_struct *napi_by_id(unsigned int napi_id);
169
170 /*
171 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
172 * semaphore.
173 *
174 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
175 *
176 * Writers must hold the rtnl semaphore while they loop through the
177 * dev_base_head list, and hold dev_base_lock for writing when they do the
178 * actual updates. This allows pure readers to access the list even
179 * while a writer is preparing to update it.
180 *
181 * To put it another way, dev_base_lock is held for writing only to
182 * protect against pure readers; the rtnl semaphore provides the
183 * protection against other writers.
184 *
185 * See, for example usages, register_netdevice() and
186 * unregister_netdevice(), which must be called with the rtnl
187 * semaphore held.
188 */
189 DEFINE_RWLOCK(dev_base_lock);
190 EXPORT_SYMBOL(dev_base_lock);
191
192 static DEFINE_MUTEX(ifalias_mutex);
193
194 /* protects napi_hash addition/deletion and napi_gen_id */
195 static DEFINE_SPINLOCK(napi_hash_lock);
196
197 static unsigned int napi_gen_id = NR_CPUS;
198 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
199
200 static DECLARE_RWSEM(devnet_rename_sem);
201
202 static inline void dev_base_seq_inc(struct net *net)
203 {
204 while (++net->dev_base_seq == 0)
205 ;
206 }
207
208 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
209 {
210 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
211
212 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
213 }
214
215 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
216 {
217 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
218 }
219
220 static inline void rps_lock_irqsave(struct softnet_data *sd,
221 unsigned long *flags)
222 {
223 if (IS_ENABLED(CONFIG_RPS))
224 spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags);
225 else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
226 local_irq_save(*flags);
227 }
228
229 static inline void rps_lock_irq_disable(struct softnet_data *sd)
230 {
231 if (IS_ENABLED(CONFIG_RPS))
232 spin_lock_irq(&sd->input_pkt_queue.lock);
233 else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
234 local_irq_disable();
235 }
236
237 static inline void rps_unlock_irq_restore(struct softnet_data *sd,
238 unsigned long *flags)
239 {
240 if (IS_ENABLED(CONFIG_RPS))
241 spin_unlock_irqrestore(&sd->input_pkt_queue.lock, *flags);
242 else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
243 local_irq_restore(*flags);
244 }
245
246 static inline void rps_unlock_irq_enable(struct softnet_data *sd)
247 {
248 if (IS_ENABLED(CONFIG_RPS))
249 spin_unlock_irq(&sd->input_pkt_queue.lock);
250 else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
251 local_irq_enable();
252 }
253
254 static struct netdev_name_node *netdev_name_node_alloc(struct net_device *dev,
255 const char *name)
256 {
257 struct netdev_name_node *name_node;
258
259 name_node = kmalloc(sizeof(*name_node), GFP_KERNEL);
260 if (!name_node)
261 return NULL;
262 INIT_HLIST_NODE(&name_node->hlist);
263 name_node->dev = dev;
264 name_node->name = name;
265 return name_node;
266 }
267
268 static struct netdev_name_node *
269 netdev_name_node_head_alloc(struct net_device *dev)
270 {
271 struct netdev_name_node *name_node;
272
273 name_node = netdev_name_node_alloc(dev, dev->name);
274 if (!name_node)
275 return NULL;
276 INIT_LIST_HEAD(&name_node->list);
277 return name_node;
278 }
279
280 static void netdev_name_node_free(struct netdev_name_node *name_node)
281 {
282 kfree(name_node);
283 }
284
285 static void netdev_name_node_add(struct net *net,
286 struct netdev_name_node *name_node)
287 {
288 hlist_add_head_rcu(&name_node->hlist,
289 dev_name_hash(net, name_node->name));
290 }
291
292 static void netdev_name_node_del(struct netdev_name_node *name_node)
293 {
294 hlist_del_rcu(&name_node->hlist);
295 }
296
297 static struct netdev_name_node *netdev_name_node_lookup(struct net *net,
298 const char *name)
299 {
300 struct hlist_head *head = dev_name_hash(net, name);
301 struct netdev_name_node *name_node;
302
303 hlist_for_each_entry(name_node, head, hlist)
304 if (!strcmp(name_node->name, name))
305 return name_node;
306 return NULL;
307 }
308
309 static struct netdev_name_node *netdev_name_node_lookup_rcu(struct net *net,
310 const char *name)
311 {
312 struct hlist_head *head = dev_name_hash(net, name);
313 struct netdev_name_node *name_node;
314
315 hlist_for_each_entry_rcu(name_node, head, hlist)
316 if (!strcmp(name_node->name, name))
317 return name_node;
318 return NULL;
319 }
320
321 bool netdev_name_in_use(struct net *net, const char *name)
322 {
323 return netdev_name_node_lookup(net, name);
324 }
325 EXPORT_SYMBOL(netdev_name_in_use);
326
327 int netdev_name_node_alt_create(struct net_device *dev, const char *name)
328 {
329 struct netdev_name_node *name_node;
330 struct net *net = dev_net(dev);
331
332 name_node = netdev_name_node_lookup(net, name);
333 if (name_node)
334 return -EEXIST;
335 name_node = netdev_name_node_alloc(dev, name);
336 if (!name_node)
337 return -ENOMEM;
338 netdev_name_node_add(net, name_node);
339 /* The node that holds dev->name acts as a head of per-device list. */
340 list_add_tail(&name_node->list, &dev->name_node->list);
341
342 return 0;
343 }
344
345 static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node)
346 {
347 list_del(&name_node->list);
348 netdev_name_node_del(name_node);
349 kfree(name_node->name);
350 netdev_name_node_free(name_node);
351 }
352
353 int netdev_name_node_alt_destroy(struct net_device *dev, const char *name)
354 {
355 struct netdev_name_node *name_node;
356 struct net *net = dev_net(dev);
357
358 name_node = netdev_name_node_lookup(net, name);
359 if (!name_node)
360 return -ENOENT;
361 /* lookup might have found our primary name or a name belonging
362 * to another device.
363 */
364 if (name_node == dev->name_node || name_node->dev != dev)
365 return -EINVAL;
366
367 __netdev_name_node_alt_destroy(name_node);
368
369 return 0;
370 }
371
372 static void netdev_name_node_alt_flush(struct net_device *dev)
373 {
374 struct netdev_name_node *name_node, *tmp;
375
376 list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list)
377 __netdev_name_node_alt_destroy(name_node);
378 }
379
380 /* Device list insertion */
381 static void list_netdevice(struct net_device *dev)
382 {
383 struct net *net = dev_net(dev);
384
385 ASSERT_RTNL();
386
387 write_lock(&dev_base_lock);
388 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
389 netdev_name_node_add(net, dev->name_node);
390 hlist_add_head_rcu(&dev->index_hlist,
391 dev_index_hash(net, dev->ifindex));
392 write_unlock(&dev_base_lock);
393
394 dev_base_seq_inc(net);
395 }
396
397 /* Device list removal
398 * caller must respect a RCU grace period before freeing/reusing dev
399 */
400 static void unlist_netdevice(struct net_device *dev, bool lock)
401 {
402 ASSERT_RTNL();
403
404 /* Unlink dev from the device chain */
405 if (lock)
406 write_lock(&dev_base_lock);
407 list_del_rcu(&dev->dev_list);
408 netdev_name_node_del(dev->name_node);
409 hlist_del_rcu(&dev->index_hlist);
410 if (lock)
411 write_unlock(&dev_base_lock);
412
413 dev_base_seq_inc(dev_net(dev));
414 }
415
416 /*
417 * Our notifier list
418 */
419
420 static RAW_NOTIFIER_HEAD(netdev_chain);
421
422 /*
423 * Device drivers call our routines to queue packets here. We empty the
424 * queue in the local softnet handler.
425 */
426
427 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
428 EXPORT_PER_CPU_SYMBOL(softnet_data);
429
430 #ifdef CONFIG_LOCKDEP
431 /*
432 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
433 * according to dev->type
434 */
435 static const unsigned short netdev_lock_type[] = {
436 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
437 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
438 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
439 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
440 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
441 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
442 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
443 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
444 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
445 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
446 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
447 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
448 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
449 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
450 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
451
452 static const char *const netdev_lock_name[] = {
453 "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
454 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
455 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
456 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
457 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
458 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
459 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
460 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
461 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
462 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
463 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
464 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
465 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
466 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
467 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
468
469 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
470 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
471
472 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
473 {
474 int i;
475
476 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
477 if (netdev_lock_type[i] == dev_type)
478 return i;
479 /* the last key is used by default */
480 return ARRAY_SIZE(netdev_lock_type) - 1;
481 }
482
483 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
484 unsigned short dev_type)
485 {
486 int i;
487
488 i = netdev_lock_pos(dev_type);
489 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
490 netdev_lock_name[i]);
491 }
492
493 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
494 {
495 int i;
496
497 i = netdev_lock_pos(dev->type);
498 lockdep_set_class_and_name(&dev->addr_list_lock,
499 &netdev_addr_lock_key[i],
500 netdev_lock_name[i]);
501 }
502 #else
503 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
504 unsigned short dev_type)
505 {
506 }
507
508 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
509 {
510 }
511 #endif
512
513 /*******************************************************************************
514 *
515 * Protocol management and registration routines
516 *
517 *******************************************************************************/
518
519
520 /*
521 * Add a protocol ID to the list. Now that the input handler is
522 * smarter we can dispense with all the messy stuff that used to be
523 * here.
524 *
525 * BEWARE!!! Protocol handlers, mangling input packets,
526 * MUST BE last in hash buckets and checking protocol handlers
527 * MUST start from promiscuous ptype_all chain in net_bh.
528 * It is true now, do not change it.
529 * Explanation follows: if protocol handler, mangling packet, will
530 * be the first on list, it is not able to sense, that packet
531 * is cloned and should be copied-on-write, so that it will
532 * change it and subsequent readers will get broken packet.
533 * --ANK (980803)
534 */
535
536 static inline struct list_head *ptype_head(const struct packet_type *pt)
537 {
538 if (pt->type == htons(ETH_P_ALL))
539 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
540 else
541 return pt->dev ? &pt->dev->ptype_specific :
542 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
543 }
544
545 /**
546 * dev_add_pack - add packet handler
547 * @pt: packet type declaration
548 *
549 * Add a protocol handler to the networking stack. The passed &packet_type
550 * is linked into kernel lists and may not be freed until it has been
551 * removed from the kernel lists.
552 *
553 * This call does not sleep therefore it can not
554 * guarantee all CPU's that are in middle of receiving packets
555 * will see the new packet type (until the next received packet).
556 */
557
558 void dev_add_pack(struct packet_type *pt)
559 {
560 struct list_head *head = ptype_head(pt);
561
562 spin_lock(&ptype_lock);
563 list_add_rcu(&pt->list, head);
564 spin_unlock(&ptype_lock);
565 }
566 EXPORT_SYMBOL(dev_add_pack);
567
568 /**
569 * __dev_remove_pack - remove packet handler
570 * @pt: packet type declaration
571 *
572 * Remove a protocol handler that was previously added to the kernel
573 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
574 * from the kernel lists and can be freed or reused once this function
575 * returns.
576 *
577 * The packet type might still be in use by receivers
578 * and must not be freed until after all the CPU's have gone
579 * through a quiescent state.
580 */
581 void __dev_remove_pack(struct packet_type *pt)
582 {
583 struct list_head *head = ptype_head(pt);
584 struct packet_type *pt1;
585
586 spin_lock(&ptype_lock);
587
588 list_for_each_entry(pt1, head, list) {
589 if (pt == pt1) {
590 list_del_rcu(&pt->list);
591 goto out;
592 }
593 }
594
595 pr_warn("dev_remove_pack: %p not found\n", pt);
596 out:
597 spin_unlock(&ptype_lock);
598 }
599 EXPORT_SYMBOL(__dev_remove_pack);
600
601 /**
602 * dev_remove_pack - remove packet handler
603 * @pt: packet type declaration
604 *
605 * Remove a protocol handler that was previously added to the kernel
606 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
607 * from the kernel lists and can be freed or reused once this function
608 * returns.
609 *
610 * This call sleeps to guarantee that no CPU is looking at the packet
611 * type after return.
612 */
613 void dev_remove_pack(struct packet_type *pt)
614 {
615 __dev_remove_pack(pt);
616
617 synchronize_net();
618 }
619 EXPORT_SYMBOL(dev_remove_pack);
620
621
622 /*******************************************************************************
623 *
624 * Device Interface Subroutines
625 *
626 *******************************************************************************/
627
628 /**
629 * dev_get_iflink - get 'iflink' value of a interface
630 * @dev: targeted interface
631 *
632 * Indicates the ifindex the interface is linked to.
633 * Physical interfaces have the same 'ifindex' and 'iflink' values.
634 */
635
636 int dev_get_iflink(const struct net_device *dev)
637 {
638 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
639 return dev->netdev_ops->ndo_get_iflink(dev);
640
641 return dev->ifindex;
642 }
643 EXPORT_SYMBOL(dev_get_iflink);
644
645 /**
646 * dev_fill_metadata_dst - Retrieve tunnel egress information.
647 * @dev: targeted interface
648 * @skb: The packet.
649 *
650 * For better visibility of tunnel traffic OVS needs to retrieve
651 * egress tunnel information for a packet. Following API allows
652 * user to get this info.
653 */
654 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
655 {
656 struct ip_tunnel_info *info;
657
658 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)
659 return -EINVAL;
660
661 info = skb_tunnel_info_unclone(skb);
662 if (!info)
663 return -ENOMEM;
664 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
665 return -EINVAL;
666
667 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
668 }
669 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
670
671 static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack)
672 {
673 int k = stack->num_paths++;
674
675 if (WARN_ON_ONCE(k >= NET_DEVICE_PATH_STACK_MAX))
676 return NULL;
677
678 return &stack->path[k];
679 }
680
681 int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
682 struct net_device_path_stack *stack)
683 {
684 const struct net_device *last_dev;
685 struct net_device_path_ctx ctx = {
686 .dev = dev,
687 };
688 struct net_device_path *path;
689 int ret = 0;
690
691 memcpy(ctx.daddr, daddr, sizeof(ctx.daddr));
692 stack->num_paths = 0;
693 while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) {
694 last_dev = ctx.dev;
695 path = dev_fwd_path(stack);
696 if (!path)
697 return -1;
698
699 memset(path, 0, sizeof(struct net_device_path));
700 ret = ctx.dev->netdev_ops->ndo_fill_forward_path(&ctx, path);
701 if (ret < 0)
702 return -1;
703
704 if (WARN_ON_ONCE(last_dev == ctx.dev))
705 return -1;
706 }
707
708 if (!ctx.dev)
709 return ret;
710
711 path = dev_fwd_path(stack);
712 if (!path)
713 return -1;
714 path->type = DEV_PATH_ETHERNET;
715 path->dev = ctx.dev;
716
717 return ret;
718 }
719 EXPORT_SYMBOL_GPL(dev_fill_forward_path);
720
721 /**
722 * __dev_get_by_name - find a device by its name
723 * @net: the applicable net namespace
724 * @name: name to find
725 *
726 * Find an interface by name. Must be called under RTNL semaphore
727 * or @dev_base_lock. If the name is found a pointer to the device
728 * is returned. If the name is not found then %NULL is returned. The
729 * reference counters are not incremented so the caller must be
730 * careful with locks.
731 */
732
733 struct net_device *__dev_get_by_name(struct net *net, const char *name)
734 {
735 struct netdev_name_node *node_name;
736
737 node_name = netdev_name_node_lookup(net, name);
738 return node_name ? node_name->dev : NULL;
739 }
740 EXPORT_SYMBOL(__dev_get_by_name);
741
742 /**
743 * dev_get_by_name_rcu - find a device by its name
744 * @net: the applicable net namespace
745 * @name: name to find
746 *
747 * Find an interface by name.
748 * If the name is found a pointer to the device is returned.
749 * If the name is not found then %NULL is returned.
750 * The reference counters are not incremented so the caller must be
751 * careful with locks. The caller must hold RCU lock.
752 */
753
754 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
755 {
756 struct netdev_name_node *node_name;
757
758 node_name = netdev_name_node_lookup_rcu(net, name);
759 return node_name ? node_name->dev : NULL;
760 }
761 EXPORT_SYMBOL(dev_get_by_name_rcu);
762
763 /**
764 * dev_get_by_name - find a device by its name
765 * @net: the applicable net namespace
766 * @name: name to find
767 *
768 * Find an interface by name. This can be called from any
769 * context and does its own locking. The returned handle has
770 * the usage count incremented and the caller must use dev_put() to
771 * release it when it is no longer needed. %NULL is returned if no
772 * matching device is found.
773 */
774
775 struct net_device *dev_get_by_name(struct net *net, const char *name)
776 {
777 struct net_device *dev;
778
779 rcu_read_lock();
780 dev = dev_get_by_name_rcu(net, name);
781 dev_hold(dev);
782 rcu_read_unlock();
783 return dev;
784 }
785 EXPORT_SYMBOL(dev_get_by_name);
786
787 /**
788 * __dev_get_by_index - find a device by its ifindex
789 * @net: the applicable net namespace
790 * @ifindex: index of device
791 *
792 * Search for an interface by index. Returns %NULL if the device
793 * is not found or a pointer to the device. The device has not
794 * had its reference counter increased so the caller must be careful
795 * about locking. The caller must hold either the RTNL semaphore
796 * or @dev_base_lock.
797 */
798
799 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
800 {
801 struct net_device *dev;
802 struct hlist_head *head = dev_index_hash(net, ifindex);
803
804 hlist_for_each_entry(dev, head, index_hlist)
805 if (dev->ifindex == ifindex)
806 return dev;
807
808 return NULL;
809 }
810 EXPORT_SYMBOL(__dev_get_by_index);
811
812 /**
813 * dev_get_by_index_rcu - find a device by its ifindex
814 * @net: the applicable net namespace
815 * @ifindex: index of device
816 *
817 * Search for an interface by index. Returns %NULL if the device
818 * is not found or a pointer to the device. The device has not
819 * had its reference counter increased so the caller must be careful
820 * about locking. The caller must hold RCU lock.
821 */
822
823 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
824 {
825 struct net_device *dev;
826 struct hlist_head *head = dev_index_hash(net, ifindex);
827
828 hlist_for_each_entry_rcu(dev, head, index_hlist)
829 if (dev->ifindex == ifindex)
830 return dev;
831
832 return NULL;
833 }
834 EXPORT_SYMBOL(dev_get_by_index_rcu);
835
836
837 /**
838 * dev_get_by_index - find a device by its ifindex
839 * @net: the applicable net namespace
840 * @ifindex: index of device
841 *
842 * Search for an interface by index. Returns NULL if the device
843 * is not found or a pointer to the device. The device returned has
844 * had a reference added and the pointer is safe until the user calls
845 * dev_put to indicate they have finished with it.
846 */
847
848 struct net_device *dev_get_by_index(struct net *net, int ifindex)
849 {
850 struct net_device *dev;
851
852 rcu_read_lock();
853 dev = dev_get_by_index_rcu(net, ifindex);
854 dev_hold(dev);
855 rcu_read_unlock();
856 return dev;
857 }
858 EXPORT_SYMBOL(dev_get_by_index);
859
860 /**
861 * dev_get_by_napi_id - find a device by napi_id
862 * @napi_id: ID of the NAPI struct
863 *
864 * Search for an interface by NAPI ID. Returns %NULL if the device
865 * is not found or a pointer to the device. The device has not had
866 * its reference counter increased so the caller must be careful
867 * about locking. The caller must hold RCU lock.
868 */
869
870 struct net_device *dev_get_by_napi_id(unsigned int napi_id)
871 {
872 struct napi_struct *napi;
873
874 WARN_ON_ONCE(!rcu_read_lock_held());
875
876 if (napi_id < MIN_NAPI_ID)
877 return NULL;
878
879 napi = napi_by_id(napi_id);
880
881 return napi ? napi->dev : NULL;
882 }
883 EXPORT_SYMBOL(dev_get_by_napi_id);
884
885 /**
886 * netdev_get_name - get a netdevice name, knowing its ifindex.
887 * @net: network namespace
888 * @name: a pointer to the buffer where the name will be stored.
889 * @ifindex: the ifindex of the interface to get the name from.
890 */
891 int netdev_get_name(struct net *net, char *name, int ifindex)
892 {
893 struct net_device *dev;
894 int ret;
895
896 down_read(&devnet_rename_sem);
897 rcu_read_lock();
898
899 dev = dev_get_by_index_rcu(net, ifindex);
900 if (!dev) {
901 ret = -ENODEV;
902 goto out;
903 }
904
905 strcpy(name, dev->name);
906
907 ret = 0;
908 out:
909 rcu_read_unlock();
910 up_read(&devnet_rename_sem);
911 return ret;
912 }
913
914 /**
915 * dev_getbyhwaddr_rcu - find a device by its hardware address
916 * @net: the applicable net namespace
917 * @type: media type of device
918 * @ha: hardware address
919 *
920 * Search for an interface by MAC address. Returns NULL if the device
921 * is not found or a pointer to the device.
922 * The caller must hold RCU or RTNL.
923 * The returned device has not had its ref count increased
924 * and the caller must therefore be careful about locking
925 *
926 */
927
928 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
929 const char *ha)
930 {
931 struct net_device *dev;
932
933 for_each_netdev_rcu(net, dev)
934 if (dev->type == type &&
935 !memcmp(dev->dev_addr, ha, dev->addr_len))
936 return dev;
937
938 return NULL;
939 }
940 EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
941
942 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
943 {
944 struct net_device *dev, *ret = NULL;
945
946 rcu_read_lock();
947 for_each_netdev_rcu(net, dev)
948 if (dev->type == type) {
949 dev_hold(dev);
950 ret = dev;
951 break;
952 }
953 rcu_read_unlock();
954 return ret;
955 }
956 EXPORT_SYMBOL(dev_getfirstbyhwtype);
957
958 /**
959 * __dev_get_by_flags - find any device with given flags
960 * @net: the applicable net namespace
961 * @if_flags: IFF_* values
962 * @mask: bitmask of bits in if_flags to check
963 *
964 * Search for any interface with the given flags. Returns NULL if a device
965 * is not found or a pointer to the device. Must be called inside
966 * rtnl_lock(), and result refcount is unchanged.
967 */
968
969 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
970 unsigned short mask)
971 {
972 struct net_device *dev, *ret;
973
974 ASSERT_RTNL();
975
976 ret = NULL;
977 for_each_netdev(net, dev) {
978 if (((dev->flags ^ if_flags) & mask) == 0) {
979 ret = dev;
980 break;
981 }
982 }
983 return ret;
984 }
985 EXPORT_SYMBOL(__dev_get_by_flags);
986
987 /**
988 * dev_valid_name - check if name is okay for network device
989 * @name: name string
990 *
991 * Network device names need to be valid file names to
992 * allow sysfs to work. We also disallow any kind of
993 * whitespace.
994 */
995 bool dev_valid_name(const char *name)
996 {
997 if (*name == '\0')
998 return false;
999 if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
1000 return false;
1001 if (!strcmp(name, ".") || !strcmp(name, ".."))
1002 return false;
1003
1004 while (*name) {
1005 if (*name == '/' || *name == ':' || isspace(*name))
1006 return false;
1007 name++;
1008 }
1009 return true;
1010 }
1011 EXPORT_SYMBOL(dev_valid_name);
1012
1013 /**
1014 * __dev_alloc_name - allocate a name for a device
1015 * @net: network namespace to allocate the device name in
1016 * @name: name format string
1017 * @buf: scratch buffer and result name string
1018 *
1019 * Passed a format string - eg "lt%d" it will try and find a suitable
1020 * id. It scans list of devices to build up a free map, then chooses
1021 * the first empty slot. The caller must hold the dev_base or rtnl lock
1022 * while allocating the name and adding the device in order to avoid
1023 * duplicates.
1024 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1025 * Returns the number of the unit assigned or a negative errno code.
1026 */
1027
1028 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1029 {
1030 int i = 0;
1031 const char *p;
1032 const int max_netdevices = 8*PAGE_SIZE;
1033 unsigned long *inuse;
1034 struct net_device *d;
1035
1036 if (!dev_valid_name(name))
1037 return -EINVAL;
1038
1039 p = strchr(name, '%');
1040 if (p) {
1041 /*
1042 * Verify the string as this thing may have come from
1043 * the user. There must be either one "%d" and no other "%"
1044 * characters.
1045 */
1046 if (p[1] != 'd' || strchr(p + 2, '%'))
1047 return -EINVAL;
1048
1049 /* Use one page as a bit array of possible slots */
1050 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1051 if (!inuse)
1052 return -ENOMEM;
1053
1054 for_each_netdev(net, d) {
1055 struct netdev_name_node *name_node;
1056 list_for_each_entry(name_node, &d->name_node->list, list) {
1057 if (!sscanf(name_node->name, name, &i))
1058 continue;
1059 if (i < 0 || i >= max_netdevices)
1060 continue;
1061
1062 /* avoid cases where sscanf is not exact inverse of printf */
1063 snprintf(buf, IFNAMSIZ, name, i);
1064 if (!strncmp(buf, name_node->name, IFNAMSIZ))
1065 __set_bit(i, inuse);
1066 }
1067 if (!sscanf(d->name, name, &i))
1068 continue;
1069 if (i < 0 || i >= max_netdevices)
1070 continue;
1071
1072 /* avoid cases where sscanf is not exact inverse of printf */
1073 snprintf(buf, IFNAMSIZ, name, i);
1074 if (!strncmp(buf, d->name, IFNAMSIZ))
1075 __set_bit(i, inuse);
1076 }
1077
1078 i = find_first_zero_bit(inuse, max_netdevices);
1079 free_page((unsigned long) inuse);
1080 }
1081
1082 snprintf(buf, IFNAMSIZ, name, i);
1083 if (!netdev_name_in_use(net, buf))
1084 return i;
1085
1086 /* It is possible to run out of possible slots
1087 * when the name is long and there isn't enough space left
1088 * for the digits, or if all bits are used.
1089 */
1090 return -ENFILE;
1091 }
1092
1093 static int dev_alloc_name_ns(struct net *net,
1094 struct net_device *dev,
1095 const char *name)
1096 {
1097 char buf[IFNAMSIZ];
1098 int ret;
1099
1100 BUG_ON(!net);
1101 ret = __dev_alloc_name(net, name, buf);
1102 if (ret >= 0)
1103 strlcpy(dev->name, buf, IFNAMSIZ);
1104 return ret;
1105 }
1106
1107 /**
1108 * dev_alloc_name - allocate a name for a device
1109 * @dev: device
1110 * @name: name format string
1111 *
1112 * Passed a format string - eg "lt%d" it will try and find a suitable
1113 * id. It scans list of devices to build up a free map, then chooses
1114 * the first empty slot. The caller must hold the dev_base or rtnl lock
1115 * while allocating the name and adding the device in order to avoid
1116 * duplicates.
1117 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1118 * Returns the number of the unit assigned or a negative errno code.
1119 */
1120
1121 int dev_alloc_name(struct net_device *dev, const char *name)
1122 {
1123 return dev_alloc_name_ns(dev_net(dev), dev, name);
1124 }
1125 EXPORT_SYMBOL(dev_alloc_name);
1126
1127 static int dev_get_valid_name(struct net *net, struct net_device *dev,
1128 const char *name)
1129 {
1130 BUG_ON(!net);
1131
1132 if (!dev_valid_name(name))
1133 return -EINVAL;
1134
1135 if (strchr(name, '%'))
1136 return dev_alloc_name_ns(net, dev, name);
1137 else if (netdev_name_in_use(net, name))
1138 return -EEXIST;
1139 else if (dev->name != name)
1140 strlcpy(dev->name, name, IFNAMSIZ);
1141
1142 return 0;
1143 }
1144
1145 /**
1146 * dev_change_name - change name of a device
1147 * @dev: device
1148 * @newname: name (or format string) must be at least IFNAMSIZ
1149 *
1150 * Change name of a device, can pass format strings "eth%d".
1151 * for wildcarding.
1152 */
1153 int dev_change_name(struct net_device *dev, const char *newname)
1154 {
1155 unsigned char old_assign_type;
1156 char oldname[IFNAMSIZ];
1157 int err = 0;
1158 int ret;
1159 struct net *net;
1160
1161 ASSERT_RTNL();
1162 BUG_ON(!dev_net(dev));
1163
1164 net = dev_net(dev);
1165
1166 /* Some auto-enslaved devices e.g. failover slaves are
1167 * special, as userspace might rename the device after
1168 * the interface had been brought up and running since
1169 * the point kernel initiated auto-enslavement. Allow
1170 * live name change even when these slave devices are
1171 * up and running.
1172 *
1173 * Typically, users of these auto-enslaving devices
1174 * don't actually care about slave name change, as
1175 * they are supposed to operate on master interface
1176 * directly.
1177 */
1178 if (dev->flags & IFF_UP &&
1179 likely(!(dev->priv_flags & IFF_LIVE_RENAME_OK)))
1180 return -EBUSY;
1181
1182 down_write(&devnet_rename_sem);
1183
1184 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1185 up_write(&devnet_rename_sem);
1186 return 0;
1187 }
1188
1189 memcpy(oldname, dev->name, IFNAMSIZ);
1190
1191 err = dev_get_valid_name(net, dev, newname);
1192 if (err < 0) {
1193 up_write(&devnet_rename_sem);
1194 return err;
1195 }
1196
1197 if (oldname[0] && !strchr(oldname, '%'))
1198 netdev_info(dev, "renamed from %s\n", oldname);
1199
1200 old_assign_type = dev->name_assign_type;
1201 dev->name_assign_type = NET_NAME_RENAMED;
1202
1203 rollback:
1204 ret = device_rename(&dev->dev, dev->name);
1205 if (ret) {
1206 memcpy(dev->name, oldname, IFNAMSIZ);
1207 dev->name_assign_type = old_assign_type;
1208 up_write(&devnet_rename_sem);
1209 return ret;
1210 }
1211
1212 up_write(&devnet_rename_sem);
1213
1214 netdev_adjacent_rename_links(dev, oldname);
1215
1216 write_lock(&dev_base_lock);
1217 netdev_name_node_del(dev->name_node);
1218 write_unlock(&dev_base_lock);
1219
1220 synchronize_rcu();
1221
1222 write_lock(&dev_base_lock);
1223 netdev_name_node_add(net, dev->name_node);
1224 write_unlock(&dev_base_lock);
1225
1226 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1227 ret = notifier_to_errno(ret);
1228
1229 if (ret) {
1230 /* err >= 0 after dev_alloc_name() or stores the first errno */
1231 if (err >= 0) {
1232 err = ret;
1233 down_write(&devnet_rename_sem);
1234 memcpy(dev->name, oldname, IFNAMSIZ);
1235 memcpy(oldname, newname, IFNAMSIZ);
1236 dev->name_assign_type = old_assign_type;
1237 old_assign_type = NET_NAME_RENAMED;
1238 goto rollback;
1239 } else {
1240 netdev_err(dev, "name change rollback failed: %d\n",
1241 ret);
1242 }
1243 }
1244
1245 return err;
1246 }
1247
1248 /**
1249 * dev_set_alias - change ifalias of a device
1250 * @dev: device
1251 * @alias: name up to IFALIASZ
1252 * @len: limit of bytes to copy from info
1253 *
1254 * Set ifalias for a device,
1255 */
1256 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1257 {
1258 struct dev_ifalias *new_alias = NULL;
1259
1260 if (len >= IFALIASZ)
1261 return -EINVAL;
1262
1263 if (len) {
1264 new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL);
1265 if (!new_alias)
1266 return -ENOMEM;
1267
1268 memcpy(new_alias->ifalias, alias, len);
1269 new_alias->ifalias[len] = 0;
1270 }
1271
1272 mutex_lock(&ifalias_mutex);
1273 new_alias = rcu_replace_pointer(dev->ifalias, new_alias,
1274 mutex_is_locked(&ifalias_mutex));
1275 mutex_unlock(&ifalias_mutex);
1276
1277 if (new_alias)
1278 kfree_rcu(new_alias, rcuhead);
1279
1280 return len;
1281 }
1282 EXPORT_SYMBOL(dev_set_alias);
1283
1284 /**
1285 * dev_get_alias - get ifalias of a device
1286 * @dev: device
1287 * @name: buffer to store name of ifalias
1288 * @len: size of buffer
1289 *
1290 * get ifalias for a device. Caller must make sure dev cannot go
1291 * away, e.g. rcu read lock or own a reference count to device.
1292 */
1293 int dev_get_alias(const struct net_device *dev, char *name, size_t len)
1294 {
1295 const struct dev_ifalias *alias;
1296 int ret = 0;
1297
1298 rcu_read_lock();
1299 alias = rcu_dereference(dev->ifalias);
1300 if (alias)
1301 ret = snprintf(name, len, "%s", alias->ifalias);
1302 rcu_read_unlock();
1303
1304 return ret;
1305 }
1306
1307 /**
1308 * netdev_features_change - device changes features
1309 * @dev: device to cause notification
1310 *
1311 * Called to indicate a device has changed features.
1312 */
1313 void netdev_features_change(struct net_device *dev)
1314 {
1315 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1316 }
1317 EXPORT_SYMBOL(netdev_features_change);
1318
1319 /**
1320 * netdev_state_change - device changes state
1321 * @dev: device to cause notification
1322 *
1323 * Called to indicate a device has changed state. This function calls
1324 * the notifier chains for netdev_chain and sends a NEWLINK message
1325 * to the routing socket.
1326 */
1327 void netdev_state_change(struct net_device *dev)
1328 {
1329 if (dev->flags & IFF_UP) {
1330 struct netdev_notifier_change_info change_info = {
1331 .info.dev = dev,
1332 };
1333
1334 call_netdevice_notifiers_info(NETDEV_CHANGE,
1335 &change_info.info);
1336 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1337 }
1338 }
1339 EXPORT_SYMBOL(netdev_state_change);
1340
1341 /**
1342 * __netdev_notify_peers - notify network peers about existence of @dev,
1343 * to be called when rtnl lock is already held.
1344 * @dev: network device
1345 *
1346 * Generate traffic such that interested network peers are aware of
1347 * @dev, such as by generating a gratuitous ARP. This may be used when
1348 * a device wants to inform the rest of the network about some sort of
1349 * reconfiguration such as a failover event or virtual machine
1350 * migration.
1351 */
1352 void __netdev_notify_peers(struct net_device *dev)
1353 {
1354 ASSERT_RTNL();
1355 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1356 call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1357 }
1358 EXPORT_SYMBOL(__netdev_notify_peers);
1359
1360 /**
1361 * netdev_notify_peers - notify network peers about existence of @dev
1362 * @dev: network device
1363 *
1364 * Generate traffic such that interested network peers are aware of
1365 * @dev, such as by generating a gratuitous ARP. This may be used when
1366 * a device wants to inform the rest of the network about some sort of
1367 * reconfiguration such as a failover event or virtual machine
1368 * migration.
1369 */
1370 void netdev_notify_peers(struct net_device *dev)
1371 {
1372 rtnl_lock();
1373 __netdev_notify_peers(dev);
1374 rtnl_unlock();
1375 }
1376 EXPORT_SYMBOL(netdev_notify_peers);
1377
1378 static int napi_threaded_poll(void *data);
1379
1380 static int napi_kthread_create(struct napi_struct *n)
1381 {
1382 int err = 0;
1383
1384 /* Create and wake up the kthread once to put it in
1385 * TASK_INTERRUPTIBLE mode to avoid the blocked task
1386 * warning and work with loadavg.
1387 */
1388 n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d",
1389 n->dev->name, n->napi_id);
1390 if (IS_ERR(n->thread)) {
1391 err = PTR_ERR(n->thread);
1392 pr_err("kthread_run failed with err %d\n", err);
1393 n->thread = NULL;
1394 }
1395
1396 return err;
1397 }
1398
1399 static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1400 {
1401 const struct net_device_ops *ops = dev->netdev_ops;
1402 int ret;
1403
1404 ASSERT_RTNL();
1405 dev_addr_check(dev);
1406
1407 if (!netif_device_present(dev)) {
1408 /* may be detached because parent is runtime-suspended */
1409 if (dev->dev.parent)
1410 pm_runtime_resume(dev->dev.parent);
1411 if (!netif_device_present(dev))
1412 return -ENODEV;
1413 }
1414
1415 /* Block netpoll from trying to do any rx path servicing.
1416 * If we don't do this there is a chance ndo_poll_controller
1417 * or ndo_poll may be running while we open the device
1418 */
1419 netpoll_poll_disable(dev);
1420
1421 ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);
1422 ret = notifier_to_errno(ret);
1423 if (ret)
1424 return ret;
1425
1426 set_bit(__LINK_STATE_START, &dev->state);
1427
1428 if (ops->ndo_validate_addr)
1429 ret = ops->ndo_validate_addr(dev);
1430
1431 if (!ret && ops->ndo_open)
1432 ret = ops->ndo_open(dev);
1433
1434 netpoll_poll_enable(dev);
1435
1436 if (ret)
1437 clear_bit(__LINK_STATE_START, &dev->state);
1438 else {
1439 dev->flags |= IFF_UP;
1440 dev_set_rx_mode(dev);
1441 dev_activate(dev);
1442 add_device_randomness(dev->dev_addr, dev->addr_len);
1443 }
1444
1445 return ret;
1446 }
1447
1448 /**
1449 * dev_open - prepare an interface for use.
1450 * @dev: device to open
1451 * @extack: netlink extended ack
1452 *
1453 * Takes a device from down to up state. The device's private open
1454 * function is invoked and then the multicast lists are loaded. Finally
1455 * the device is moved into the up state and a %NETDEV_UP message is
1456 * sent to the netdev notifier chain.
1457 *
1458 * Calling this function on an active interface is a nop. On a failure
1459 * a negative errno code is returned.
1460 */
1461 int dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1462 {
1463 int ret;
1464
1465 if (dev->flags & IFF_UP)
1466 return 0;
1467
1468 ret = __dev_open(dev, extack);
1469 if (ret < 0)
1470 return ret;
1471
1472 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1473 call_netdevice_notifiers(NETDEV_UP, dev);
1474
1475 return ret;
1476 }
1477 EXPORT_SYMBOL(dev_open);
1478
1479 static void __dev_close_many(struct list_head *head)
1480 {
1481 struct net_device *dev;
1482
1483 ASSERT_RTNL();
1484 might_sleep();
1485
1486 list_for_each_entry(dev, head, close_list) {
1487 /* Temporarily disable netpoll until the interface is down */
1488 netpoll_poll_disable(dev);
1489
1490 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1491
1492 clear_bit(__LINK_STATE_START, &dev->state);
1493
1494 /* Synchronize to scheduled poll. We cannot touch poll list, it
1495 * can be even on different cpu. So just clear netif_running().
1496 *
1497 * dev->stop() will invoke napi_disable() on all of it's
1498 * napi_struct instances on this device.
1499 */
1500 smp_mb__after_atomic(); /* Commit netif_running(). */
1501 }
1502
1503 dev_deactivate_many(head);
1504
1505 list_for_each_entry(dev, head, close_list) {
1506 const struct net_device_ops *ops = dev->netdev_ops;
1507
1508 /*
1509 * Call the device specific close. This cannot fail.
1510 * Only if device is UP
1511 *
1512 * We allow it to be called even after a DETACH hot-plug
1513 * event.
1514 */
1515 if (ops->ndo_stop)
1516 ops->ndo_stop(dev);
1517
1518 dev->flags &= ~IFF_UP;
1519 netpoll_poll_enable(dev);
1520 }
1521 }
1522
1523 static void __dev_close(struct net_device *dev)
1524 {
1525 LIST_HEAD(single);
1526
1527 list_add(&dev->close_list, &single);
1528 __dev_close_many(&single);
1529 list_del(&single);
1530 }
1531
1532 void dev_close_many(struct list_head *head, bool unlink)
1533 {
1534 struct net_device *dev, *tmp;
1535
1536 /* Remove the devices that don't need to be closed */
1537 list_for_each_entry_safe(dev, tmp, head, close_list)
1538 if (!(dev->flags & IFF_UP))
1539 list_del_init(&dev->close_list);
1540
1541 __dev_close_many(head);
1542
1543 list_for_each_entry_safe(dev, tmp, head, close_list) {
1544 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1545 call_netdevice_notifiers(NETDEV_DOWN, dev);
1546 if (unlink)
1547 list_del_init(&dev->close_list);
1548 }
1549 }
1550 EXPORT_SYMBOL(dev_close_many);
1551
1552 /**
1553 * dev_close - shutdown an interface.
1554 * @dev: device to shutdown
1555 *
1556 * This function moves an active device into down state. A
1557 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1558 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1559 * chain.
1560 */
1561 void dev_close(struct net_device *dev)
1562 {
1563 if (dev->flags & IFF_UP) {
1564 LIST_HEAD(single);
1565
1566 list_add(&dev->close_list, &single);
1567 dev_close_many(&single, true);
1568 list_del(&single);
1569 }
1570 }
1571 EXPORT_SYMBOL(dev_close);
1572
1573
1574 /**
1575 * dev_disable_lro - disable Large Receive Offload on a device
1576 * @dev: device
1577 *
1578 * Disable Large Receive Offload (LRO) on a net device. Must be
1579 * called under RTNL. This is needed if received packets may be
1580 * forwarded to another interface.
1581 */
1582 void dev_disable_lro(struct net_device *dev)
1583 {
1584 struct net_device *lower_dev;
1585 struct list_head *iter;
1586
1587 dev->wanted_features &= ~NETIF_F_LRO;
1588 netdev_update_features(dev);
1589
1590 if (unlikely(dev->features & NETIF_F_LRO))
1591 netdev_WARN(dev, "failed to disable LRO!\n");
1592
1593 netdev_for_each_lower_dev(dev, lower_dev, iter)
1594 dev_disable_lro(lower_dev);
1595 }
1596 EXPORT_SYMBOL(dev_disable_lro);
1597
1598 /**
1599 * dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1600 * @dev: device
1601 *
1602 * Disable HW Generic Receive Offload (GRO_HW) on a net device. Must be
1603 * called under RTNL. This is needed if Generic XDP is installed on
1604 * the device.
1605 */
1606 static void dev_disable_gro_hw(struct net_device *dev)
1607 {
1608 dev->wanted_features &= ~NETIF_F_GRO_HW;
1609 netdev_update_features(dev);
1610
1611 if (unlikely(dev->features & NETIF_F_GRO_HW))
1612 netdev_WARN(dev, "failed to disable GRO_HW!\n");
1613 }
1614
1615 const char *netdev_cmd_to_name(enum netdev_cmd cmd)
1616 {
1617 #define N(val) \
1618 case NETDEV_##val: \
1619 return "NETDEV_" __stringify(val);
1620 switch (cmd) {
1621 N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
1622 N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
1623 N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
1624 N(POST_INIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN) N(CHANGEUPPER)
1625 N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA) N(BONDING_INFO)
1626 N(PRECHANGEUPPER) N(CHANGELOWERSTATE) N(UDP_TUNNEL_PUSH_INFO)
1627 N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
1628 N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
1629 N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
1630 N(PRE_CHANGEADDR) N(OFFLOAD_XSTATS_ENABLE) N(OFFLOAD_XSTATS_DISABLE)
1631 N(OFFLOAD_XSTATS_REPORT_USED) N(OFFLOAD_XSTATS_REPORT_DELTA)
1632 }
1633 #undef N
1634 return "UNKNOWN_NETDEV_EVENT";
1635 }
1636 EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
1637
1638 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1639 struct net_device *dev)
1640 {
1641 struct netdev_notifier_info info = {
1642 .dev = dev,
1643 };
1644
1645 return nb->notifier_call(nb, val, &info);
1646 }
1647
1648 static int call_netdevice_register_notifiers(struct notifier_block *nb,
1649 struct net_device *dev)
1650 {
1651 int err;
1652
1653 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1654 err = notifier_to_errno(err);
1655 if (err)
1656 return err;
1657
1658 if (!(dev->flags & IFF_UP))
1659 return 0;
1660
1661 call_netdevice_notifier(nb, NETDEV_UP, dev);
1662 return 0;
1663 }
1664
1665 static void call_netdevice_unregister_notifiers(struct notifier_block *nb,
1666 struct net_device *dev)
1667 {
1668 if (dev->flags & IFF_UP) {
1669 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1670 dev);
1671 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1672 }
1673 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1674 }
1675
1676 static int call_netdevice_register_net_notifiers(struct notifier_block *nb,
1677 struct net *net)
1678 {
1679 struct net_device *dev;
1680 int err;
1681
1682 for_each_netdev(net, dev) {
1683 err = call_netdevice_register_notifiers(nb, dev);
1684 if (err)
1685 goto rollback;
1686 }
1687 return 0;
1688
1689 rollback:
1690 for_each_netdev_continue_reverse(net, dev)
1691 call_netdevice_unregister_notifiers(nb, dev);
1692 return err;
1693 }
1694
1695 static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb,
1696 struct net *net)
1697 {
1698 struct net_device *dev;
1699
1700 for_each_netdev(net, dev)
1701 call_netdevice_unregister_notifiers(nb, dev);
1702 }
1703
1704 static int dev_boot_phase = 1;
1705
1706 /**
1707 * register_netdevice_notifier - register a network notifier block
1708 * @nb: notifier
1709 *
1710 * Register a notifier to be called when network device events occur.
1711 * The notifier passed is linked into the kernel structures and must
1712 * not be reused until it has been unregistered. A negative errno code
1713 * is returned on a failure.
1714 *
1715 * When registered all registration and up events are replayed
1716 * to the new notifier to allow device to have a race free
1717 * view of the network device list.
1718 */
1719
1720 int register_netdevice_notifier(struct notifier_block *nb)
1721 {
1722 struct net *net;
1723 int err;
1724
1725 /* Close race with setup_net() and cleanup_net() */
1726 down_write(&pernet_ops_rwsem);
1727 rtnl_lock();
1728 err = raw_notifier_chain_register(&netdev_chain, nb);
1729 if (err)
1730 goto unlock;
1731 if (dev_boot_phase)
1732 goto unlock;
1733 for_each_net(net) {
1734 err = call_netdevice_register_net_notifiers(nb, net);
1735 if (err)
1736 goto rollback;
1737 }
1738
1739 unlock:
1740 rtnl_unlock();
1741 up_write(&pernet_ops_rwsem);
1742 return err;
1743
1744 rollback:
1745 for_each_net_continue_reverse(net)
1746 call_netdevice_unregister_net_notifiers(nb, net);
1747
1748 raw_notifier_chain_unregister(&netdev_chain, nb);
1749 goto unlock;
1750 }
1751 EXPORT_SYMBOL(register_netdevice_notifier);
1752
1753 /**
1754 * unregister_netdevice_notifier - unregister a network notifier block
1755 * @nb: notifier
1756 *
1757 * Unregister a notifier previously registered by
1758 * register_netdevice_notifier(). The notifier is unlinked into the
1759 * kernel structures and may then be reused. A negative errno code
1760 * is returned on a failure.
1761 *
1762 * After unregistering unregister and down device events are synthesized
1763 * for all devices on the device list to the removed notifier to remove
1764 * the need for special case cleanup code.
1765 */
1766
1767 int unregister_netdevice_notifier(struct notifier_block *nb)
1768 {
1769 struct net *net;
1770 int err;
1771
1772 /* Close race with setup_net() and cleanup_net() */
1773 down_write(&pernet_ops_rwsem);
1774 rtnl_lock();
1775 err = raw_notifier_chain_unregister(&netdev_chain, nb);
1776 if (err)
1777 goto unlock;
1778
1779 for_each_net(net)
1780 call_netdevice_unregister_net_notifiers(nb, net);
1781
1782 unlock:
1783 rtnl_unlock();
1784 up_write(&pernet_ops_rwsem);
1785 return err;
1786 }
1787 EXPORT_SYMBOL(unregister_netdevice_notifier);
1788
1789 static int __register_netdevice_notifier_net(struct net *net,
1790 struct notifier_block *nb,
1791 bool ignore_call_fail)
1792 {
1793 int err;
1794
1795 err = raw_notifier_chain_register(&net->netdev_chain, nb);
1796 if (err)
1797 return err;
1798 if (dev_boot_phase)
1799 return 0;
1800
1801 err = call_netdevice_register_net_notifiers(nb, net);
1802 if (err && !ignore_call_fail)
1803 goto chain_unregister;
1804
1805 return 0;
1806
1807 chain_unregister:
1808 raw_notifier_chain_unregister(&net->netdev_chain, nb);
1809 return err;
1810 }
1811
1812 static int __unregister_netdevice_notifier_net(struct net *net,
1813 struct notifier_block *nb)
1814 {
1815 int err;
1816
1817 err = raw_notifier_chain_unregister(&net->netdev_chain, nb);
1818 if (err)
1819 return err;
1820
1821 call_netdevice_unregister_net_notifiers(nb, net);
1822 return 0;
1823 }
1824
1825 /**
1826 * register_netdevice_notifier_net - register a per-netns network notifier block
1827 * @net: network namespace
1828 * @nb: notifier
1829 *
1830 * Register a notifier to be called when network device events occur.
1831 * The notifier passed is linked into the kernel structures and must
1832 * not be reused until it has been unregistered. A negative errno code
1833 * is returned on a failure.
1834 *
1835 * When registered all registration and up events are replayed
1836 * to the new notifier to allow device to have a race free
1837 * view of the network device list.
1838 */
1839
1840 int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb)
1841 {
1842 int err;
1843
1844 rtnl_lock();
1845 err = __register_netdevice_notifier_net(net, nb, false);
1846 rtnl_unlock();
1847 return err;
1848 }
1849 EXPORT_SYMBOL(register_netdevice_notifier_net);
1850
1851 /**
1852 * unregister_netdevice_notifier_net - unregister a per-netns
1853 * network notifier block
1854 * @net: network namespace
1855 * @nb: notifier
1856 *
1857 * Unregister a notifier previously registered by
1858 * register_netdevice_notifier(). The notifier is unlinked into the
1859 * kernel structures and may then be reused. A negative errno code
1860 * is returned on a failure.
1861 *
1862 * After unregistering unregister and down device events are synthesized
1863 * for all devices on the device list to the removed notifier to remove
1864 * the need for special case cleanup code.
1865 */
1866
1867 int unregister_netdevice_notifier_net(struct net *net,
1868 struct notifier_block *nb)
1869 {
1870 int err;
1871
1872 rtnl_lock();
1873 err = __unregister_netdevice_notifier_net(net, nb);
1874 rtnl_unlock();
1875 return err;
1876 }
1877 EXPORT_SYMBOL(unregister_netdevice_notifier_net);
1878
1879 int register_netdevice_notifier_dev_net(struct net_device *dev,
1880 struct notifier_block *nb,
1881 struct netdev_net_notifier *nn)
1882 {
1883 int err;
1884
1885 rtnl_lock();
1886 err = __register_netdevice_notifier_net(dev_net(dev), nb, false);
1887 if (!err) {
1888 nn->nb = nb;
1889 list_add(&nn->list, &dev->net_notifier_list);
1890 }
1891 rtnl_unlock();
1892 return err;
1893 }
1894 EXPORT_SYMBOL(register_netdevice_notifier_dev_net);
1895
1896 int unregister_netdevice_notifier_dev_net(struct net_device *dev,
1897 struct notifier_block *nb,
1898 struct netdev_net_notifier *nn)
1899 {
1900 int err;
1901
1902 rtnl_lock();
1903 list_del(&nn->list);
1904 err = __unregister_netdevice_notifier_net(dev_net(dev), nb);
1905 rtnl_unlock();
1906 return err;
1907 }
1908 EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net);
1909
1910 static void move_netdevice_notifiers_dev_net(struct net_device *dev,
1911 struct net *net)
1912 {
1913 struct netdev_net_notifier *nn;
1914
1915 list_for_each_entry(nn, &dev->net_notifier_list, list) {
1916 __unregister_netdevice_notifier_net(dev_net(dev), nn->nb);
1917 __register_netdevice_notifier_net(net, nn->nb, true);
1918 }
1919 }
1920
1921 /**
1922 * call_netdevice_notifiers_info - call all network notifier blocks
1923 * @val: value passed unmodified to notifier function
1924 * @info: notifier information data
1925 *
1926 * Call all network notifier blocks. Parameters and return value
1927 * are as for raw_notifier_call_chain().
1928 */
1929
1930 static int call_netdevice_notifiers_info(unsigned long val,
1931 struct netdev_notifier_info *info)
1932 {
1933 struct net *net = dev_net(info->dev);
1934 int ret;
1935
1936 ASSERT_RTNL();
1937
1938 /* Run per-netns notifier block chain first, then run the global one.
1939 * Hopefully, one day, the global one is going to be removed after
1940 * all notifier block registrators get converted to be per-netns.
1941 */
1942 ret = raw_notifier_call_chain(&net->netdev_chain, val, info);
1943 if (ret & NOTIFY_STOP_MASK)
1944 return ret;
1945 return raw_notifier_call_chain(&netdev_chain, val, info);
1946 }
1947
1948 /**
1949 * call_netdevice_notifiers_info_robust - call per-netns notifier blocks
1950 * for and rollback on error
1951 * @val_up: value passed unmodified to notifier function
1952 * @val_down: value passed unmodified to the notifier function when
1953 * recovering from an error on @val_up
1954 * @info: notifier information data
1955 *
1956 * Call all per-netns network notifier blocks, but not notifier blocks on
1957 * the global notifier chain. Parameters and return value are as for
1958 * raw_notifier_call_chain_robust().
1959 */
1960
1961 static int
1962 call_netdevice_notifiers_info_robust(unsigned long val_up,
1963 unsigned long val_down,
1964 struct netdev_notifier_info *info)
1965 {
1966 struct net *net = dev_net(info->dev);
1967
1968 ASSERT_RTNL();
1969
1970 return raw_notifier_call_chain_robust(&net->netdev_chain,
1971 val_up, val_down, info);
1972 }
1973
1974 static int call_netdevice_notifiers_extack(unsigned long val,
1975 struct net_device *dev,
1976 struct netlink_ext_ack *extack)
1977 {
1978 struct netdev_notifier_info info = {
1979 .dev = dev,
1980 .extack = extack,
1981 };
1982
1983 return call_netdevice_notifiers_info(val, &info);
1984 }
1985
1986 /**
1987 * call_netdevice_notifiers - call all network notifier blocks
1988 * @val: value passed unmodified to notifier function
1989 * @dev: net_device pointer passed unmodified to notifier function
1990 *
1991 * Call all network notifier blocks. Parameters and return value
1992 * are as for raw_notifier_call_chain().
1993 */
1994
1995 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1996 {
1997 return call_netdevice_notifiers_extack(val, dev, NULL);
1998 }
1999 EXPORT_SYMBOL(call_netdevice_notifiers);
2000
2001 /**
2002 * call_netdevice_notifiers_mtu - call all network notifier blocks
2003 * @val: value passed unmodified to notifier function
2004 * @dev: net_device pointer passed unmodified to notifier function
2005 * @arg: additional u32 argument passed to the notifier function
2006 *
2007 * Call all network notifier blocks. Parameters and return value
2008 * are as for raw_notifier_call_chain().
2009 */
2010 static int call_netdevice_notifiers_mtu(unsigned long val,
2011 struct net_device *dev, u32 arg)
2012 {
2013 struct netdev_notifier_info_ext info = {
2014 .info.dev = dev,
2015 .ext.mtu = arg,
2016 };
2017
2018 BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
2019
2020 return call_netdevice_notifiers_info(val, &info.info);
2021 }
2022
2023 #ifdef CONFIG_NET_INGRESS
2024 static DEFINE_STATIC_KEY_FALSE(ingress_needed_key);
2025
2026 void net_inc_ingress_queue(void)
2027 {
2028 static_branch_inc(&ingress_needed_key);
2029 }
2030 EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
2031
2032 void net_dec_ingress_queue(void)
2033 {
2034 static_branch_dec(&ingress_needed_key);
2035 }
2036 EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
2037 #endif
2038
2039 #ifdef CONFIG_NET_EGRESS
2040 static DEFINE_STATIC_KEY_FALSE(egress_needed_key);
2041
2042 void net_inc_egress_queue(void)
2043 {
2044 static_branch_inc(&egress_needed_key);
2045 }
2046 EXPORT_SYMBOL_GPL(net_inc_egress_queue);
2047
2048 void net_dec_egress_queue(void)
2049 {
2050 static_branch_dec(&egress_needed_key);
2051 }
2052 EXPORT_SYMBOL_GPL(net_dec_egress_queue);
2053 #endif
2054
2055 DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
2056 EXPORT_SYMBOL(netstamp_needed_key);
2057 #ifdef CONFIG_JUMP_LABEL
2058 static atomic_t netstamp_needed_deferred;
2059 static atomic_t netstamp_wanted;
2060 static void netstamp_clear(struct work_struct *work)
2061 {
2062 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
2063 int wanted;
2064
2065 wanted = atomic_add_return(deferred, &netstamp_wanted);
2066 if (wanted > 0)
2067 static_branch_enable(&netstamp_needed_key);
2068 else
2069 static_branch_disable(&netstamp_needed_key);
2070 }
2071 static DECLARE_WORK(netstamp_work, netstamp_clear);
2072 #endif
2073
2074 void net_enable_timestamp(void)
2075 {
2076 #ifdef CONFIG_JUMP_LABEL
2077 int wanted;
2078
2079 while (1) {
2080 wanted = atomic_read(&netstamp_wanted);
2081 if (wanted <= 0)
2082 break;
2083 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
2084 return;
2085 }
2086 atomic_inc(&netstamp_needed_deferred);
2087 schedule_work(&netstamp_work);
2088 #else
2089 static_branch_inc(&netstamp_needed_key);
2090 #endif
2091 }
2092 EXPORT_SYMBOL(net_enable_timestamp);
2093
2094 void net_disable_timestamp(void)
2095 {
2096 #ifdef CONFIG_JUMP_LABEL
2097 int wanted;
2098
2099 while (1) {
2100 wanted = atomic_read(&netstamp_wanted);
2101 if (wanted <= 1)
2102 break;
2103 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
2104 return;
2105 }
2106 atomic_dec(&netstamp_needed_deferred);
2107 schedule_work(&netstamp_work);
2108 #else
2109 static_branch_dec(&netstamp_needed_key);
2110 #endif
2111 }
2112 EXPORT_SYMBOL(net_disable_timestamp);
2113
2114 static inline void net_timestamp_set(struct sk_buff *skb)
2115 {
2116 skb->tstamp = 0;
2117 skb->mono_delivery_time = 0;
2118 if (static_branch_unlikely(&netstamp_needed_key))
2119 skb->tstamp = ktime_get_real();
2120 }
2121
2122 #define net_timestamp_check(COND, SKB) \
2123 if (static_branch_unlikely(&netstamp_needed_key)) { \
2124 if ((COND) && !(SKB)->tstamp) \
2125 (SKB)->tstamp = ktime_get_real(); \
2126 } \
2127
2128 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
2129 {
2130 return __is_skb_forwardable(dev, skb, true);
2131 }
2132 EXPORT_SYMBOL_GPL(is_skb_forwardable);
2133
2134 static int __dev_forward_skb2(struct net_device *dev, struct sk_buff *skb,
2135 bool check_mtu)
2136 {
2137 int ret = ____dev_forward_skb(dev, skb, check_mtu);
2138
2139 if (likely(!ret)) {
2140 skb->protocol = eth_type_trans(skb, dev);
2141 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
2142 }
2143
2144 return ret;
2145 }
2146
2147 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2148 {
2149 return __dev_forward_skb2(dev, skb, true);
2150 }
2151 EXPORT_SYMBOL_GPL(__dev_forward_skb);
2152
2153 /**
2154 * dev_forward_skb - loopback an skb to another netif
2155 *
2156 * @dev: destination network device
2157 * @skb: buffer to forward
2158 *
2159 * return values:
2160 * NET_RX_SUCCESS (no congestion)
2161 * NET_RX_DROP (packet was dropped, but freed)
2162 *
2163 * dev_forward_skb can be used for injecting an skb from the
2164 * start_xmit function of one device into the receive queue
2165 * of another device.
2166 *
2167 * The receiving device may be in another namespace, so
2168 * we have to clear all information in the skb that could
2169 * impact namespace isolation.
2170 */
2171 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2172 {
2173 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
2174 }
2175 EXPORT_SYMBOL_GPL(dev_forward_skb);
2176
2177 int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb)
2178 {
2179 return __dev_forward_skb2(dev, skb, false) ?: netif_rx_internal(skb);
2180 }
2181
2182 static inline int deliver_skb(struct sk_buff *skb,
2183 struct packet_type *pt_prev,
2184 struct net_device *orig_dev)
2185 {
2186 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
2187 return -ENOMEM;
2188 refcount_inc(&skb->users);
2189 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
2190 }
2191
2192 static inline void deliver_ptype_list_skb(struct sk_buff *skb,
2193 struct packet_type **pt,
2194 struct net_device *orig_dev,
2195 __be16 type,
2196 struct list_head *ptype_list)
2197 {
2198 struct packet_type *ptype, *pt_prev = *pt;
2199
2200 list_for_each_entry_rcu(ptype, ptype_list, list) {
2201 if (ptype->type != type)
2202 continue;
2203 if (pt_prev)
2204 deliver_skb(skb, pt_prev, orig_dev);
2205 pt_prev = ptype;
2206 }
2207 *pt = pt_prev;
2208 }
2209
2210 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
2211 {
2212 if (!ptype->af_packet_priv || !skb->sk)
2213 return false;
2214
2215 if (ptype->id_match)
2216 return ptype->id_match(ptype, skb->sk);
2217 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
2218 return true;
2219
2220 return false;
2221 }
2222
2223 /**
2224 * dev_nit_active - return true if any network interface taps are in use
2225 *
2226 * @dev: network device to check for the presence of taps
2227 */
2228 bool dev_nit_active(struct net_device *dev)
2229 {
2230 return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all);
2231 }
2232 EXPORT_SYMBOL_GPL(dev_nit_active);
2233
2234 /*
2235 * Support routine. Sends outgoing frames to any network
2236 * taps currently in use.
2237 */
2238
2239 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
2240 {
2241 struct packet_type *ptype;
2242 struct sk_buff *skb2 = NULL;
2243 struct packet_type *pt_prev = NULL;
2244 struct list_head *ptype_list = &ptype_all;
2245
2246 rcu_read_lock();
2247 again:
2248 list_for_each_entry_rcu(ptype, ptype_list, list) {
2249 if (ptype->ignore_outgoing)
2250 continue;
2251
2252 /* Never send packets back to the socket
2253 * they originated from - MvS (miquels@drinkel.ow.org)
2254 */
2255 if (skb_loop_sk(ptype, skb))
2256 continue;
2257
2258 if (pt_prev) {
2259 deliver_skb(skb2, pt_prev, skb->dev);
2260 pt_prev = ptype;
2261 continue;
2262 }
2263
2264 /* need to clone skb, done only once */
2265 skb2 = skb_clone(skb, GFP_ATOMIC);
2266 if (!skb2)
2267 goto out_unlock;
2268
2269 net_timestamp_set(skb2);
2270
2271 /* skb->nh should be correctly
2272 * set by sender, so that the second statement is
2273 * just protection against buggy protocols.
2274 */
2275 skb_reset_mac_header(skb2);
2276
2277 if (skb_network_header(skb2) < skb2->data ||
2278 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
2279 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
2280 ntohs(skb2->protocol),
2281 dev->name);
2282 skb_reset_network_header(skb2);
2283 }
2284
2285 skb2->transport_header = skb2->network_header;
2286 skb2->pkt_type = PACKET_OUTGOING;
2287 pt_prev = ptype;
2288 }
2289
2290 if (ptype_list == &ptype_all) {
2291 ptype_list = &dev->ptype_all;
2292 goto again;
2293 }
2294 out_unlock:
2295 if (pt_prev) {
2296 if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
2297 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
2298 else
2299 kfree_skb(skb2);
2300 }
2301 rcu_read_unlock();
2302 }
2303 EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
2304
2305 /**
2306 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
2307 * @dev: Network device
2308 * @txq: number of queues available
2309 *
2310 * If real_num_tx_queues is changed the tc mappings may no longer be
2311 * valid. To resolve this verify the tc mapping remains valid and if
2312 * not NULL the mapping. With no priorities mapping to this
2313 * offset/count pair it will no longer be used. In the worst case TC0
2314 * is invalid nothing can be done so disable priority mappings. If is
2315 * expected that drivers will fix this mapping if they can before
2316 * calling netif_set_real_num_tx_queues.
2317 */
2318 static void netif_setup_tc(struct net_device *dev, unsigned int txq)
2319 {
2320 int i;
2321 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2322
2323 /* If TC0 is invalidated disable TC mapping */
2324 if (tc->offset + tc->count > txq) {
2325 netdev_warn(dev, "Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
2326 dev->num_tc = 0;
2327 return;
2328 }
2329
2330 /* Invalidated prio to tc mappings set to TC0 */
2331 for (i = 1; i < TC_BITMASK + 1; i++) {
2332 int q = netdev_get_prio_tc_map(dev, i);
2333
2334 tc = &dev->tc_to_txq[q];
2335 if (tc->offset + tc->count > txq) {
2336 netdev_warn(dev, "Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2337 i, q);
2338 netdev_set_prio_tc_map(dev, i, 0);
2339 }
2340 }
2341 }
2342
2343 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2344 {
2345 if (dev->num_tc) {
2346 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2347 int i;
2348
2349 /* walk through the TCs and see if it falls into any of them */
2350 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2351 if ((txq - tc->offset) < tc->count)
2352 return i;
2353 }
2354
2355 /* didn't find it, just return -1 to indicate no match */
2356 return -1;
2357 }
2358
2359 return 0;
2360 }
2361 EXPORT_SYMBOL(netdev_txq_to_tc);
2362
2363 #ifdef CONFIG_XPS
2364 static struct static_key xps_needed __read_mostly;
2365 static struct static_key xps_rxqs_needed __read_mostly;
2366 static DEFINE_MUTEX(xps_map_mutex);
2367 #define xmap_dereference(P) \
2368 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2369
2370 static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2371 struct xps_dev_maps *old_maps, int tci, u16 index)
2372 {
2373 struct xps_map *map = NULL;
2374 int pos;
2375
2376 if (dev_maps)
2377 map = xmap_dereference(dev_maps->attr_map[tci]);
2378 if (!map)
2379 return false;
2380
2381 for (pos = map->len; pos--;) {
2382 if (map->queues[pos] != index)
2383 continue;
2384
2385 if (map->len > 1) {
2386 map->queues[pos] = map->queues[--map->len];
2387 break;
2388 }
2389
2390 if (old_maps)
2391 RCU_INIT_POINTER(old_maps->attr_map[tci], NULL);
2392 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2393 kfree_rcu(map, rcu);
2394 return false;
2395 }
2396
2397 return true;
2398 }
2399
2400 static bool remove_xps_queue_cpu(struct net_device *dev,
2401 struct xps_dev_maps *dev_maps,
2402 int cpu, u16 offset, u16 count)
2403 {
2404 int num_tc = dev_maps->num_tc;
2405 bool active = false;
2406 int tci;
2407
2408 for (tci = cpu * num_tc; num_tc--; tci++) {
2409 int i, j;
2410
2411 for (i = count, j = offset; i--; j++) {
2412 if (!remove_xps_queue(dev_maps, NULL, tci, j))
2413 break;
2414 }
2415
2416 active |= i < 0;
2417 }
2418
2419 return active;
2420 }
2421
2422 static void reset_xps_maps(struct net_device *dev,
2423 struct xps_dev_maps *dev_maps,
2424 enum xps_map_type type)
2425 {
2426 static_key_slow_dec_cpuslocked(&xps_needed);
2427 if (type == XPS_RXQS)
2428 static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
2429
2430 RCU_INIT_POINTER(dev->xps_maps[type], NULL);
2431
2432 kfree_rcu(dev_maps, rcu);
2433 }
2434
2435 static void clean_xps_maps(struct net_device *dev, enum xps_map_type type,
2436 u16 offset, u16 count)
2437 {
2438 struct xps_dev_maps *dev_maps;
2439 bool active = false;
2440 int i, j;
2441
2442 dev_maps = xmap_dereference(dev->xps_maps[type]);
2443 if (!dev_maps)
2444 return;
2445
2446 for (j = 0; j < dev_maps->nr_ids; j++)
2447 active |= remove_xps_queue_cpu(dev, dev_maps, j, offset, count);
2448 if (!active)
2449 reset_xps_maps(dev, dev_maps, type);
2450
2451 if (type == XPS_CPUS) {
2452 for (i = offset + (count - 1); count--; i--)
2453 netdev_queue_numa_node_write(
2454 netdev_get_tx_queue(dev, i), NUMA_NO_NODE);
2455 }
2456 }
2457
2458 static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2459 u16 count)
2460 {
2461 if (!static_key_false(&xps_needed))
2462 return;
2463
2464 cpus_read_lock();
2465 mutex_lock(&xps_map_mutex);
2466
2467 if (static_key_false(&xps_rxqs_needed))
2468 clean_xps_maps(dev, XPS_RXQS, offset, count);
2469
2470 clean_xps_maps(dev, XPS_CPUS, offset, count);
2471
2472 mutex_unlock(&xps_map_mutex);
2473 cpus_read_unlock();
2474 }
2475
2476 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2477 {
2478 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2479 }
2480
2481 static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index,
2482 u16 index, bool is_rxqs_map)
2483 {
2484 struct xps_map *new_map;
2485 int alloc_len = XPS_MIN_MAP_ALLOC;
2486 int i, pos;
2487
2488 for (pos = 0; map && pos < map->len; pos++) {
2489 if (map->queues[pos] != index)
2490 continue;
2491 return map;
2492 }
2493
2494 /* Need to add tx-queue to this CPU's/rx-queue's existing map */
2495 if (map) {
2496 if (pos < map->alloc_len)
2497 return map;
2498
2499 alloc_len = map->alloc_len * 2;
2500 }
2501
2502 /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's
2503 * map
2504 */
2505 if (is_rxqs_map)
2506 new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL);
2507 else
2508 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2509 cpu_to_node(attr_index));
2510 if (!new_map)
2511 return NULL;
2512
2513 for (i = 0; i < pos; i++)
2514 new_map->queues[i] = map->queues[i];
2515 new_map->alloc_len = alloc_len;
2516 new_map->len = pos;
2517
2518 return new_map;
2519 }
2520
2521 /* Copy xps maps at a given index */
2522 static void xps_copy_dev_maps(struct xps_dev_maps *dev_maps,
2523 struct xps_dev_maps *new_dev_maps, int index,
2524 int tc, bool skip_tc)
2525 {
2526 int i, tci = index * dev_maps->num_tc;
2527 struct xps_map *map;
2528
2529 /* copy maps belonging to foreign traffic classes */
2530 for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2531 if (i == tc && skip_tc)
2532 continue;
2533
2534 /* fill in the new device map from the old device map */
2535 map = xmap_dereference(dev_maps->attr_map[tci]);
2536 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2537 }
2538 }
2539
2540 /* Must be called under cpus_read_lock */
2541 int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
2542 u16 index, enum xps_map_type type)
2543 {
2544 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL, *old_dev_maps = NULL;
2545 const unsigned long *online_mask = NULL;
2546 bool active = false, copy = false;
2547 int i, j, tci, numa_node_id = -2;
2548 int maps_sz, num_tc = 1, tc = 0;
2549 struct xps_map *map, *new_map;
2550 unsigned int nr_ids;
2551
2552 if (dev->num_tc) {
2553 /* Do not allow XPS on subordinate device directly */
2554 num_tc = dev->num_tc;
2555 if (num_tc < 0)
2556 return -EINVAL;
2557
2558 /* If queue belongs to subordinate dev use its map */
2559 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
2560
2561 tc = netdev_txq_to_tc(dev, index);
2562 if (tc < 0)
2563 return -EINVAL;
2564 }
2565
2566 mutex_lock(&xps_map_mutex);
2567
2568 dev_maps = xmap_dereference(dev->xps_maps[type]);
2569 if (type == XPS_RXQS) {
2570 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
2571 nr_ids = dev->num_rx_queues;
2572 } else {
2573 maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
2574 if (num_possible_cpus() > 1)
2575 online_mask = cpumask_bits(cpu_online_mask);
2576 nr_ids = nr_cpu_ids;
2577 }
2578
2579 if (maps_sz < L1_CACHE_BYTES)
2580 maps_sz = L1_CACHE_BYTES;
2581
2582 /* The old dev_maps could be larger or smaller than the one we're
2583 * setting up now, as dev->num_tc or nr_ids could have been updated in
2584 * between. We could try to be smart, but let's be safe instead and only
2585 * copy foreign traffic classes if the two map sizes match.
2586 */
2587 if (dev_maps &&
2588 dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids)
2589 copy = true;
2590
2591 /* allocate memory for queue storage */
2592 for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
2593 j < nr_ids;) {
2594 if (!new_dev_maps) {
2595 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2596 if (!new_dev_maps) {
2597 mutex_unlock(&xps_map_mutex);
2598 return -ENOMEM;
2599 }
2600
2601 new_dev_maps->nr_ids = nr_ids;
2602 new_dev_maps->num_tc = num_tc;
2603 }
2604
2605 tci = j * num_tc + tc;
2606 map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL;
2607
2608 map = expand_xps_map(map, j, index, type == XPS_RXQS);
2609 if (!map)
2610 goto error;
2611
2612 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2613 }
2614
2615 if (!new_dev_maps)
2616 goto out_no_new_maps;
2617
2618 if (!dev_maps) {
2619 /* Increment static keys at most once per type */
2620 static_key_slow_inc_cpuslocked(&xps_needed);
2621 if (type == XPS_RXQS)
2622 static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
2623 }
2624
2625 for (j = 0; j < nr_ids; j++) {
2626 bool skip_tc = false;
2627
2628 tci = j * num_tc + tc;
2629 if (netif_attr_test_mask(j, mask, nr_ids) &&
2630 netif_attr_test_online(j, online_mask, nr_ids)) {
2631 /* add tx-queue to CPU/rx-queue maps */
2632 int pos = 0;
2633
2634 skip_tc = true;
2635
2636 map = xmap_dereference(new_dev_maps->attr_map[tci]);
2637 while ((pos < map->len) && (map->queues[pos] != index))
2638 pos++;
2639
2640 if (pos == map->len)
2641 map->queues[map->len++] = index;
2642 #ifdef CONFIG_NUMA
2643 if (type == XPS_CPUS) {
2644 if (numa_node_id == -2)
2645 numa_node_id = cpu_to_node(j);
2646 else if (numa_node_id != cpu_to_node(j))
2647 numa_node_id = -1;
2648 }
2649 #endif
2650 }
2651
2652 if (copy)
2653 xps_copy_dev_maps(dev_maps, new_dev_maps, j, tc,
2654 skip_tc);
2655 }
2656
2657 rcu_assign_pointer(dev->xps_maps[type], new_dev_maps);
2658
2659 /* Cleanup old maps */
2660 if (!dev_maps)
2661 goto out_no_old_maps;
2662
2663 for (j = 0; j < dev_maps->nr_ids; j++) {
2664 for (i = num_tc, tci = j * dev_maps->num_tc; i--; tci++) {
2665 map = xmap_dereference(dev_maps->attr_map[tci]);
2666 if (!map)
2667 continue;
2668
2669 if (copy) {
2670 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2671 if (map == new_map)
2672 continue;
2673 }
2674
2675 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2676 kfree_rcu(map, rcu);
2677 }
2678 }
2679
2680 old_dev_maps = dev_maps;
2681
2682 out_no_old_maps:
2683 dev_maps = new_dev_maps;
2684 active = true;
2685
2686 out_no_new_maps:
2687 if (type == XPS_CPUS)
2688 /* update Tx queue numa node */
2689 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2690 (numa_node_id >= 0) ?
2691 numa_node_id : NUMA_NO_NODE);
2692
2693 if (!dev_maps)
2694 goto out_no_maps;
2695
2696 /* removes tx-queue from unused CPUs/rx-queues */
2697 for (j = 0; j < dev_maps->nr_ids; j++) {
2698 tci = j * dev_maps->num_tc;
2699
2700 for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2701 if (i == tc &&
2702 netif_attr_test_mask(j, mask, dev_maps->nr_ids) &&
2703 netif_attr_test_online(j, online_mask, dev_maps->nr_ids))
2704 continue;
2705
2706 active |= remove_xps_queue(dev_maps,
2707 copy ? old_dev_maps : NULL,
2708 tci, index);
2709 }
2710 }
2711
2712 if (old_dev_maps)
2713 kfree_rcu(old_dev_maps, rcu);
2714
2715 /* free map if not active */
2716 if (!active)
2717 reset_xps_maps(dev, dev_maps, type);
2718
2719 out_no_maps:
2720 mutex_unlock(&xps_map_mutex);
2721
2722 return 0;
2723 error:
2724 /* remove any maps that we added */
2725 for (j = 0; j < nr_ids; j++) {
2726 for (i = num_tc, tci = j * num_tc; i--; tci++) {
2727 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2728 map = copy ?
2729 xmap_dereference(dev_maps->attr_map[tci]) :
2730 NULL;
2731 if (new_map && new_map != map)
2732 kfree(new_map);
2733 }
2734 }
2735
2736 mutex_unlock(&xps_map_mutex);
2737
2738 kfree(new_dev_maps);
2739 return -ENOMEM;
2740 }
2741 EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
2742
2743 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2744 u16 index)
2745 {
2746 int ret;
2747
2748 cpus_read_lock();
2749 ret = __netif_set_xps_queue(dev, cpumask_bits(mask), index, XPS_CPUS);
2750 cpus_read_unlock();
2751
2752 return ret;
2753 }
2754 EXPORT_SYMBOL(netif_set_xps_queue);
2755
2756 #endif
2757 static void netdev_unbind_all_sb_channels(struct net_device *dev)
2758 {
2759 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2760
2761 /* Unbind any subordinate channels */
2762 while (txq-- != &dev->_tx[0]) {
2763 if (txq->sb_dev)
2764 netdev_unbind_sb_channel(dev, txq->sb_dev);
2765 }
2766 }
2767
2768 void netdev_reset_tc(struct net_device *dev)
2769 {
2770 #ifdef CONFIG_XPS
2771 netif_reset_xps_queues_gt(dev, 0);
2772 #endif
2773 netdev_unbind_all_sb_channels(dev);
2774
2775 /* Reset TC configuration of device */
2776 dev->num_tc = 0;
2777 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2778 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2779 }
2780 EXPORT_SYMBOL(netdev_reset_tc);
2781
2782 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2783 {
2784 if (tc >= dev->num_tc)
2785 return -EINVAL;
2786
2787 #ifdef CONFIG_XPS
2788 netif_reset_xps_queues(dev, offset, count);
2789 #endif
2790 dev->tc_to_txq[tc].count = count;
2791 dev->tc_to_txq[tc].offset = offset;
2792 return 0;
2793 }
2794 EXPORT_SYMBOL(netdev_set_tc_queue);
2795
2796 int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2797 {
2798 if (num_tc > TC_MAX_QUEUE)
2799 return -EINVAL;
2800
2801 #ifdef CONFIG_XPS
2802 netif_reset_xps_queues_gt(dev, 0);
2803 #endif
2804 netdev_unbind_all_sb_channels(dev);
2805
2806 dev->num_tc = num_tc;
2807 return 0;
2808 }
2809 EXPORT_SYMBOL(netdev_set_num_tc);
2810
2811 void netdev_unbind_sb_channel(struct net_device *dev,
2812 struct net_device *sb_dev)
2813 {
2814 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2815
2816 #ifdef CONFIG_XPS
2817 netif_reset_xps_queues_gt(sb_dev, 0);
2818 #endif
2819 memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq));
2820 memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map));
2821
2822 while (txq-- != &dev->_tx[0]) {
2823 if (txq->sb_dev == sb_dev)
2824 txq->sb_dev = NULL;
2825 }
2826 }
2827 EXPORT_SYMBOL(netdev_unbind_sb_channel);
2828
2829 int netdev_bind_sb_channel_queue(struct net_device *dev,
2830 struct net_device *sb_dev,
2831 u8 tc, u16 count, u16 offset)
2832 {
2833 /* Make certain the sb_dev and dev are already configured */
2834 if (sb_dev->num_tc >= 0 || tc >= dev->num_tc)
2835 return -EINVAL;
2836
2837 /* We cannot hand out queues we don't have */
2838 if ((offset + count) > dev->real_num_tx_queues)
2839 return -EINVAL;
2840
2841 /* Record the mapping */
2842 sb_dev->tc_to_txq[tc].count = count;
2843 sb_dev->tc_to_txq[tc].offset = offset;
2844
2845 /* Provide a way for Tx queue to find the tc_to_txq map or
2846 * XPS map for itself.
2847 */
2848 while (count--)
2849 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev;
2850
2851 return 0;
2852 }
2853 EXPORT_SYMBOL(netdev_bind_sb_channel_queue);
2854
2855 int netdev_set_sb_channel(struct net_device *dev, u16 channel)
2856 {
2857 /* Do not use a multiqueue device to represent a subordinate channel */
2858 if (netif_is_multiqueue(dev))
2859 return -ENODEV;
2860
2861 /* We allow channels 1 - 32767 to be used for subordinate channels.
2862 * Channel 0 is meant to be "native" mode and used only to represent
2863 * the main root device. We allow writing 0 to reset the device back
2864 * to normal mode after being used as a subordinate channel.
2865 */
2866 if (channel > S16_MAX)
2867 return -EINVAL;
2868
2869 dev->num_tc = -channel;
2870
2871 return 0;
2872 }
2873 EXPORT_SYMBOL(netdev_set_sb_channel);
2874
2875 /*
2876 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2877 * greater than real_num_tx_queues stale skbs on the qdisc must be flushed.
2878 */
2879 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2880 {
2881 bool disabling;
2882 int rc;
2883
2884 disabling = txq < dev->real_num_tx_queues;
2885
2886 if (txq < 1 || txq > dev->num_tx_queues)
2887 return -EINVAL;
2888
2889 if (dev->reg_state == NETREG_REGISTERED ||
2890 dev->reg_state == NETREG_UNREGISTERING) {
2891 ASSERT_RTNL();
2892
2893 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2894 txq);
2895 if (rc)
2896 return rc;
2897
2898 if (dev->num_tc)
2899 netif_setup_tc(dev, txq);
2900
2901 dev_qdisc_change_real_num_tx(dev, txq);
2902
2903 dev->real_num_tx_queues = txq;
2904
2905 if (disabling) {
2906 synchronize_net();
2907 qdisc_reset_all_tx_gt(dev, txq);
2908 #ifdef CONFIG_XPS
2909 netif_reset_xps_queues_gt(dev, txq);
2910 #endif
2911 }
2912 } else {
2913 dev->real_num_tx_queues = txq;
2914 }
2915
2916 return 0;
2917 }
2918 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2919
2920 #ifdef CONFIG_SYSFS
2921 /**
2922 * netif_set_real_num_rx_queues - set actual number of RX queues used
2923 * @dev: Network device
2924 * @rxq: Actual number of RX queues
2925 *
2926 * This must be called either with the rtnl_lock held or before
2927 * registration of the net device. Returns 0 on success, or a
2928 * negative error code. If called before registration, it always
2929 * succeeds.
2930 */
2931 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2932 {
2933 int rc;
2934
2935 if (rxq < 1 || rxq > dev->num_rx_queues)
2936 return -EINVAL;
2937
2938 if (dev->reg_state == NETREG_REGISTERED) {
2939 ASSERT_RTNL();
2940
2941 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2942 rxq);
2943 if (rc)
2944 return rc;
2945 }
2946
2947 dev->real_num_rx_queues = rxq;
2948 return 0;
2949 }
2950 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2951 #endif
2952
2953 /**
2954 * netif_set_real_num_queues - set actual number of RX and TX queues used
2955 * @dev: Network device
2956 * @txq: Actual number of TX queues
2957 * @rxq: Actual number of RX queues
2958 *
2959 * Set the real number of both TX and RX queues.
2960 * Does nothing if the number of queues is already correct.
2961 */
2962 int netif_set_real_num_queues(struct net_device *dev,
2963 unsigned int txq, unsigned int rxq)
2964 {
2965 unsigned int old_rxq = dev->real_num_rx_queues;
2966 int err;
2967
2968 if (txq < 1 || txq > dev->num_tx_queues ||
2969 rxq < 1 || rxq > dev->num_rx_queues)
2970 return -EINVAL;
2971
2972 /* Start from increases, so the error path only does decreases -
2973 * decreases can't fail.
2974 */
2975 if (rxq > dev->real_num_rx_queues) {
2976 err = netif_set_real_num_rx_queues(dev, rxq);
2977 if (err)
2978 return err;
2979 }
2980 if (txq > dev->real_num_tx_queues) {
2981 err = netif_set_real_num_tx_queues(dev, txq);
2982 if (err)
2983 goto undo_rx;
2984 }
2985 if (rxq < dev->real_num_rx_queues)
2986 WARN_ON(netif_set_real_num_rx_queues(dev, rxq));
2987 if (txq < dev->real_num_tx_queues)
2988 WARN_ON(netif_set_real_num_tx_queues(dev, txq));
2989
2990 return 0;
2991 undo_rx:
2992 WARN_ON(netif_set_real_num_rx_queues(dev, old_rxq));
2993 return err;
2994 }
2995 EXPORT_SYMBOL(netif_set_real_num_queues);
2996
2997 /**
2998 * netif_set_tso_max_size() - set the max size of TSO frames supported
2999 * @dev: netdev to update
3000 * @size: max skb->len of a TSO frame
3001 *
3002 * Set the limit on the size of TSO super-frames the device can handle.
3003 * Unless explicitly set the stack will assume the value of
3004 * %GSO_LEGACY_MAX_SIZE.
3005 */
3006 void netif_set_tso_max_size(struct net_device *dev, unsigned int size)
3007 {
3008 dev->tso_max_size = min(GSO_MAX_SIZE, size);
3009 if (size < READ_ONCE(dev->gso_max_size))
3010 netif_set_gso_max_size(dev, size);
3011 }
3012 EXPORT_SYMBOL(netif_set_tso_max_size);
3013
3014 /**
3015 * netif_set_tso_max_segs() - set the max number of segs supported for TSO
3016 * @dev: netdev to update
3017 * @segs: max number of TCP segments
3018 *
3019 * Set the limit on the number of TCP segments the device can generate from
3020 * a single TSO super-frame.
3021 * Unless explicitly set the stack will assume the value of %GSO_MAX_SEGS.
3022 */
3023 void netif_set_tso_max_segs(struct net_device *dev, unsigned int segs)
3024 {
3025 dev->tso_max_segs = segs;
3026 if (segs < READ_ONCE(dev->gso_max_segs))
3027 netif_set_gso_max_segs(dev, segs);
3028 }
3029 EXPORT_SYMBOL(netif_set_tso_max_segs);
3030
3031 /**
3032 * netif_inherit_tso_max() - copy all TSO limits from a lower device to an upper
3033 * @to: netdev to update
3034 * @from: netdev from which to copy the limits
3035 */
3036 void netif_inherit_tso_max(struct net_device *to, const struct net_device *from)
3037 {
3038 netif_set_tso_max_size(to, from->tso_max_size);
3039 netif_set_tso_max_segs(to, from->tso_max_segs);
3040 }
3041 EXPORT_SYMBOL(netif_inherit_tso_max);
3042
3043 /**
3044 * netif_get_num_default_rss_queues - default number of RSS queues
3045 *
3046 * Default value is the number of physical cores if there are only 1 or 2, or
3047 * divided by 2 if there are more.
3048 */
3049 int netif_get_num_default_rss_queues(void)
3050 {
3051 cpumask_var_t cpus;
3052 int cpu, count = 0;
3053
3054 if (unlikely(is_kdump_kernel() || !zalloc_cpumask_var(&cpus, GFP_KERNEL)))
3055 return 1;
3056
3057 cpumask_copy(cpus, cpu_online_mask);
3058 for_each_cpu(cpu, cpus) {
3059 ++count;
3060 cpumask_andnot(cpus, cpus, topology_sibling_cpumask(cpu));
3061 }
3062 free_cpumask_var(cpus);
3063
3064 return count > 2 ? DIV_ROUND_UP(count, 2) : count;
3065 }
3066 EXPORT_SYMBOL(netif_get_num_default_rss_queues);
3067
3068 static void __netif_reschedule(struct Qdisc *q)
3069 {
3070 struct softnet_data *sd;
3071 unsigned long flags;
3072
3073 local_irq_save(flags);
3074 sd = this_cpu_ptr(&softnet_data);
3075 q->next_sched = NULL;
3076 *sd->output_queue_tailp = q;
3077 sd->output_queue_tailp = &q->next_sched;
3078 raise_softirq_irqoff(NET_TX_SOFTIRQ);
3079 local_irq_restore(flags);
3080 }
3081
3082 void __netif_schedule(struct Qdisc *q)
3083 {
3084 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
3085 __netif_reschedule(q);
3086 }
3087 EXPORT_SYMBOL(__netif_schedule);
3088
3089 struct dev_kfree_skb_cb {
3090 enum skb_free_reason reason;
3091 };
3092
3093 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
3094 {
3095 return (struct dev_kfree_skb_cb *)skb->cb;
3096 }
3097
3098 void netif_schedule_queue(struct netdev_queue *txq)
3099 {
3100 rcu_read_lock();
3101 if (!netif_xmit_stopped(txq)) {
3102 struct Qdisc *q = rcu_dereference(txq->qdisc);
3103
3104 __netif_schedule(q);
3105 }
3106 rcu_read_unlock();
3107 }
3108 EXPORT_SYMBOL(netif_schedule_queue);
3109
3110 void netif_tx_wake_queue(struct netdev_queue *dev_queue)
3111 {
3112 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
3113 struct Qdisc *q;
3114
3115 rcu_read_lock();
3116 q = rcu_dereference(dev_queue->qdisc);
3117 __netif_schedule(q);
3118 rcu_read_unlock();
3119 }
3120 }
3121 EXPORT_SYMBOL(netif_tx_wake_queue);
3122
3123 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
3124 {
3125 unsigned long flags;
3126
3127 if (unlikely(!skb))
3128 return;
3129
3130 if (likely(refcount_read(&skb->users) == 1)) {
3131 smp_rmb();
3132 refcount_set(&skb->users, 0);
3133 } else if (likely(!refcount_dec_and_test(&skb->users))) {
3134 return;
3135 }
3136 get_kfree_skb_cb(skb)->reason = reason;
3137 local_irq_save(flags);
3138 skb->next = __this_cpu_read(softnet_data.completion_queue);
3139 __this_cpu_write(softnet_data.completion_queue, skb);
3140 raise_softirq_irqoff(NET_TX_SOFTIRQ);
3141 local_irq_restore(flags);
3142 }
3143 EXPORT_SYMBOL(__dev_kfree_skb_irq);
3144
3145 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
3146 {
3147 if (in_hardirq() || irqs_disabled())
3148 __dev_kfree_skb_irq(skb, reason);
3149 else
3150 dev_kfree_skb(skb);
3151 }
3152 EXPORT_SYMBOL(__dev_kfree_skb_any);
3153
3154
3155 /**
3156 * netif_device_detach - mark device as removed
3157 * @dev: network device
3158 *
3159 * Mark device as removed from system and therefore no longer available.
3160 */
3161 void netif_device_detach(struct net_device *dev)
3162 {
3163 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
3164 netif_running(dev)) {
3165 netif_tx_stop_all_queues(dev);
3166 }
3167 }
3168 EXPORT_SYMBOL(netif_device_detach);
3169
3170 /**
3171 * netif_device_attach - mark device as attached
3172 * @dev: network device
3173 *
3174 * Mark device as attached from system and restart if needed.
3175 */
3176 void netif_device_attach(struct net_device *dev)
3177 {
3178 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
3179 netif_running(dev)) {
3180 netif_tx_wake_all_queues(dev);
3181 __netdev_watchdog_up(dev);
3182 }
3183 }
3184 EXPORT_SYMBOL(netif_device_attach);
3185
3186 /*
3187 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
3188 * to be used as a distribution range.
3189 */
3190 static u16 skb_tx_hash(const struct net_device *dev,
3191 const struct net_device *sb_dev,
3192 struct sk_buff *skb)
3193 {
3194 u32 hash;
3195 u16 qoffset = 0;
3196 u16 qcount = dev->real_num_tx_queues;
3197
3198 if (dev->num_tc) {
3199 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
3200
3201 qoffset = sb_dev->tc_to_txq[tc].offset;
3202 qcount = sb_dev->tc_to_txq[tc].count;
3203 if (unlikely(!qcount)) {
3204 net_warn_ratelimited("%s: invalid qcount, qoffset %u for tc %u\n",
3205 sb_dev->name, qoffset, tc);
3206 qoffset = 0;
3207 qcount = dev->real_num_tx_queues;
3208 }
3209 }
3210
3211 if (skb_rx_queue_recorded(skb)) {
3212 hash = skb_get_rx_queue(skb);
3213 if (hash >= qoffset)
3214 hash -= qoffset;
3215 while (unlikely(hash >= qcount))
3216 hash -= qcount;
3217 return hash + qoffset;
3218 }
3219
3220 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
3221 }
3222
3223 static void skb_warn_bad_offload(const struct sk_buff *skb)
3224 {
3225 static const netdev_features_t null_features;
3226 struct net_device *dev = skb->dev;
3227 const char *name = "";
3228
3229 if (!net_ratelimit())
3230 return;
3231
3232 if (dev) {
3233 if (dev->dev.parent)
3234 name = dev_driver_string(dev->dev.parent);
3235 else
3236 name = netdev_name(dev);
3237 }
3238 skb_dump(KERN_WARNING, skb, false);
3239 WARN(1, "%s: caps=(%pNF, %pNF)\n",
3240 name, dev ? &dev->features : &null_features,
3241 skb->sk ? &skb->sk->sk_route_caps : &null_features);
3242 }
3243
3244 /*
3245 * Invalidate hardware checksum when packet is to be mangled, and
3246 * complete checksum manually on outgoing path.
3247 */
3248 int skb_checksum_help(struct sk_buff *skb)
3249 {
3250 __wsum csum;
3251 int ret = 0, offset;
3252
3253 if (skb->ip_summed == CHECKSUM_COMPLETE)
3254 goto out_set_summed;
3255
3256 if (unlikely(skb_is_gso(skb))) {
3257 skb_warn_bad_offload(skb);
3258 return -EINVAL;
3259 }
3260
3261 /* Before computing a checksum, we should make sure no frag could
3262 * be modified by an external entity : checksum could be wrong.
3263 */
3264 if (skb_has_shared_frag(skb)) {
3265 ret = __skb_linearize(skb);
3266 if (ret)
3267 goto out;
3268 }
3269
3270 offset = skb_checksum_start_offset(skb);
3271 ret = -EINVAL;
3272 if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
3273 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3274 goto out;
3275 }
3276 csum = skb_checksum(skb, offset, skb->len - offset, 0);
3277
3278 offset += skb->csum_offset;
3279 if (WARN_ON_ONCE(offset + sizeof(__sum16) > skb_headlen(skb))) {
3280 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3281 goto out;
3282 }
3283 ret = skb_ensure_writable(skb, offset + sizeof(__sum16));
3284 if (ret)
3285 goto out;
3286
3287 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
3288 out_set_summed:
3289 skb->ip_summed = CHECKSUM_NONE;
3290 out:
3291 return ret;
3292 }
3293 EXPORT_SYMBOL(skb_checksum_help);
3294
3295 int skb_crc32c_csum_help(struct sk_buff *skb)
3296 {
3297 __le32 crc32c_csum;
3298 int ret = 0, offset, start;
3299
3300 if (skb->ip_summed != CHECKSUM_PARTIAL)
3301 goto out;
3302
3303 if (unlikely(skb_is_gso(skb)))
3304 goto out;
3305
3306 /* Before computing a checksum, we should make sure no frag could
3307 * be modified by an external entity : checksum could be wrong.
3308 */
3309 if (unlikely(skb_has_shared_frag(skb))) {
3310 ret = __skb_linearize(skb);
3311 if (ret)
3312 goto out;
3313 }
3314 start = skb_checksum_start_offset(skb);
3315 offset = start + offsetof(struct sctphdr, checksum);
3316 if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
3317 ret = -EINVAL;
3318 goto out;
3319 }
3320
3321 ret = skb_ensure_writable(skb, offset + sizeof(__le32));
3322 if (ret)
3323 goto out;
3324
3325 crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
3326 skb->len - start, ~(__u32)0,
3327 crc32c_csum_stub));
3328 *(__le32 *)(skb->data + offset) = crc32c_csum;
3329 skb->ip_summed = CHECKSUM_NONE;
3330 skb->csum_not_inet = 0;
3331 out:
3332 return ret;
3333 }
3334
3335 __be16 skb_network_protocol(struct sk_buff *skb, int *depth)
3336 {
3337 __be16 type = skb->protocol;
3338
3339 /* Tunnel gso handlers can set protocol to ethernet. */
3340 if (type == htons(ETH_P_TEB)) {
3341 struct ethhdr *eth;
3342
3343 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
3344 return 0;
3345
3346 eth = (struct ethhdr *)skb->data;
3347 type = eth->h_proto;
3348 }
3349
3350 return __vlan_get_protocol(skb, type, depth);
3351 }
3352
3353 /* openvswitch calls this on rx path, so we need a different check.
3354 */
3355 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
3356 {
3357 if (tx_path)
3358 return skb->ip_summed != CHECKSUM_PARTIAL &&
3359 skb->ip_summed != CHECKSUM_UNNECESSARY;
3360
3361 return skb->ip_summed == CHECKSUM_NONE;
3362 }
3363
3364 /**
3365 * __skb_gso_segment - Perform segmentation on skb.
3366 * @skb: buffer to segment
3367 * @features: features for the output path (see dev->features)
3368 * @tx_path: whether it is called in TX path
3369 *
3370 * This function segments the given skb and returns a list of segments.
3371 *
3372 * It may return NULL if the skb requires no segmentation. This is
3373 * only possible when GSO is used for verifying header integrity.
3374 *
3375 * Segmentation preserves SKB_GSO_CB_OFFSET bytes of previous skb cb.
3376 */
3377 struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
3378 netdev_features_t features, bool tx_path)
3379 {
3380 struct sk_buff *segs;
3381
3382 if (unlikely(skb_needs_check(skb, tx_path))) {
3383 int err;
3384
3385 /* We're going to init ->check field in TCP or UDP header */
3386 err = skb_cow_head(skb, 0);
3387 if (err < 0)
3388 return ERR_PTR(err);
3389 }
3390
3391 /* Only report GSO partial support if it will enable us to
3392 * support segmentation on this frame without needing additional
3393 * work.
3394 */
3395 if (features & NETIF_F_GSO_PARTIAL) {
3396 netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
3397 struct net_device *dev = skb->dev;
3398
3399 partial_features |= dev->features & dev->gso_partial_features;
3400 if (!skb_gso_ok(skb, features | partial_features))
3401 features &= ~NETIF_F_GSO_PARTIAL;
3402 }
3403
3404 BUILD_BUG_ON(SKB_GSO_CB_OFFSET +
3405 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
3406
3407 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
3408 SKB_GSO_CB(skb)->encap_level = 0;
3409
3410 skb_reset_mac_header(skb);
3411 skb_reset_mac_len(skb);
3412
3413 segs = skb_mac_gso_segment(skb, features);
3414
3415 if (segs != skb && unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
3416 skb_warn_bad_offload(skb);
3417
3418 return segs;
3419 }
3420 EXPORT_SYMBOL(__skb_gso_segment);
3421
3422 /* Take action when hardware reception checksum errors are detected. */
3423 #ifdef CONFIG_BUG
3424 static void do_netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3425 {
3426 netdev_err(dev, "hw csum failure\n");
3427 skb_dump(KERN_ERR, skb, true);
3428 dump_stack();
3429 }
3430
3431 void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3432 {
3433 DO_ONCE_LITE(do_netdev_rx_csum_fault, dev, skb);
3434 }
3435 EXPORT_SYMBOL(netdev_rx_csum_fault);
3436 #endif
3437
3438 /* XXX: check that highmem exists at all on the given machine. */
3439 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
3440 {
3441 #ifdef CONFIG_HIGHMEM
3442 int i;
3443
3444 if (!(dev->features & NETIF_F_HIGHDMA)) {
3445 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3446 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3447
3448 if (PageHighMem(skb_frag_page(frag)))
3449 return 1;
3450 }
3451 }
3452 #endif
3453 return 0;
3454 }
3455
3456 /* If MPLS offload request, verify we are testing hardware MPLS features
3457 * instead of standard features for the netdev.
3458 */
3459 #if IS_ENABLED(CONFIG_NET_MPLS_GSO)
3460 static netdev_features_t net_mpls_features(struct sk_buff *skb,
3461 netdev_features_t features,
3462 __be16 type)
3463 {
3464 if (eth_p_mpls(type))
3465 features &= skb->dev->mpls_features;
3466
3467 return features;
3468 }
3469 #else
3470 static netdev_features_t net_mpls_features(struct sk_buff *skb,
3471 netdev_features_t features,
3472 __be16 type)
3473 {
3474 return features;
3475 }
3476 #endif
3477
3478 static netdev_features_t harmonize_features(struct sk_buff *skb,
3479 netdev_features_t features)
3480 {
3481 __be16 type;
3482
3483 type = skb_network_protocol(skb, NULL);
3484 features = net_mpls_features(skb, features, type);
3485
3486 if (skb->ip_summed != CHECKSUM_NONE &&
3487 !can_checksum_protocol(features, type)) {
3488 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3489 }
3490 if (illegal_highdma(skb->dev, skb))
3491 features &= ~NETIF_F_SG;
3492
3493 return features;
3494 }
3495
3496 netdev_features_t passthru_features_check(struct sk_buff *skb,
3497 struct net_device *dev,
3498 netdev_features_t features)
3499 {
3500 return features;
3501 }
3502 EXPORT_SYMBOL(passthru_features_check);
3503
3504 static netdev_features_t dflt_features_check(struct sk_buff *skb,
3505 struct net_device *dev,
3506 netdev_features_t features)
3507 {
3508 return vlan_features_check(skb, features);
3509 }
3510
3511 static netdev_features_t gso_features_check(const struct sk_buff *skb,
3512 struct net_device *dev,
3513 netdev_features_t features)
3514 {
3515 u16 gso_segs = skb_shinfo(skb)->gso_segs;
3516
3517 if (gso_segs > READ_ONCE(dev->gso_max_segs))
3518 return features & ~NETIF_F_GSO_MASK;
3519
3520 if (!skb_shinfo(skb)->gso_type) {
3521 skb_warn_bad_offload(skb);
3522 return features & ~NETIF_F_GSO_MASK;
3523 }
3524
3525 /* Support for GSO partial features requires software
3526 * intervention before we can actually process the packets
3527 * so we need to strip support for any partial features now
3528 * and we can pull them back in after we have partially
3529 * segmented the frame.
3530 */
3531 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
3532 features &= ~dev->gso_partial_features;
3533
3534 /* Make sure to clear the IPv4 ID mangling feature if the
3535 * IPv4 header has the potential to be fragmented.
3536 */
3537 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
3538 struct iphdr *iph = skb->encapsulation ?
3539 inner_ip_hdr(skb) : ip_hdr(skb);
3540
3541 if (!(iph->frag_off & htons(IP_DF)))
3542 features &= ~NETIF_F_TSO_MANGLEID;
3543 }
3544
3545 return features;
3546 }
3547
3548 netdev_features_t netif_skb_features(struct sk_buff *skb)
3549 {
3550 struct net_device *dev = skb->dev;
3551 netdev_features_t features = dev->features;
3552
3553 if (skb_is_gso(skb))
3554 features = gso_features_check(skb, dev, features);
3555
3556 /* If encapsulation offload request, verify we are testing
3557 * hardware encapsulation features instead of standard
3558 * features for the netdev
3559 */
3560 if (skb->encapsulation)
3561 features &= dev->hw_enc_features;
3562
3563 if (skb_vlan_tagged(skb))
3564 features = netdev_intersect_features(features,
3565 dev->vlan_features |
3566 NETIF_F_HW_VLAN_CTAG_TX |
3567 NETIF_F_HW_VLAN_STAG_TX);
3568
3569 if (dev->netdev_ops->ndo_features_check)
3570 features &= dev->netdev_ops->ndo_features_check(skb, dev,
3571 features);
3572 else
3573 features &= dflt_features_check(skb, dev, features);
3574
3575 return harmonize_features(skb, features);
3576 }
3577 EXPORT_SYMBOL(netif_skb_features);
3578
3579 static int xmit_one(struct sk_buff *skb, struct net_device *dev,
3580 struct netdev_queue *txq, bool more)
3581 {
3582 unsigned int len;
3583 int rc;
3584
3585 if (dev_nit_active(dev))
3586 dev_queue_xmit_nit(skb, dev);
3587
3588 len = skb->len;
3589 trace_net_dev_start_xmit(skb, dev);
3590 rc = netdev_start_xmit(skb, dev, txq, more);
3591 trace_net_dev_xmit(skb, rc, dev, len);
3592
3593 return rc;
3594 }
3595
3596 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
3597 struct netdev_queue *txq, int *ret)
3598 {
3599 struct sk_buff *skb = first;
3600 int rc = NETDEV_TX_OK;
3601
3602 while (skb) {
3603 struct sk_buff *next = skb->next;
3604
3605 skb_mark_not_on_list(skb);
3606 rc = xmit_one(skb, dev, txq, next != NULL);
3607 if (unlikely(!dev_xmit_complete(rc))) {
3608 skb->next = next;
3609 goto out;
3610 }
3611
3612 skb = next;
3613 if (netif_tx_queue_stopped(txq) && skb) {
3614 rc = NETDEV_TX_BUSY;
3615 break;
3616 }
3617 }
3618
3619 out:
3620 *ret = rc;
3621 return skb;
3622 }
3623
3624 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
3625 netdev_features_t features)
3626 {
3627 if (skb_vlan_tag_present(skb) &&
3628 !vlan_hw_offload_capable(features, skb->vlan_proto))
3629 skb = __vlan_hwaccel_push_inside(skb);
3630 return skb;
3631 }
3632
3633 int skb_csum_hwoffload_help(struct sk_buff *skb,
3634 const netdev_features_t features)
3635 {
3636 if (unlikely(skb_csum_is_sctp(skb)))
3637 return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3638 skb_crc32c_csum_help(skb);
3639
3640 if (features & NETIF_F_HW_CSUM)
3641 return 0;
3642
3643 if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
3644 switch (skb->csum_offset) {
3645 case offsetof(struct tcphdr, check):
3646 case offsetof(struct udphdr, check):
3647 return 0;
3648 }
3649 }
3650
3651 return skb_checksum_help(skb);
3652 }
3653 EXPORT_SYMBOL(skb_csum_hwoffload_help);
3654
3655 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)
3656 {
3657 netdev_features_t features;
3658
3659 features = netif_skb_features(skb);
3660 skb = validate_xmit_vlan(skb, features);
3661 if (unlikely(!skb))
3662 goto out_null;
3663
3664 skb = sk_validate_xmit_skb(skb, dev);
3665 if (unlikely(!skb))
3666 goto out_null;
3667
3668 if (netif_needs_gso(skb, features)) {
3669 struct sk_buff *segs;
3670
3671 segs = skb_gso_segment(skb, features);
3672 if (IS_ERR(segs)) {
3673 goto out_kfree_skb;
3674 } else if (segs) {
3675 consume_skb(skb);
3676 skb = segs;
3677 }
3678 } else {
3679 if (skb_needs_linearize(skb, features) &&
3680 __skb_linearize(skb))
3681 goto out_kfree_skb;
3682
3683 /* If packet is not checksummed and device does not
3684 * support checksumming for this protocol, complete
3685 * checksumming here.
3686 */
3687 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3688 if (skb->encapsulation)
3689 skb_set_inner_transport_header(skb,
3690 skb_checksum_start_offset(skb));
3691 else
3692 skb_set_transport_header(skb,
3693 skb_checksum_start_offset(skb));
3694 if (skb_csum_hwoffload_help(skb, features))
3695 goto out_kfree_skb;
3696 }
3697 }
3698
3699 skb = validate_xmit_xfrm(skb, features, again);
3700
3701 return skb;
3702
3703 out_kfree_skb:
3704 kfree_skb(skb);
3705 out_null:
3706 dev_core_stats_tx_dropped_inc(dev);
3707 return NULL;
3708 }
3709
3710 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again)
3711 {
3712 struct sk_buff *next, *head = NULL, *tail;
3713
3714 for (; skb != NULL; skb = next) {
3715 next = skb->next;
3716 skb_mark_not_on_list(skb);
3717
3718 /* in case skb wont be segmented, point to itself */
3719 skb->prev = skb;
3720
3721 skb = validate_xmit_skb(skb, dev, again);
3722 if (!skb)
3723 continue;
3724
3725 if (!head)
3726 head = skb;
3727 else
3728 tail->next = skb;
3729 /* If skb was segmented, skb->prev points to
3730 * the last segment. If not, it still contains skb.
3731 */
3732 tail = skb->prev;
3733 }
3734 return head;
3735 }
3736 EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3737
3738 static void qdisc_pkt_len_init(struct sk_buff *skb)
3739 {
3740 const struct skb_shared_info *shinfo = skb_shinfo(skb);
3741
3742 qdisc_skb_cb(skb)->pkt_len = skb->len;
3743
3744 /* To get more precise estimation of bytes sent on wire,
3745 * we add to pkt_len the headers size of all segments
3746 */
3747 if (shinfo->gso_size && skb_transport_header_was_set(skb)) {
3748 unsigned int hdr_len;
3749 u16 gso_segs = shinfo->gso_segs;
3750
3751 /* mac layer + network layer */
3752 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3753
3754 /* + transport layer */
3755 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
3756 const struct tcphdr *th;
3757 struct tcphdr _tcphdr;
3758
3759 th = skb_header_pointer(skb, skb_transport_offset(skb),
3760 sizeof(_tcphdr), &_tcphdr);
3761 if (likely(th))
3762 hdr_len += __tcp_hdrlen(th);
3763 } else {
3764 struct udphdr _udphdr;
3765
3766 if (skb_header_pointer(skb, skb_transport_offset(skb),
3767 sizeof(_udphdr), &_udphdr))
3768 hdr_len += sizeof(struct udphdr);
3769 }
3770
3771 if (shinfo->gso_type & SKB_GSO_DODGY)
3772 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3773 shinfo->gso_size);
3774
3775 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3776 }
3777 }
3778
3779 static int dev_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *q,
3780 struct sk_buff **to_free,
3781 struct netdev_queue *txq)
3782 {
3783 int rc;
3784
3785 rc = q->enqueue(skb, q, to_free) & NET_XMIT_MASK;
3786 if (rc == NET_XMIT_SUCCESS)
3787 trace_qdisc_enqueue(q, txq, skb);
3788 return rc;
3789 }
3790
3791 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3792 struct net_device *dev,
3793 struct netdev_queue *txq)
3794 {
3795 spinlock_t *root_lock = qdisc_lock(q);
3796 struct sk_buff *to_free = NULL;
3797 bool contended;
3798 int rc;
3799
3800 qdisc_calculate_pkt_len(skb, q);
3801
3802 if (q->flags & TCQ_F_NOLOCK) {
3803 if (q->flags & TCQ_F_CAN_BYPASS && nolock_qdisc_is_empty(q) &&
3804 qdisc_run_begin(q)) {
3805 /* Retest nolock_qdisc_is_empty() within the protection
3806 * of q->seqlock to protect from racing with requeuing.
3807 */
3808 if (unlikely(!nolock_qdisc_is_empty(q))) {
3809 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3810 __qdisc_run(q);
3811 qdisc_run_end(q);
3812
3813 goto no_lock_out;
3814 }
3815
3816 qdisc_bstats_cpu_update(q, skb);
3817 if (sch_direct_xmit(skb, q, dev, txq, NULL, true) &&
3818 !nolock_qdisc_is_empty(q))
3819 __qdisc_run(q);
3820
3821 qdisc_run_end(q);
3822 return NET_XMIT_SUCCESS;
3823 }
3824
3825 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3826 qdisc_run(q);
3827
3828 no_lock_out:
3829 if (unlikely(to_free))
3830 kfree_skb_list_reason(to_free,
3831 SKB_DROP_REASON_QDISC_DROP);
3832 return rc;
3833 }
3834
3835 /*
3836 * Heuristic to force contended enqueues to serialize on a
3837 * separate lock before trying to get qdisc main lock.
3838 * This permits qdisc->running owner to get the lock more
3839 * often and dequeue packets faster.
3840 * On PREEMPT_RT it is possible to preempt the qdisc owner during xmit
3841 * and then other tasks will only enqueue packets. The packets will be
3842 * sent after the qdisc owner is scheduled again. To prevent this
3843 * scenario the task always serialize on the lock.
3844 */
3845 contended = qdisc_is_running(q) || IS_ENABLED(CONFIG_PREEMPT_RT);
3846 if (unlikely(contended))
3847 spin_lock(&q->busylock);
3848
3849 spin_lock(root_lock);
3850 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3851 __qdisc_drop(skb, &to_free);
3852 rc = NET_XMIT_DROP;
3853 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3854 qdisc_run_begin(q)) {
3855 /*
3856 * This is a work-conserving queue; there are no old skbs
3857 * waiting to be sent out; and the qdisc is not running -
3858 * xmit the skb directly.
3859 */
3860
3861 qdisc_bstats_update(q, skb);
3862
3863 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3864 if (unlikely(contended)) {
3865 spin_unlock(&q->busylock);
3866 contended = false;
3867 }
3868 __qdisc_run(q);
3869 }
3870
3871 qdisc_run_end(q);
3872 rc = NET_XMIT_SUCCESS;
3873 } else {
3874 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3875 if (qdisc_run_begin(q)) {
3876 if (unlikely(contended)) {
3877 spin_unlock(&q->busylock);
3878 contended = false;
3879 }
3880 __qdisc_run(q);
3881 qdisc_run_end(q);
3882 }
3883 }
3884 spin_unlock(root_lock);
3885 if (unlikely(to_free))
3886 kfree_skb_list_reason(to_free, SKB_DROP_REASON_QDISC_DROP);
3887 if (unlikely(contended))
3888 spin_unlock(&q->busylock);
3889 return rc;
3890 }
3891
3892 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3893 static void skb_update_prio(struct sk_buff *skb)
3894 {
3895 const struct netprio_map *map;
3896 const struct sock *sk;
3897 unsigned int prioidx;
3898
3899 if (skb->priority)
3900 return;
3901 map = rcu_dereference_bh(skb->dev->priomap);
3902 if (!map)
3903 return;
3904 sk = skb_to_full_sk(skb);
3905 if (!sk)
3906 return;
3907
3908 prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
3909
3910 if (prioidx < map->priomap_len)
3911 skb->priority = map->priomap[prioidx];
3912 }
3913 #else
3914 #define skb_update_prio(skb)
3915 #endif
3916
3917 /**
3918 * dev_loopback_xmit - loop back @skb
3919 * @net: network namespace this loopback is happening in
3920 * @sk: sk needed to be a netfilter okfn
3921 * @skb: buffer to transmit
3922 */
3923 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3924 {
3925 skb_reset_mac_header(skb);
3926 __skb_pull(skb, skb_network_offset(skb));
3927 skb->pkt_type = PACKET_LOOPBACK;
3928 if (skb->ip_summed == CHECKSUM_NONE)
3929 skb->ip_summed = CHECKSUM_UNNECESSARY;
3930 DEBUG_NET_WARN_ON_ONCE(!skb_dst(skb));
3931 skb_dst_force(skb);
3932 netif_rx(skb);
3933 return 0;
3934 }
3935 EXPORT_SYMBOL(dev_loopback_xmit);
3936
3937 #ifdef CONFIG_NET_EGRESS
3938 static struct sk_buff *
3939 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3940 {
3941 #ifdef CONFIG_NET_CLS_ACT
3942 struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress);
3943 struct tcf_result cl_res;
3944
3945 if (!miniq)
3946 return skb;
3947
3948 /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
3949 tc_skb_cb(skb)->mru = 0;
3950 tc_skb_cb(skb)->post_ct = false;
3951 mini_qdisc_bstats_cpu_update(miniq, skb);
3952
3953 switch (tcf_classify(skb, miniq->block, miniq->filter_list, &cl_res, false)) {
3954 case TC_ACT_OK:
3955 case TC_ACT_RECLASSIFY:
3956 skb->tc_index = TC_H_MIN(cl_res.classid);
3957 break;
3958 case TC_ACT_SHOT:
3959 mini_qdisc_qstats_cpu_drop(miniq);
3960 *ret = NET_XMIT_DROP;
3961 kfree_skb_reason(skb, SKB_DROP_REASON_TC_EGRESS);
3962 return NULL;
3963 case TC_ACT_STOLEN:
3964 case TC_ACT_QUEUED:
3965 case TC_ACT_TRAP:
3966 *ret = NET_XMIT_SUCCESS;
3967 consume_skb(skb);
3968 return NULL;
3969 case TC_ACT_REDIRECT:
3970 /* No need to push/pop skb's mac_header here on egress! */
3971 skb_do_redirect(skb);
3972 *ret = NET_XMIT_SUCCESS;
3973 return NULL;
3974 default:
3975 break;
3976 }
3977 #endif /* CONFIG_NET_CLS_ACT */
3978
3979 return skb;
3980 }
3981
3982 static struct netdev_queue *
3983 netdev_tx_queue_mapping(struct net_device *dev, struct sk_buff *skb)
3984 {
3985 int qm = skb_get_queue_mapping(skb);
3986
3987 return netdev_get_tx_queue(dev, netdev_cap_txqueue(dev, qm));
3988 }
3989
3990 static bool netdev_xmit_txqueue_skipped(void)
3991 {
3992 return __this_cpu_read(softnet_data.xmit.skip_txqueue);
3993 }
3994
3995 void netdev_xmit_skip_txqueue(bool skip)
3996 {
3997 __this_cpu_write(softnet_data.xmit.skip_txqueue, skip);
3998 }
3999 EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue);
4000 #endif /* CONFIG_NET_EGRESS */
4001
4002 #ifdef CONFIG_XPS
4003 static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb,
4004 struct xps_dev_maps *dev_maps, unsigned int tci)
4005 {
4006 int tc = netdev_get_prio_tc_map(dev, skb->priority);
4007 struct xps_map *map;
4008 int queue_index = -1;
4009
4010 if (tc >= dev_maps->num_tc || tci >= dev_maps->nr_ids)
4011 return queue_index;
4012
4013 tci *= dev_maps->num_tc;
4014 tci += tc;
4015
4016 map = rcu_dereference(dev_maps->attr_map[tci]);
4017 if (map) {
4018 if (map->len == 1)
4019 queue_index = map->queues[0];
4020 else
4021 queue_index = map->queues[reciprocal_scale(
4022 skb_get_hash(skb), map->len)];
4023 if (unlikely(queue_index >= dev->real_num_tx_queues))
4024 queue_index = -1;
4025 }
4026 return queue_index;
4027 }
4028 #endif
4029
4030 static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev,
4031 struct sk_buff *skb)
4032 {
4033 #ifdef CONFIG_XPS
4034 struct xps_dev_maps *dev_maps;
4035 struct sock *sk = skb->sk;
4036 int queue_index = -1;
4037
4038 if (!static_key_false(&xps_needed))
4039 return -1;
4040
4041 rcu_read_lock();
4042 if (!static_key_false(&xps_rxqs_needed))
4043 goto get_cpus_map;
4044
4045 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]);
4046 if (dev_maps) {
4047 int tci = sk_rx_queue_get(sk);
4048
4049 if (tci >= 0)
4050 queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4051 tci);
4052 }
4053
4054 get_cpus_map:
4055 if (queue_index < 0) {
4056 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]);
4057 if (dev_maps) {
4058 unsigned int tci = skb->sender_cpu - 1;
4059
4060 queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4061 tci);
4062 }
4063 }
4064 rcu_read_unlock();
4065
4066 return queue_index;
4067 #else
4068 return -1;
4069 #endif
4070 }
4071
4072 u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
4073 struct net_device *sb_dev)
4074 {
4075 return 0;
4076 }
4077 EXPORT_SYMBOL(dev_pick_tx_zero);
4078
4079 u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb,
4080 struct net_device *sb_dev)
4081 {
4082 return (u16)raw_smp_processor_id() % dev->real_num_tx_queues;
4083 }
4084 EXPORT_SYMBOL(dev_pick_tx_cpu_id);
4085
4086 u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
4087 struct net_device *sb_dev)
4088 {
4089 struct sock *sk = skb->sk;
4090 int queue_index = sk_tx_queue_get(sk);
4091
4092 sb_dev = sb_dev ? : dev;
4093
4094 if (queue_index < 0 || skb->ooo_okay ||
4095 queue_index >= dev->real_num_tx_queues) {
4096 int new_index = get_xps_queue(dev, sb_dev, skb);
4097
4098 if (new_index < 0)
4099 new_index = skb_tx_hash(dev, sb_dev, skb);
4100
4101 if (queue_index != new_index && sk &&
4102 sk_fullsock(sk) &&
4103 rcu_access_pointer(sk->sk_dst_cache))
4104 sk_tx_queue_set(sk, new_index);
4105
4106 queue_index = new_index;
4107 }
4108
4109 return queue_index;
4110 }
4111 EXPORT_SYMBOL(netdev_pick_tx);
4112
4113 struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
4114 struct sk_buff *skb,
4115 struct net_device *sb_dev)
4116 {
4117 int queue_index = 0;
4118
4119 #ifdef CONFIG_XPS
4120 u32 sender_cpu = skb->sender_cpu - 1;
4121
4122 if (sender_cpu >= (u32)NR_CPUS)
4123 skb->sender_cpu = raw_smp_processor_id() + 1;
4124 #endif
4125
4126 if (dev->real_num_tx_queues != 1) {
4127 const struct net_device_ops *ops = dev->netdev_ops;
4128
4129 if (ops->ndo_select_queue)
4130 queue_index = ops->ndo_select_queue(dev, skb, sb_dev);
4131 else
4132 queue_index = netdev_pick_tx(dev, skb, sb_dev);
4133
4134 queue_index = netdev_cap_txqueue(dev, queue_index);
4135 }
4136
4137 skb_set_queue_mapping(skb, queue_index);
4138 return netdev_get_tx_queue(dev, queue_index);
4139 }
4140
4141 /**
4142 * __dev_queue_xmit() - transmit a buffer
4143 * @skb: buffer to transmit
4144 * @sb_dev: suboordinate device used for L2 forwarding offload
4145 *
4146 * Queue a buffer for transmission to a network device. The caller must
4147 * have set the device and priority and built the buffer before calling
4148 * this function. The function can be called from an interrupt.
4149 *
4150 * When calling this method, interrupts MUST be enabled. This is because
4151 * the BH enable code must have IRQs enabled so that it will not deadlock.
4152 *
4153 * Regardless of the return value, the skb is consumed, so it is currently
4154 * difficult to retry a send to this method. (You can bump the ref count
4155 * before sending to hold a reference for retry if you are careful.)
4156 *
4157 * Return:
4158 * * 0 - buffer successfully transmitted
4159 * * positive qdisc return code - NET_XMIT_DROP etc.
4160 * * negative errno - other errors
4161 */
4162 int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
4163 {
4164 struct net_device *dev = skb->dev;
4165 struct netdev_queue *txq = NULL;
4166 struct Qdisc *q;
4167 int rc = -ENOMEM;
4168 bool again = false;
4169
4170 skb_reset_mac_header(skb);
4171 skb_assert_len(skb);
4172
4173 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
4174 __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED);
4175
4176 /* Disable soft irqs for various locks below. Also
4177 * stops preemption for RCU.
4178 */
4179 rcu_read_lock_bh();
4180
4181 skb_update_prio(skb);
4182
4183 qdisc_pkt_len_init(skb);
4184 #ifdef CONFIG_NET_CLS_ACT
4185 skb->tc_at_ingress = 0;
4186 #endif
4187 #ifdef CONFIG_NET_EGRESS
4188 if (static_branch_unlikely(&egress_needed_key)) {
4189 if (nf_hook_egress_active()) {
4190 skb = nf_hook_egress(skb, &rc, dev);
4191 if (!skb)
4192 goto out;
4193 }
4194
4195 netdev_xmit_skip_txqueue(false);
4196
4197 nf_skip_egress(skb, true);
4198 skb = sch_handle_egress(skb, &rc, dev);
4199 if (!skb)
4200 goto out;
4201 nf_skip_egress(skb, false);
4202
4203 if (netdev_xmit_txqueue_skipped())
4204 txq = netdev_tx_queue_mapping(dev, skb);
4205 }
4206 #endif
4207 /* If device/qdisc don't need skb->dst, release it right now while
4208 * its hot in this cpu cache.
4209 */
4210 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
4211 skb_dst_drop(skb);
4212 else
4213 skb_dst_force(skb);
4214
4215 if (!txq)
4216 txq = netdev_core_pick_tx(dev, skb, sb_dev);
4217
4218 q = rcu_dereference_bh(txq->qdisc);
4219
4220 trace_net_dev_queue(skb);
4221 if (q->enqueue) {
4222 rc = __dev_xmit_skb(skb, q, dev, txq);
4223 goto out;
4224 }
4225
4226 /* The device has no queue. Common case for software devices:
4227 * loopback, all the sorts of tunnels...
4228
4229 * Really, it is unlikely that netif_tx_lock protection is necessary
4230 * here. (f.e. loopback and IP tunnels are clean ignoring statistics
4231 * counters.)
4232 * However, it is possible, that they rely on protection
4233 * made by us here.
4234
4235 * Check this and shot the lock. It is not prone from deadlocks.
4236 *Either shot noqueue qdisc, it is even simpler 8)
4237 */
4238 if (dev->flags & IFF_UP) {
4239 int cpu = smp_processor_id(); /* ok because BHs are off */
4240
4241 /* Other cpus might concurrently change txq->xmit_lock_owner
4242 * to -1 or to their cpu id, but not to our id.
4243 */
4244 if (READ_ONCE(txq->xmit_lock_owner) != cpu) {
4245 if (dev_xmit_recursion())
4246 goto recursion_alert;
4247
4248 skb = validate_xmit_skb(skb, dev, &again);
4249 if (!skb)
4250 goto out;
4251
4252 HARD_TX_LOCK(dev, txq, cpu);
4253
4254 if (!netif_xmit_stopped(txq)) {
4255 dev_xmit_recursion_inc();
4256 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
4257 dev_xmit_recursion_dec();
4258 if (dev_xmit_complete(rc)) {
4259 HARD_TX_UNLOCK(dev, txq);
4260 goto out;
4261 }
4262 }
4263 HARD_TX_UNLOCK(dev, txq);
4264 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
4265 dev->name);
4266 } else {
4267 /* Recursion is detected! It is possible,
4268 * unfortunately
4269 */
4270 recursion_alert:
4271 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
4272 dev->name);
4273 }
4274 }
4275
4276 rc = -ENETDOWN;
4277 rcu_read_unlock_bh();
4278
4279 dev_core_stats_tx_dropped_inc(dev);
4280 kfree_skb_list(skb);
4281 return rc;
4282 out:
4283 rcu_read_unlock_bh();
4284 return rc;
4285 }
4286 EXPORT_SYMBOL(__dev_queue_xmit);
4287
4288 int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
4289 {
4290 struct net_device *dev = skb->dev;
4291 struct sk_buff *orig_skb = skb;
4292 struct netdev_queue *txq;
4293 int ret = NETDEV_TX_BUSY;
4294 bool again = false;
4295
4296 if (unlikely(!netif_running(dev) ||
4297 !netif_carrier_ok(dev)))
4298 goto drop;
4299
4300 skb = validate_xmit_skb_list(skb, dev, &again);
4301 if (skb != orig_skb)
4302 goto drop;
4303
4304 skb_set_queue_mapping(skb, queue_id);
4305 txq = skb_get_tx_queue(dev, skb);
4306
4307 local_bh_disable();
4308
4309 dev_xmit_recursion_inc();
4310 HARD_TX_LOCK(dev, txq, smp_processor_id());
4311 if (!netif_xmit_frozen_or_drv_stopped(txq))
4312 ret = netdev_start_xmit(skb, dev, txq, false);
4313 HARD_TX_UNLOCK(dev, txq);
4314 dev_xmit_recursion_dec();
4315
4316 local_bh_enable();
4317 return ret;
4318 drop:
4319 dev_core_stats_tx_dropped_inc(dev);
4320 kfree_skb_list(skb);
4321 return NET_XMIT_DROP;
4322 }
4323 EXPORT_SYMBOL(__dev_direct_xmit);
4324
4325 /*************************************************************************
4326 * Receiver routines
4327 *************************************************************************/
4328
4329 int netdev_max_backlog __read_mostly = 1000;
4330 EXPORT_SYMBOL(netdev_max_backlog);
4331
4332 int netdev_tstamp_prequeue __read_mostly = 1;
4333 unsigned int sysctl_skb_defer_max __read_mostly = 64;
4334 int netdev_budget __read_mostly = 300;
4335 /* Must be at least 2 jiffes to guarantee 1 jiffy timeout */
4336 unsigned int __read_mostly netdev_budget_usecs = 2 * USEC_PER_SEC / HZ;
4337 int weight_p __read_mostly = 64; /* old backlog weight */
4338 int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */
4339 int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */
4340 int dev_rx_weight __read_mostly = 64;
4341 int dev_tx_weight __read_mostly = 64;
4342
4343 /* Called with irq disabled */
4344 static inline void ____napi_schedule(struct softnet_data *sd,
4345 struct napi_struct *napi)
4346 {
4347 struct task_struct *thread;
4348
4349 lockdep_assert_irqs_disabled();
4350
4351 if (test_bit(NAPI_STATE_THREADED, &napi->state)) {
4352 /* Paired with smp_mb__before_atomic() in
4353 * napi_enable()/dev_set_threaded().
4354 * Use READ_ONCE() to guarantee a complete
4355 * read on napi->thread. Only call
4356 * wake_up_process() when it's not NULL.
4357 */
4358 thread = READ_ONCE(napi->thread);
4359 if (thread) {
4360 /* Avoid doing set_bit() if the thread is in
4361 * INTERRUPTIBLE state, cause napi_thread_wait()
4362 * makes sure to proceed with napi polling
4363 * if the thread is explicitly woken from here.
4364 */
4365 if (READ_ONCE(thread->__state) != TASK_INTERRUPTIBLE)
4366 set_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
4367 wake_up_process(thread);
4368 return;
4369 }
4370 }
4371
4372 list_add_tail(&napi->poll_list, &sd->poll_list);
4373 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4374 }
4375
4376 #ifdef CONFIG_RPS
4377
4378 /* One global table that all flow-based protocols share. */
4379 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
4380 EXPORT_SYMBOL(rps_sock_flow_table);
4381 u32 rps_cpu_mask __read_mostly;
4382 EXPORT_SYMBOL(rps_cpu_mask);
4383
4384 struct static_key_false rps_needed __read_mostly;
4385 EXPORT_SYMBOL(rps_needed);
4386 struct static_key_false rfs_needed __read_mostly;
4387 EXPORT_SYMBOL(rfs_needed);
4388
4389 static struct rps_dev_flow *
4390 set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4391 struct rps_dev_flow *rflow, u16 next_cpu)
4392 {
4393 if (next_cpu < nr_cpu_ids) {
4394 #ifdef CONFIG_RFS_ACCEL
4395 struct netdev_rx_queue *rxqueue;
4396 struct rps_dev_flow_table *flow_table;
4397 struct rps_dev_flow *old_rflow;
4398 u32 flow_id;
4399 u16 rxq_index;
4400 int rc;
4401
4402 /* Should we steer this flow to a different hardware queue? */
4403 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
4404 !(dev->features & NETIF_F_NTUPLE))
4405 goto out;
4406 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
4407 if (rxq_index == skb_get_rx_queue(skb))
4408 goto out;
4409
4410 rxqueue = dev->_rx + rxq_index;
4411 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4412 if (!flow_table)
4413 goto out;
4414 flow_id = skb_get_hash(skb) & flow_table->mask;
4415 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
4416 rxq_index, flow_id);
4417 if (rc < 0)
4418 goto out;
4419 old_rflow = rflow;
4420 rflow = &flow_table->flows[flow_id];
4421 rflow->filter = rc;
4422 if (old_rflow->filter == rflow->filter)
4423 old_rflow->filter = RPS_NO_FILTER;
4424 out:
4425 #endif
4426 rflow->last_qtail =
4427 per_cpu(softnet_data, next_cpu).input_queue_head;
4428 }
4429
4430 rflow->cpu = next_cpu;
4431 return rflow;
4432 }
4433
4434 /*
4435 * get_rps_cpu is called from netif_receive_skb and returns the target
4436 * CPU from the RPS map of the receiving queue for a given skb.
4437 * rcu_read_lock must be held on entry.
4438 */
4439 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4440 struct rps_dev_flow **rflowp)
4441 {
4442 const struct rps_sock_flow_table *sock_flow_table;
4443 struct netdev_rx_queue *rxqueue = dev->_rx;
4444 struct rps_dev_flow_table *flow_table;
4445 struct rps_map *map;
4446 int cpu = -1;
4447 u32 tcpu;
4448 u32 hash;
4449
4450 if (skb_rx_queue_recorded(skb)) {
4451 u16 index = skb_get_rx_queue(skb);
4452
4453 if (unlikely(index >= dev->real_num_rx_queues)) {
4454 WARN_ONCE(dev->real_num_rx_queues > 1,
4455 "%s received packet on queue %u, but number "
4456 "of RX queues is %u\n",
4457 dev->name, index, dev->real_num_rx_queues);
4458 goto done;
4459 }
4460 rxqueue += index;
4461 }
4462
4463 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
4464
4465 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4466 map = rcu_dereference(rxqueue->rps_map);
4467 if (!flow_table && !map)
4468 goto done;
4469
4470 skb_reset_network_header(skb);
4471 hash = skb_get_hash(skb);
4472 if (!hash)
4473 goto done;
4474
4475 sock_flow_table = rcu_dereference(rps_sock_flow_table);
4476 if (flow_table && sock_flow_table) {
4477 struct rps_dev_flow *rflow;
4478 u32 next_cpu;
4479 u32 ident;
4480
4481 /* First check into global flow table if there is a match */
4482 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
4483 if ((ident ^ hash) & ~rps_cpu_mask)
4484 goto try_rps;
4485
4486 next_cpu = ident & rps_cpu_mask;
4487
4488 /* OK, now we know there is a match,
4489 * we can look at the local (per receive queue) flow table
4490 */
4491 rflow = &flow_table->flows[hash & flow_table->mask];
4492 tcpu = rflow->cpu;
4493
4494 /*
4495 * If the desired CPU (where last recvmsg was done) is
4496 * different from current CPU (one in the rx-queue flow
4497 * table entry), switch if one of the following holds:
4498 * - Current CPU is unset (>= nr_cpu_ids).
4499 * - Current CPU is offline.
4500 * - The current CPU's queue tail has advanced beyond the
4501 * last packet that was enqueued using this table entry.
4502 * This guarantees that all previous packets for the flow
4503 * have been dequeued, thus preserving in order delivery.
4504 */
4505 if (unlikely(tcpu != next_cpu) &&
4506 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
4507 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
4508 rflow->last_qtail)) >= 0)) {
4509 tcpu = next_cpu;
4510 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
4511 }
4512
4513 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
4514 *rflowp = rflow;
4515 cpu = tcpu;
4516 goto done;
4517 }
4518 }
4519
4520 try_rps:
4521
4522 if (map) {
4523 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
4524 if (cpu_online(tcpu)) {
4525 cpu = tcpu;
4526 goto done;
4527 }
4528 }
4529
4530 done:
4531 return cpu;
4532 }
4533
4534 #ifdef CONFIG_RFS_ACCEL
4535
4536 /**
4537 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
4538 * @dev: Device on which the filter was set
4539 * @rxq_index: RX queue index
4540 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
4541 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
4542 *
4543 * Drivers that implement ndo_rx_flow_steer() should periodically call
4544 * this function for each installed filter and remove the filters for
4545 * which it returns %true.
4546 */
4547 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
4548 u32 flow_id, u16 filter_id)
4549 {
4550 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
4551 struct rps_dev_flow_table *flow_table;
4552 struct rps_dev_flow *rflow;
4553 bool expire = true;
4554 unsigned int cpu;
4555
4556 rcu_read_lock();
4557 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4558 if (flow_table && flow_id <= flow_table->mask) {
4559 rflow = &flow_table->flows[flow_id];
4560 cpu = READ_ONCE(rflow->cpu);
4561 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
4562 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
4563 rflow->last_qtail) <
4564 (int)(10 * flow_table->mask)))
4565 expire = false;
4566 }
4567 rcu_read_unlock();
4568 return expire;
4569 }
4570 EXPORT_SYMBOL(rps_may_expire_flow);
4571
4572 #endif /* CONFIG_RFS_ACCEL */
4573
4574 /* Called from hardirq (IPI) context */
4575 static void rps_trigger_softirq(void *data)
4576 {
4577 struct softnet_data *sd = data;
4578
4579 ____napi_schedule(sd, &sd->backlog);
4580 sd->received_rps++;
4581 }
4582
4583 #endif /* CONFIG_RPS */
4584
4585 /* Called from hardirq (IPI) context */
4586 static void trigger_rx_softirq(void *data)
4587 {
4588 struct softnet_data *sd = data;
4589
4590 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4591 smp_store_release(&sd->defer_ipi_scheduled, 0);
4592 }
4593
4594 /*
4595 * Check if this softnet_data structure is another cpu one
4596 * If yes, queue it to our IPI list and return 1
4597 * If no, return 0
4598 */
4599 static int napi_schedule_rps(struct softnet_data *sd)
4600 {
4601 struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
4602
4603 #ifdef CONFIG_RPS
4604 if (sd != mysd) {
4605 sd->rps_ipi_next = mysd->rps_ipi_list;
4606 mysd->rps_ipi_list = sd;
4607
4608 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4609 return 1;
4610 }
4611 #endif /* CONFIG_RPS */
4612 __napi_schedule_irqoff(&mysd->backlog);
4613 return 0;
4614 }
4615
4616 #ifdef CONFIG_NET_FLOW_LIMIT
4617 int netdev_flow_limit_table_len __read_mostly = (1 << 12);
4618 #endif
4619
4620 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
4621 {
4622 #ifdef CONFIG_NET_FLOW_LIMIT
4623 struct sd_flow_limit *fl;
4624 struct softnet_data *sd;
4625 unsigned int old_flow, new_flow;
4626
4627 if (qlen < (READ_ONCE(netdev_max_backlog) >> 1))
4628 return false;
4629
4630 sd = this_cpu_ptr(&softnet_data);
4631
4632 rcu_read_lock();
4633 fl = rcu_dereference(sd->flow_limit);
4634 if (fl) {
4635 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
4636 old_flow = fl->history[fl->history_head];
4637 fl->history[fl->history_head] = new_flow;
4638
4639 fl->history_head++;
4640 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
4641
4642 if (likely(fl->buckets[old_flow]))
4643 fl->buckets[old_flow]--;
4644
4645 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
4646 fl->count++;
4647 rcu_read_unlock();
4648 return true;
4649 }
4650 }
4651 rcu_read_unlock();
4652 #endif
4653 return false;
4654 }
4655
4656 /*
4657 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
4658 * queue (may be a remote CPU queue).
4659 */
4660 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
4661 unsigned int *qtail)
4662 {
4663 enum skb_drop_reason reason;
4664 struct softnet_data *sd;
4665 unsigned long flags;
4666 unsigned int qlen;
4667
4668 reason = SKB_DROP_REASON_NOT_SPECIFIED;
4669 sd = &per_cpu(softnet_data, cpu);
4670
4671 rps_lock_irqsave(sd, &flags);
4672 if (!netif_running(skb->dev))
4673 goto drop;
4674 qlen = skb_queue_len(&sd->input_pkt_queue);
4675 if (qlen <= READ_ONCE(netdev_max_backlog) && !skb_flow_limit(skb, qlen)) {
4676 if (qlen) {
4677 enqueue:
4678 __skb_queue_tail(&sd->input_pkt_queue, skb);
4679 input_queue_tail_incr_save(sd, qtail);
4680 rps_unlock_irq_restore(sd, &flags);
4681 return NET_RX_SUCCESS;
4682 }
4683
4684 /* Schedule NAPI for backlog device
4685 * We can use non atomic operation since we own the queue lock
4686 */
4687 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state))
4688 napi_schedule_rps(sd);
4689 goto enqueue;
4690 }
4691 reason = SKB_DROP_REASON_CPU_BACKLOG;
4692
4693 drop:
4694 sd->dropped++;
4695 rps_unlock_irq_restore(sd, &flags);
4696
4697 dev_core_stats_rx_dropped_inc(skb->dev);
4698 kfree_skb_reason(skb, reason);
4699 return NET_RX_DROP;
4700 }
4701
4702 static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb)
4703 {
4704 struct net_device *dev = skb->dev;
4705 struct netdev_rx_queue *rxqueue;
4706
4707 rxqueue = dev->_rx;
4708
4709 if (skb_rx_queue_recorded(skb)) {
4710 u16 index = skb_get_rx_queue(skb);
4711
4712 if (unlikely(index >= dev->real_num_rx_queues)) {
4713 WARN_ONCE(dev->real_num_rx_queues > 1,
4714 "%s received packet on queue %u, but number "
4715 "of RX queues is %u\n",
4716 dev->name, index, dev->real_num_rx_queues);
4717
4718 return rxqueue; /* Return first rxqueue */
4719 }
4720 rxqueue += index;
4721 }
4722 return rxqueue;
4723 }
4724
4725 u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
4726 struct bpf_prog *xdp_prog)
4727 {
4728 void *orig_data, *orig_data_end, *hard_start;
4729 struct netdev_rx_queue *rxqueue;
4730 bool orig_bcast, orig_host;
4731 u32 mac_len, frame_sz;
4732 __be16 orig_eth_type;
4733 struct ethhdr *eth;
4734 u32 metalen, act;
4735 int off;
4736
4737 /* The XDP program wants to see the packet starting at the MAC
4738 * header.
4739 */
4740 mac_len = skb->data - skb_mac_header(skb);
4741 hard_start = skb->data - skb_headroom(skb);
4742
4743 /* SKB "head" area always have tailroom for skb_shared_info */
4744 frame_sz = (void *)skb_end_pointer(skb) - hard_start;
4745 frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4746
4747 rxqueue = netif_get_rxqueue(skb);
4748 xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq);
4749 xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len,
4750 skb_headlen(skb) + mac_len, true);
4751
4752 orig_data_end = xdp->data_end;
4753 orig_data = xdp->data;
4754 eth = (struct ethhdr *)xdp->data;
4755 orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr);
4756 orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest);
4757 orig_eth_type = eth->h_proto;
4758
4759 act = bpf_prog_run_xdp(xdp_prog, xdp);
4760
4761 /* check if bpf_xdp_adjust_head was used */
4762 off = xdp->data - orig_data;
4763 if (off) {
4764 if (off > 0)
4765 __skb_pull(skb, off);
4766 else if (off < 0)
4767 __skb_push(skb, -off);
4768
4769 skb->mac_header += off;
4770 skb_reset_network_header(skb);
4771 }
4772
4773 /* check if bpf_xdp_adjust_tail was used */
4774 off = xdp->data_end - orig_data_end;
4775 if (off != 0) {
4776 skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
4777 skb->len += off; /* positive on grow, negative on shrink */
4778 }
4779
4780 /* check if XDP changed eth hdr such SKB needs update */
4781 eth = (struct ethhdr *)xdp->data;
4782 if ((orig_eth_type != eth->h_proto) ||
4783 (orig_host != ether_addr_equal_64bits(eth->h_dest,
4784 skb->dev->dev_addr)) ||
4785 (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) {
4786 __skb_push(skb, ETH_HLEN);
4787 skb->pkt_type = PACKET_HOST;
4788 skb->protocol = eth_type_trans(skb, skb->dev);
4789 }
4790
4791 /* Redirect/Tx gives L2 packet, code that will reuse skb must __skb_pull
4792 * before calling us again on redirect path. We do not call do_redirect
4793 * as we leave that up to the caller.
4794 *
4795 * Caller is responsible for managing lifetime of skb (i.e. calling
4796 * kfree_skb in response to actions it cannot handle/XDP_DROP).
4797 */
4798 switch (act) {
4799 case XDP_REDIRECT:
4800 case XDP_TX:
4801 __skb_push(skb, mac_len);
4802 break;
4803 case XDP_PASS:
4804 metalen = xdp->data - xdp->data_meta;
4805 if (metalen)
4806 skb_metadata_set(skb, metalen);
4807 break;
4808 }
4809
4810 return act;
4811 }
4812
4813 static u32 netif_receive_generic_xdp(struct sk_buff *skb,
4814 struct xdp_buff *xdp,
4815 struct bpf_prog *xdp_prog)
4816 {
4817 u32 act = XDP_DROP;
4818
4819 /* Reinjected packets coming from act_mirred or similar should
4820 * not get XDP generic processing.
4821 */
4822 if (skb_is_redirected(skb))
4823 return XDP_PASS;
4824
4825 /* XDP packets must be linear and must have sufficient headroom
4826 * of XDP_PACKET_HEADROOM bytes. This is the guarantee that also
4827 * native XDP provides, thus we need to do it here as well.
4828 */
4829 if (skb_cloned(skb) || skb_is_nonlinear(skb) ||
4830 skb_headroom(skb) < XDP_PACKET_HEADROOM) {
4831 int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
4832 int troom = skb->tail + skb->data_len - skb->end;
4833
4834 /* In case we have to go down the path and also linearize,
4835 * then lets do the pskb_expand_head() work just once here.
4836 */
4837 if (pskb_expand_head(skb,
4838 hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0,
4839 troom > 0 ? troom + 128 : 0, GFP_ATOMIC))
4840 goto do_drop;
4841 if (skb_linearize(skb))
4842 goto do_drop;
4843 }
4844
4845 act = bpf_prog_run_generic_xdp(skb, xdp, xdp_prog);
4846 switch (act) {
4847 case XDP_REDIRECT:
4848 case XDP_TX:
4849 case XDP_PASS:
4850 break;
4851 default:
4852 bpf_warn_invalid_xdp_action(skb->dev, xdp_prog, act);
4853 fallthrough;
4854 case XDP_ABORTED:
4855 trace_xdp_exception(skb->dev, xdp_prog, act);
4856 fallthrough;
4857 case XDP_DROP:
4858 do_drop:
4859 kfree_skb(skb);
4860 break;
4861 }
4862
4863 return act;
4864 }
4865
4866 /* When doing generic XDP we have to bypass the qdisc layer and the
4867 * network taps in order to match in-driver-XDP behavior. This also means
4868 * that XDP packets are able to starve other packets going through a qdisc,
4869 * and DDOS attacks will be more effective. In-driver-XDP use dedicated TX
4870 * queues, so they do not have this starvation issue.
4871 */
4872 void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
4873 {
4874 struct net_device *dev = skb->dev;
4875 struct netdev_queue *txq;
4876 bool free_skb = true;
4877 int cpu, rc;
4878
4879 txq = netdev_core_pick_tx(dev, skb, NULL);
4880 cpu = smp_processor_id();
4881 HARD_TX_LOCK(dev, txq, cpu);
4882 if (!netif_xmit_frozen_or_drv_stopped(txq)) {
4883 rc = netdev_start_xmit(skb, dev, txq, 0);
4884 if (dev_xmit_complete(rc))
4885 free_skb = false;
4886 }
4887 HARD_TX_UNLOCK(dev, txq);
4888 if (free_skb) {
4889 trace_xdp_exception(dev, xdp_prog, XDP_TX);
4890 dev_core_stats_tx_dropped_inc(dev);
4891 kfree_skb(skb);
4892 }
4893 }
4894
4895 static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
4896
4897 int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
4898 {
4899 if (xdp_prog) {
4900 struct xdp_buff xdp;
4901 u32 act;
4902 int err;
4903
4904 act = netif_receive_generic_xdp(skb, &xdp, xdp_prog);
4905 if (act != XDP_PASS) {
4906 switch (act) {
4907 case XDP_REDIRECT:
4908 err = xdp_do_generic_redirect(skb->dev, skb,
4909 &xdp, xdp_prog);
4910 if (err)
4911 goto out_redir;
4912 break;
4913 case XDP_TX:
4914 generic_xdp_tx(skb, xdp_prog);
4915 break;
4916 }
4917 return XDP_DROP;
4918 }
4919 }
4920 return XDP_PASS;
4921 out_redir:
4922 kfree_skb_reason(skb, SKB_DROP_REASON_XDP);
4923 return XDP_DROP;
4924 }
4925 EXPORT_SYMBOL_GPL(do_xdp_generic);
4926
4927 static int netif_rx_internal(struct sk_buff *skb)
4928 {
4929 int ret;
4930
4931 net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);
4932
4933 trace_netif_rx(skb);
4934
4935 #ifdef CONFIG_RPS
4936 if (static_branch_unlikely(&rps_needed)) {
4937 struct rps_dev_flow voidflow, *rflow = &voidflow;
4938 int cpu;
4939
4940 rcu_read_lock();
4941
4942 cpu = get_rps_cpu(skb->dev, skb, &rflow);
4943 if (cpu < 0)
4944 cpu = smp_processor_id();
4945
4946 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4947
4948 rcu_read_unlock();
4949 } else
4950 #endif
4951 {
4952 unsigned int qtail;
4953
4954 ret = enqueue_to_backlog(skb, smp_processor_id(), &qtail);
4955 }
4956 return ret;
4957 }
4958
4959 /**
4960 * __netif_rx - Slightly optimized version of netif_rx
4961 * @skb: buffer to post
4962 *
4963 * This behaves as netif_rx except that it does not disable bottom halves.
4964 * As a result this function may only be invoked from the interrupt context
4965 * (either hard or soft interrupt).
4966 */
4967 int __netif_rx(struct sk_buff *skb)
4968 {
4969 int ret;
4970
4971 lockdep_assert_once(hardirq_count() | softirq_count());
4972
4973 trace_netif_rx_entry(skb);
4974 ret = netif_rx_internal(skb);
4975 trace_netif_rx_exit(ret);
4976 return ret;
4977 }
4978 EXPORT_SYMBOL(__netif_rx);
4979
4980 /**
4981 * netif_rx - post buffer to the network code
4982 * @skb: buffer to post
4983 *
4984 * This function receives a packet from a device driver and queues it for
4985 * the upper (protocol) levels to process via the backlog NAPI device. It
4986 * always succeeds. The buffer may be dropped during processing for
4987 * congestion control or by the protocol layers.
4988 * The network buffer is passed via the backlog NAPI device. Modern NIC
4989 * driver should use NAPI and GRO.
4990 * This function can used from interrupt and from process context. The
4991 * caller from process context must not disable interrupts before invoking
4992 * this function.
4993 *
4994 * return values:
4995 * NET_RX_SUCCESS (no congestion)
4996 * NET_RX_DROP (packet was dropped)
4997 *
4998 */
4999 int netif_rx(struct sk_buff *skb)
5000 {
5001 bool need_bh_off = !(hardirq_count() | softirq_count());
5002 int ret;
5003
5004 if (need_bh_off)
5005 local_bh_disable();
5006 trace_netif_rx_entry(skb);
5007 ret = netif_rx_internal(skb);
5008 trace_netif_rx_exit(ret);
5009 if (need_bh_off)
5010 local_bh_enable();
5011 return ret;
5012 }
5013 EXPORT_SYMBOL(netif_rx);
5014
5015 static __latent_entropy void net_tx_action(struct softirq_action *h)
5016 {
5017 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
5018
5019 if (sd->completion_queue) {
5020 struct sk_buff *clist;
5021
5022 local_irq_disable();
5023 clist = sd->completion_queue;
5024 sd->completion_queue = NULL;
5025 local_irq_enable();
5026
5027 while (clist) {
5028 struct sk_buff *skb = clist;
5029
5030 clist = clist->next;
5031
5032 WARN_ON(refcount_read(&skb->users));
5033 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
5034 trace_consume_skb(skb);
5035 else
5036 trace_kfree_skb(skb, net_tx_action,
5037 SKB_DROP_REASON_NOT_SPECIFIED);
5038
5039 if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
5040 __kfree_skb(skb);
5041 else
5042 __kfree_skb_defer(skb);
5043 }
5044 }
5045
5046 if (sd->output_queue) {
5047 struct Qdisc *head;
5048
5049 local_irq_disable();
5050 head = sd->output_queue;
5051 sd->output_queue = NULL;
5052 sd->output_queue_tailp = &sd->output_queue;
5053 local_irq_enable();
5054
5055 rcu_read_lock();
5056
5057 while (head) {
5058 struct Qdisc *q = head;
5059 spinlock_t *root_lock = NULL;
5060
5061 head = head->next_sched;
5062
5063 /* We need to make sure head->next_sched is read
5064 * before clearing __QDISC_STATE_SCHED
5065 */
5066 smp_mb__before_atomic();
5067
5068 if (!(q->flags & TCQ_F_NOLOCK)) {
5069 root_lock = qdisc_lock(q);
5070 spin_lock(root_lock);
5071 } else if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED,
5072 &q->state))) {
5073 /* There is a synchronize_net() between
5074 * STATE_DEACTIVATED flag being set and
5075 * qdisc_reset()/some_qdisc_is_busy() in
5076 * dev_deactivate(), so we can safely bail out
5077 * early here to avoid data race between
5078 * qdisc_deactivate() and some_qdisc_is_busy()
5079 * for lockless qdisc.
5080 */
5081 clear_bit(__QDISC_STATE_SCHED, &q->state);
5082 continue;
5083 }
5084
5085 clear_bit(__QDISC_STATE_SCHED, &q->state);
5086 qdisc_run(q);
5087 if (root_lock)
5088 spin_unlock(root_lock);
5089 }
5090
5091 rcu_read_unlock();
5092 }
5093
5094 xfrm_dev_backlog(sd);
5095 }
5096
5097 #if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
5098 /* This hook is defined here for ATM LANE */
5099 int (*br_fdb_test_addr_hook)(struct net_device *dev,
5100 unsigned char *addr) __read_mostly;
5101 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
5102 #endif
5103
5104 static inline struct sk_buff *
5105 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
5106 struct net_device *orig_dev, bool *another)
5107 {
5108 #ifdef CONFIG_NET_CLS_ACT
5109 struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress);
5110 struct tcf_result cl_res;
5111
5112 /* If there's at least one ingress present somewhere (so
5113 * we get here via enabled static key), remaining devices
5114 * that are not configured with an ingress qdisc will bail
5115 * out here.
5116 */
5117 if (!miniq)
5118 return skb;
5119
5120 if (*pt_prev) {
5121 *ret = deliver_skb(skb, *pt_prev, orig_dev);
5122 *pt_prev = NULL;
5123 }
5124
5125 qdisc_skb_cb(skb)->pkt_len = skb->len;
5126 tc_skb_cb(skb)->mru = 0;
5127 tc_skb_cb(skb)->post_ct = false;
5128 skb->tc_at_ingress = 1;
5129 mini_qdisc_bstats_cpu_update(miniq, skb);
5130
5131 switch (tcf_classify(skb, miniq->block, miniq->filter_list, &cl_res, false)) {
5132 case TC_ACT_OK:
5133 case TC_ACT_RECLASSIFY:
5134 skb->tc_index = TC_H_MIN(cl_res.classid);
5135 break;
5136 case TC_ACT_SHOT:
5137 mini_qdisc_qstats_cpu_drop(miniq);
5138 kfree_skb_reason(skb, SKB_DROP_REASON_TC_INGRESS);
5139 *ret = NET_RX_DROP;
5140 return NULL;
5141 case TC_ACT_STOLEN:
5142 case TC_ACT_QUEUED:
5143 case TC_ACT_TRAP:
5144 consume_skb(skb);
5145 *ret = NET_RX_SUCCESS;
5146 return NULL;
5147 case TC_ACT_REDIRECT:
5148 /* skb_mac_header check was done by cls/act_bpf, so
5149 * we can safely push the L2 header back before
5150 * redirecting to another netdev
5151 */
5152 __skb_push(skb, skb->mac_len);
5153 if (skb_do_redirect(skb) == -EAGAIN) {
5154 __skb_pull(skb, skb->mac_len);
5155 *another = true;
5156 break;
5157 }
5158 *ret = NET_RX_SUCCESS;
5159 return NULL;
5160 case TC_ACT_CONSUMED:
5161 *ret = NET_RX_SUCCESS;
5162 return NULL;
5163 default:
5164 break;
5165 }
5166 #endif /* CONFIG_NET_CLS_ACT */
5167 return skb;
5168 }
5169
5170 /**
5171 * netdev_is_rx_handler_busy - check if receive handler is registered
5172 * @dev: device to check
5173 *
5174 * Check if a receive handler is already registered for a given device.
5175 * Return true if there one.
5176 *
5177 * The caller must hold the rtnl_mutex.
5178 */
5179 bool netdev_is_rx_handler_busy(struct net_device *dev)
5180 {
5181 ASSERT_RTNL();
5182 return dev && rtnl_dereference(dev->rx_handler);
5183 }
5184 EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
5185
5186 /**
5187 * netdev_rx_handler_register - register receive handler
5188 * @dev: device to register a handler for
5189 * @rx_handler: receive handler to register
5190 * @rx_handler_data: data pointer that is used by rx handler
5191 *
5192 * Register a receive handler for a device. This handler will then be
5193 * called from __netif_receive_skb. A negative errno code is returned
5194 * on a failure.
5195 *
5196 * The caller must hold the rtnl_mutex.
5197 *
5198 * For a general description of rx_handler, see enum rx_handler_result.
5199 */
5200 int netdev_rx_handler_register(struct net_device *dev,
5201 rx_handler_func_t *rx_handler,
5202 void *rx_handler_data)
5203 {
5204 if (netdev_is_rx_handler_busy(dev))
5205 return -EBUSY;
5206
5207 if (dev->priv_flags & IFF_NO_RX_HANDLER)
5208 return -EINVAL;
5209
5210 /* Note: rx_handler_data must be set before rx_handler */
5211 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
5212 rcu_assign_pointer(dev->rx_handler, rx_handler);
5213
5214 return 0;
5215 }
5216 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
5217
5218 /**
5219 * netdev_rx_handler_unregister - unregister receive handler
5220 * @dev: device to unregister a handler from
5221 *
5222 * Unregister a receive handler from a device.
5223 *
5224 * The caller must hold the rtnl_mutex.
5225 */
5226 void netdev_rx_handler_unregister(struct net_device *dev)
5227 {
5228
5229 ASSERT_RTNL();
5230 RCU_INIT_POINTER(dev->rx_handler, NULL);
5231 /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
5232 * section has a guarantee to see a non NULL rx_handler_data
5233 * as well.
5234 */
5235 synchronize_net();
5236 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
5237 }
5238 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
5239
5240 /*
5241 * Limit the use of PFMEMALLOC reserves to those protocols that implement
5242 * the special handling of PFMEMALLOC skbs.
5243 */
5244 static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
5245 {
5246 switch (skb->protocol) {
5247 case htons(ETH_P_ARP):
5248 case htons(ETH_P_IP):
5249 case htons(ETH_P_IPV6):
5250 case htons(ETH_P_8021Q):
5251 case htons(ETH_P_8021AD):
5252 return true;
5253 default:
5254 return false;
5255 }
5256 }
5257
5258 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
5259 int *ret, struct net_device *orig_dev)
5260 {
5261 if (nf_hook_ingress_active(skb)) {
5262 int ingress_retval;
5263
5264 if (*pt_prev) {
5265 *ret = deliver_skb(skb, *pt_prev, orig_dev);
5266 *pt_prev = NULL;
5267 }
5268
5269 rcu_read_lock();
5270 ingress_retval = nf_hook_ingress(skb);
5271 rcu_read_unlock();
5272 return ingress_retval;
5273 }
5274 return 0;
5275 }
5276
5277 static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,
5278 struct packet_type **ppt_prev)
5279 {
5280 struct packet_type *ptype, *pt_prev;
5281 rx_handler_func_t *rx_handler;
5282 struct sk_buff *skb = *pskb;
5283 struct net_device *orig_dev;
5284 bool deliver_exact = false;
5285 int ret = NET_RX_DROP;
5286 __be16 type;
5287
5288 net_timestamp_check(!READ_ONCE(netdev_tstamp_prequeue), skb);
5289
5290 trace_netif_receive_skb(skb);
5291
5292 orig_dev = skb->dev;
5293
5294 skb_reset_network_header(skb);
5295 if (!skb_transport_header_was_set(skb))
5296 skb_reset_transport_header(skb);
5297 skb_reset_mac_len(skb);
5298
5299 pt_prev = NULL;
5300
5301 another_round:
5302 skb->skb_iif = skb->dev->ifindex;
5303
5304 __this_cpu_inc(softnet_data.processed);
5305
5306 if (static_branch_unlikely(&generic_xdp_needed_key)) {
5307 int ret2;
5308
5309 migrate_disable();
5310 ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
5311 migrate_enable();
5312
5313 if (ret2 != XDP_PASS) {
5314 ret = NET_RX_DROP;
5315 goto out;
5316 }
5317 }
5318
5319 if (eth_type_vlan(skb->protocol)) {
5320 skb = skb_vlan_untag(skb);
5321 if (unlikely(!skb))
5322 goto out;
5323 }
5324
5325 if (skb_skip_tc_classify(skb))
5326 goto skip_classify;
5327
5328 if (pfmemalloc)
5329 goto skip_taps;
5330
5331 list_for_each_entry_rcu(ptype, &ptype_all, list) {
5332 if (pt_prev)
5333 ret = deliver_skb(skb, pt_prev, orig_dev);
5334 pt_prev = ptype;
5335 }
5336
5337 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
5338 if (pt_prev)
5339 ret = deliver_skb(skb, pt_prev, orig_dev);
5340 pt_prev = ptype;
5341 }
5342
5343 skip_taps:
5344 #ifdef CONFIG_NET_INGRESS
5345 if (static_branch_unlikely(&ingress_needed_key)) {
5346 bool another = false;
5347
5348 nf_skip_egress(skb, true);
5349 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev,
5350 &another);
5351 if (another)
5352 goto another_round;
5353 if (!skb)
5354 goto out;
5355
5356 nf_skip_egress(skb, false);
5357 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
5358 goto out;
5359 }
5360 #endif
5361 skb_reset_redirect(skb);
5362 skip_classify:
5363 if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
5364 goto drop;
5365
5366 if (skb_vlan_tag_present(skb)) {
5367 if (pt_prev) {
5368 ret = deliver_skb(skb, pt_prev, orig_dev);
5369 pt_prev = NULL;
5370 }
5371 if (vlan_do_receive(&skb))
5372 goto another_round;
5373 else if (unlikely(!skb))
5374 goto out;
5375 }
5376
5377 rx_handler = rcu_dereference(skb->dev->rx_handler);
5378 if (rx_handler) {
5379 if (pt_prev) {
5380 ret = deliver_skb(skb, pt_prev, orig_dev);
5381 pt_prev = NULL;
5382 }
5383 switch (rx_handler(&skb)) {
5384 case RX_HANDLER_CONSUMED:
5385 ret = NET_RX_SUCCESS;
5386 goto out;
5387 case RX_HANDLER_ANOTHER:
5388 goto another_round;
5389 case RX_HANDLER_EXACT:
5390 deliver_exact = true;
5391 break;
5392 case RX_HANDLER_PASS:
5393 break;
5394 default:
5395 BUG();
5396 }
5397 }
5398
5399 if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) {
5400 check_vlan_id:
5401 if (skb_vlan_tag_get_id(skb)) {
5402 /* Vlan id is non 0 and vlan_do_receive() above couldn't
5403 * find vlan device.
5404 */
5405 skb->pkt_type = PACKET_OTHERHOST;
5406 } else if (eth_type_vlan(skb->protocol)) {
5407 /* Outer header is 802.1P with vlan 0, inner header is
5408 * 802.1Q or 802.1AD and vlan_do_receive() above could
5409 * not find vlan dev for vlan id 0.
5410 */
5411 __vlan_hwaccel_clear_tag(skb);
5412 skb = skb_vlan_untag(skb);
5413 if (unlikely(!skb))
5414 goto out;
5415 if (vlan_do_receive(&skb))
5416 /* After stripping off 802.1P header with vlan 0
5417 * vlan dev is found for inner header.
5418 */
5419 goto another_round;
5420 else if (unlikely(!skb))
5421 goto out;
5422 else
5423 /* We have stripped outer 802.1P vlan 0 header.
5424 * But could not find vlan dev.
5425 * check again for vlan id to set OTHERHOST.
5426 */
5427 goto check_vlan_id;
5428 }
5429 /* Note: we might in the future use prio bits
5430 * and set skb->priority like in vlan_do_receive()
5431 * For the time being, just ignore Priority Code Point
5432 */
5433 __vlan_hwaccel_clear_tag(skb);
5434 }
5435
5436 type = skb->protocol;
5437
5438 /* deliver only exact match when indicated */
5439 if (likely(!deliver_exact)) {
5440 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5441 &ptype_base[ntohs(type) &
5442 PTYPE_HASH_MASK]);
5443 }
5444
5445 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5446 &orig_dev->ptype_specific);
5447
5448 if (unlikely(skb->dev != orig_dev)) {
5449 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5450 &skb->dev->ptype_specific);
5451 }
5452
5453 if (pt_prev) {
5454 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
5455 goto drop;
5456 *ppt_prev = pt_prev;
5457 } else {
5458 drop:
5459 if (!deliver_exact)
5460 dev_core_stats_rx_dropped_inc(skb->dev);
5461 else
5462 dev_core_stats_rx_nohandler_inc(skb->dev);
5463 kfree_skb_reason(skb, SKB_DROP_REASON_UNHANDLED_PROTO);
5464 /* Jamal, now you will not able to escape explaining
5465 * me how you were going to use this. :-)
5466 */
5467 ret = NET_RX_DROP;
5468 }
5469
5470 out:
5471 /* The invariant here is that if *ppt_prev is not NULL
5472 * then skb should also be non-NULL.
5473 *
5474 * Apparently *ppt_prev assignment above holds this invariant due to
5475 * skb dereferencing near it.
5476 */
5477 *pskb = skb;
5478 return ret;
5479 }
5480
5481 static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc)
5482 {
5483 struct net_device *orig_dev = skb->dev;
5484 struct packet_type *pt_prev = NULL;
5485 int ret;
5486
5487 ret = __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
5488 if (pt_prev)
5489 ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb,
5490 skb->dev, pt_prev, orig_dev);
5491 return ret;
5492 }
5493
5494 /**
5495 * netif_receive_skb_core - special purpose version of netif_receive_skb
5496 * @skb: buffer to process
5497 *
5498 * More direct receive version of netif_receive_skb(). It should
5499 * only be used by callers that have a need to skip RPS and Generic XDP.
5500 * Caller must also take care of handling if ``(page_is_)pfmemalloc``.
5501 *
5502 * This function may only be called from softirq context and interrupts
5503 * should be enabled.
5504 *
5505 * Return values (usually ignored):
5506 * NET_RX_SUCCESS: no congestion
5507 * NET_RX_DROP: packet was dropped
5508 */
5509 int netif_receive_skb_core(struct sk_buff *skb)
5510 {
5511 int ret;
5512
5513 rcu_read_lock();
5514 ret = __netif_receive_skb_one_core(skb, false);
5515 rcu_read_unlock();
5516
5517 return ret;
5518 }
5519 EXPORT_SYMBOL(netif_receive_skb_core);
5520
5521 static inline void __netif_receive_skb_list_ptype(struct list_head *head,
5522 struct packet_type *pt_prev,
5523 struct net_device *orig_dev)
5524 {
5525 struct sk_buff *skb, *next;
5526
5527 if (!pt_prev)
5528 return;
5529 if (list_empty(head))
5530 return;
5531 if (pt_prev->list_func != NULL)
5532 INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv,
5533 ip_list_rcv, head, pt_prev, orig_dev);
5534 else
5535 list_for_each_entry_safe(skb, next, head, list) {
5536 skb_list_del_init(skb);
5537 pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
5538 }
5539 }
5540
5541 static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc)
5542 {
5543 /* Fast-path assumptions:
5544 * - There is no RX handler.
5545 * - Only one packet_type matches.
5546 * If either of these fails, we will end up doing some per-packet
5547 * processing in-line, then handling the 'last ptype' for the whole
5548 * sublist. This can't cause out-of-order delivery to any single ptype,
5549 * because the 'last ptype' must be constant across the sublist, and all
5550 * other ptypes are handled per-packet.
5551 */
5552 /* Current (common) ptype of sublist */
5553 struct packet_type *pt_curr = NULL;
5554 /* Current (common) orig_dev of sublist */
5555 struct net_device *od_curr = NULL;
5556 struct list_head sublist;
5557 struct sk_buff *skb, *next;
5558
5559 INIT_LIST_HEAD(&sublist);
5560 list_for_each_entry_safe(skb, next, head, list) {
5561 struct net_device *orig_dev = skb->dev;
5562 struct packet_type *pt_prev = NULL;
5563
5564 skb_list_del_init(skb);
5565 __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
5566 if (!pt_prev)
5567 continue;
5568 if (pt_curr != pt_prev || od_curr != orig_dev) {
5569 /* dispatch old sublist */
5570 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5571 /* start new sublist */
5572 INIT_LIST_HEAD(&sublist);
5573 pt_curr = pt_prev;
5574 od_curr = orig_dev;
5575 }
5576 list_add_tail(&skb->list, &sublist);
5577 }
5578
5579 /* dispatch final sublist */
5580 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5581 }
5582
5583 static int __netif_receive_skb(struct sk_buff *skb)
5584 {
5585 int ret;
5586
5587 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
5588 unsigned int noreclaim_flag;
5589
5590 /*
5591 * PFMEMALLOC skbs are special, they should
5592 * - be delivered to SOCK_MEMALLOC sockets only
5593 * - stay away from userspace
5594 * - have bounded memory usage
5595 *
5596 * Use PF_MEMALLOC as this saves us from propagating the allocation
5597 * context down to all allocation sites.
5598 */
5599 noreclaim_flag = memalloc_noreclaim_save();
5600 ret = __netif_receive_skb_one_core(skb, true);
5601 memalloc_noreclaim_restore(noreclaim_flag);
5602 } else
5603 ret = __netif_receive_skb_one_core(skb, false);
5604
5605 return ret;
5606 }
5607
5608 static void __netif_receive_skb_list(struct list_head *head)
5609 {
5610 unsigned long noreclaim_flag = 0;
5611 struct sk_buff *skb, *next;
5612 bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */
5613
5614 list_for_each_entry_safe(skb, next, head, list) {
5615 if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) {
5616 struct list_head sublist;
5617
5618 /* Handle the previous sublist */
5619 list_cut_before(&sublist, head, &skb->list);
5620 if (!list_empty(&sublist))
5621 __netif_receive_skb_list_core(&sublist, pfmemalloc);
5622 pfmemalloc = !pfmemalloc;
5623 /* See comments in __netif_receive_skb */
5624 if (pfmemalloc)
5625 noreclaim_flag = memalloc_noreclaim_save();
5626 else
5627 memalloc_noreclaim_restore(noreclaim_flag);
5628 }
5629 }
5630 /* Handle the remaining sublist */
5631 if (!list_empty(head))
5632 __netif_receive_skb_list_core(head, pfmemalloc);
5633 /* Restore pflags */
5634 if (pfmemalloc)
5635 memalloc_noreclaim_restore(noreclaim_flag);
5636 }
5637
5638 static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
5639 {
5640 struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
5641 struct bpf_prog *new = xdp->prog;
5642 int ret = 0;
5643
5644 switch (xdp->command) {
5645 case XDP_SETUP_PROG:
5646 rcu_assign_pointer(dev->xdp_prog, new);
5647 if (old)
5648 bpf_prog_put(old);
5649
5650 if (old && !new) {
5651 static_branch_dec(&generic_xdp_needed_key);
5652 } else if (new && !old) {
5653 static_branch_inc(&generic_xdp_needed_key);
5654 dev_disable_lro(dev);
5655 dev_disable_gro_hw(dev);
5656 }
5657 break;
5658
5659 default:
5660 ret = -EINVAL;
5661 break;
5662 }
5663
5664 return ret;
5665 }
5666
5667 static int netif_receive_skb_internal(struct sk_buff *skb)
5668 {
5669 int ret;
5670
5671 net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);
5672
5673 if (skb_defer_rx_timestamp(skb))
5674 return NET_RX_SUCCESS;
5675
5676 rcu_read_lock();
5677 #ifdef CONFIG_RPS
5678 if (static_branch_unlikely(&rps_needed)) {
5679 struct rps_dev_flow voidflow, *rflow = &voidflow;
5680 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5681
5682 if (cpu >= 0) {
5683 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5684 rcu_read_unlock();
5685 return ret;
5686 }
5687 }
5688 #endif
5689 ret = __netif_receive_skb(skb);
5690 rcu_read_unlock();
5691 return ret;
5692 }
5693
5694 void netif_receive_skb_list_internal(struct list_head *head)
5695 {
5696 struct sk_buff *skb, *next;
5697 struct list_head sublist;
5698
5699 INIT_LIST_HEAD(&sublist);
5700 list_for_each_entry_safe(skb, next, head, list) {
5701 net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);
5702 skb_list_del_init(skb);
5703 if (!skb_defer_rx_timestamp(skb))
5704 list_add_tail(&skb->list, &sublist);
5705 }
5706 list_splice_init(&sublist, head);
5707
5708 rcu_read_lock();
5709 #ifdef CONFIG_RPS
5710 if (static_branch_unlikely(&rps_needed)) {
5711 list_for_each_entry_safe(skb, next, head, list) {
5712 struct rps_dev_flow voidflow, *rflow = &voidflow;
5713 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5714
5715 if (cpu >= 0) {
5716 /* Will be handled, remove from list */
5717 skb_list_del_init(skb);
5718 enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5719 }
5720 }
5721 }
5722 #endif
5723 __netif_receive_skb_list(head);
5724 rcu_read_unlock();
5725 }
5726
5727 /**
5728 * netif_receive_skb - process receive buffer from network
5729 * @skb: buffer to process
5730 *
5731 * netif_receive_skb() is the main receive data processing function.
5732 * It always succeeds. The buffer may be dropped during processing
5733 * for congestion control or by the protocol layers.
5734 *
5735 * This function may only be called from softirq context and interrupts
5736 * should be enabled.
5737 *
5738 * Return values (usually ignored):
5739 * NET_RX_SUCCESS: no congestion
5740 * NET_RX_DROP: packet was dropped
5741 */
5742 int netif_receive_skb(struct sk_buff *skb)
5743 {
5744 int ret;
5745
5746 trace_netif_receive_skb_entry(skb);
5747
5748 ret = netif_receive_skb_internal(skb);
5749 trace_netif_receive_skb_exit(ret);
5750
5751 return ret;
5752 }
5753 EXPORT_SYMBOL(netif_receive_skb);
5754
5755 /**
5756 * netif_receive_skb_list - process many receive buffers from network
5757 * @head: list of skbs to process.
5758 *
5759 * Since return value of netif_receive_skb() is normally ignored, and
5760 * wouldn't be meaningful for a list, this function returns void.
5761 *
5762 * This function may only be called from softirq context and interrupts
5763 * should be enabled.
5764 */
5765 void netif_receive_skb_list(struct list_head *head)
5766 {
5767 struct sk_buff *skb;
5768
5769 if (list_empty(head))
5770 return;
5771 if (trace_netif_receive_skb_list_entry_enabled()) {
5772 list_for_each_entry(skb, head, list)
5773 trace_netif_receive_skb_list_entry(skb);
5774 }
5775 netif_receive_skb_list_internal(head);
5776 trace_netif_receive_skb_list_exit(0);
5777 }
5778 EXPORT_SYMBOL(netif_receive_skb_list);
5779
5780 static DEFINE_PER_CPU(struct work_struct, flush_works);
5781
5782 /* Network device is going away, flush any packets still pending */
5783 static void flush_backlog(struct work_struct *work)
5784 {
5785 struct sk_buff *skb, *tmp;
5786 struct softnet_data *sd;
5787
5788 local_bh_disable();
5789 sd = this_cpu_ptr(&softnet_data);
5790
5791 rps_lock_irq_disable(sd);
5792 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
5793 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
5794 __skb_unlink(skb, &sd->input_pkt_queue);
5795 dev_kfree_skb_irq(skb);
5796 input_queue_head_incr(sd);
5797 }
5798 }
5799 rps_unlock_irq_enable(sd);
5800
5801 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
5802 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
5803 __skb_unlink(skb, &sd->process_queue);
5804 kfree_skb(skb);
5805 input_queue_head_incr(sd);
5806 }
5807 }
5808 local_bh_enable();
5809 }
5810
5811 static bool flush_required(int cpu)
5812 {
5813 #if IS_ENABLED(CONFIG_RPS)
5814 struct softnet_data *sd = &per_cpu(softnet_data, cpu);
5815 bool do_flush;
5816
5817 rps_lock_irq_disable(sd);
5818
5819 /* as insertion into process_queue happens with the rps lock held,
5820 * process_queue access may race only with dequeue
5821 */
5822 do_flush = !skb_queue_empty(&sd->input_pkt_queue) ||
5823 !skb_queue_empty_lockless(&sd->process_queue);
5824 rps_unlock_irq_enable(sd);
5825
5826 return do_flush;
5827 #endif
5828 /* without RPS we can't safely check input_pkt_queue: during a
5829 * concurrent remote skb_queue_splice() we can detect as empty both
5830 * input_pkt_queue and process_queue even if the latter could end-up
5831 * containing a lot of packets.
5832 */
5833 return true;
5834 }
5835
5836 static void flush_all_backlogs(void)
5837 {
5838 static cpumask_t flush_cpus;
5839 unsigned int cpu;
5840
5841 /* since we are under rtnl lock protection we can use static data
5842 * for the cpumask and avoid allocating on stack the possibly
5843 * large mask
5844 */
5845 ASSERT_RTNL();
5846
5847 cpus_read_lock();
5848
5849 cpumask_clear(&flush_cpus);
5850 for_each_online_cpu(cpu) {
5851 if (flush_required(cpu)) {
5852 queue_work_on(cpu, system_highpri_wq,
5853 per_cpu_ptr(&flush_works, cpu));
5854 cpumask_set_cpu(cpu, &flush_cpus);
5855 }
5856 }
5857
5858 /* we can have in flight packet[s] on the cpus we are not flushing,
5859 * synchronize_net() in unregister_netdevice_many() will take care of
5860 * them
5861 */
5862 for_each_cpu(cpu, &flush_cpus)
5863 flush_work(per_cpu_ptr(&flush_works, cpu));
5864
5865 cpus_read_unlock();
5866 }
5867
5868 static void net_rps_send_ipi(struct softnet_data *remsd)
5869 {
5870 #ifdef CONFIG_RPS
5871 while (remsd) {
5872 struct softnet_data *next = remsd->rps_ipi_next;
5873
5874 if (cpu_online(remsd->cpu))
5875 smp_call_function_single_async(remsd->cpu, &remsd->csd);
5876 remsd = next;
5877 }
5878 #endif
5879 }
5880
5881 /*
5882 * net_rps_action_and_irq_enable sends any pending IPI's for rps.
5883 * Note: called with local irq disabled, but exits with local irq enabled.
5884 */
5885 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
5886 {
5887 #ifdef CONFIG_RPS
5888 struct softnet_data *remsd = sd->rps_ipi_list;
5889
5890 if (remsd) {
5891 sd->rps_ipi_list = NULL;
5892
5893 local_irq_enable();
5894
5895 /* Send pending IPI's to kick RPS processing on remote cpus. */
5896 net_rps_send_ipi(remsd);
5897 } else
5898 #endif
5899 local_irq_enable();
5900 }
5901
5902 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
5903 {
5904 #ifdef CONFIG_RPS
5905 return sd->rps_ipi_list != NULL;
5906 #else
5907 return false;
5908 #endif
5909 }
5910
5911 static int process_backlog(struct napi_struct *napi, int quota)
5912 {
5913 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
5914 bool again = true;
5915 int work = 0;
5916
5917 /* Check if we have pending ipi, its better to send them now,
5918 * not waiting net_rx_action() end.
5919 */
5920 if (sd_has_rps_ipi_waiting(sd)) {
5921 local_irq_disable();
5922 net_rps_action_and_irq_enable(sd);
5923 }
5924
5925 napi->weight = READ_ONCE(dev_rx_weight);
5926 while (again) {
5927 struct sk_buff *skb;
5928
5929 while ((skb = __skb_dequeue(&sd->process_queue))) {
5930 rcu_read_lock();
5931 __netif_receive_skb(skb);
5932 rcu_read_unlock();
5933 input_queue_head_incr(sd);
5934 if (++work >= quota)
5935 return work;
5936
5937 }
5938
5939 rps_lock_irq_disable(sd);
5940 if (skb_queue_empty(&sd->input_pkt_queue)) {
5941 /*
5942 * Inline a custom version of __napi_complete().
5943 * only current cpu owns and manipulates this napi,
5944 * and NAPI_STATE_SCHED is the only possible flag set
5945 * on backlog.
5946 * We can use a plain write instead of clear_bit(),
5947 * and we dont need an smp_mb() memory barrier.
5948 */
5949 napi->state = 0;
5950 again = false;
5951 } else {
5952 skb_queue_splice_tail_init(&sd->input_pkt_queue,
5953 &sd->process_queue);
5954 }
5955 rps_unlock_irq_enable(sd);
5956 }
5957
5958 return work;
5959 }
5960
5961 /**
5962 * __napi_schedule - schedule for receive
5963 * @n: entry to schedule
5964 *
5965 * The entry's receive function will be scheduled to run.
5966 * Consider using __napi_schedule_irqoff() if hard irqs are masked.
5967 */
5968 void __napi_schedule(struct napi_struct *n)
5969 {
5970 unsigned long flags;
5971
5972 local_irq_save(flags);
5973 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
5974 local_irq_restore(flags);
5975 }
5976 EXPORT_SYMBOL(__napi_schedule);
5977
5978 /**
5979 * napi_schedule_prep - check if napi can be scheduled
5980 * @n: napi context
5981 *
5982 * Test if NAPI routine is already running, and if not mark
5983 * it as running. This is used as a condition variable to
5984 * insure only one NAPI poll instance runs. We also make
5985 * sure there is no pending NAPI disable.
5986 */
5987 bool napi_schedule_prep(struct napi_struct *n)
5988 {
5989 unsigned long val, new;
5990
5991 do {
5992 val = READ_ONCE(n->state);
5993 if (unlikely(val & NAPIF_STATE_DISABLE))
5994 return false;
5995 new = val | NAPIF_STATE_SCHED;
5996
5997 /* Sets STATE_MISSED bit if STATE_SCHED was already set
5998 * This was suggested by Alexander Duyck, as compiler
5999 * emits better code than :
6000 * if (val & NAPIF_STATE_SCHED)
6001 * new |= NAPIF_STATE_MISSED;
6002 */
6003 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
6004 NAPIF_STATE_MISSED;
6005 } while (cmpxchg(&n->state, val, new) != val);
6006
6007 return !(val & NAPIF_STATE_SCHED);
6008 }
6009 EXPORT_SYMBOL(napi_schedule_prep);
6010
6011 /**
6012 * __napi_schedule_irqoff - schedule for receive
6013 * @n: entry to schedule
6014 *
6015 * Variant of __napi_schedule() assuming hard irqs are masked.
6016 *
6017 * On PREEMPT_RT enabled kernels this maps to __napi_schedule()
6018 * because the interrupt disabled assumption might not be true
6019 * due to force-threaded interrupts and spinlock substitution.
6020 */
6021 void __napi_schedule_irqoff(struct napi_struct *n)
6022 {
6023 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6024 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
6025 else
6026 __napi_schedule(n);
6027 }
6028 EXPORT_SYMBOL(__napi_schedule_irqoff);
6029
6030 bool napi_complete_done(struct napi_struct *n, int work_done)
6031 {
6032 unsigned long flags, val, new, timeout = 0;
6033 bool ret = true;
6034
6035 /*
6036 * 1) Don't let napi dequeue from the cpu poll list
6037 * just in case its running on a different cpu.
6038 * 2) If we are busy polling, do nothing here, we have
6039 * the guarantee we will be called later.
6040 */
6041 if (unlikely(n->state & (NAPIF_STATE_NPSVC |
6042 NAPIF_STATE_IN_BUSY_POLL)))
6043 return false;
6044
6045 if (work_done) {
6046 if (n->gro_bitmask)
6047 timeout = READ_ONCE(n->dev->gro_flush_timeout);
6048 n->defer_hard_irqs_count = READ_ONCE(n->dev->napi_defer_hard_irqs);
6049 }
6050 if (n->defer_hard_irqs_count > 0) {
6051 n->defer_hard_irqs_count--;
6052 timeout = READ_ONCE(n->dev->gro_flush_timeout);
6053 if (timeout)
6054 ret = false;
6055 }
6056 if (n->gro_bitmask) {
6057 /* When the NAPI instance uses a timeout and keeps postponing
6058 * it, we need to bound somehow the time packets are kept in
6059 * the GRO layer
6060 */
6061 napi_gro_flush(n, !!timeout);
6062 }
6063
6064 gro_normal_list(n);
6065
6066 if (unlikely(!list_empty(&n->poll_list))) {
6067 /* If n->poll_list is not empty, we need to mask irqs */
6068 local_irq_save(flags);
6069 list_del_init(&n->poll_list);
6070 local_irq_restore(flags);
6071 }
6072
6073 do {
6074 val = READ_ONCE(n->state);
6075
6076 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
6077
6078 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED |
6079 NAPIF_STATE_SCHED_THREADED |
6080 NAPIF_STATE_PREFER_BUSY_POLL);
6081
6082 /* If STATE_MISSED was set, leave STATE_SCHED set,
6083 * because we will call napi->poll() one more time.
6084 * This C code was suggested by Alexander Duyck to help gcc.
6085 */
6086 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
6087 NAPIF_STATE_SCHED;
6088 } while (cmpxchg(&n->state, val, new) != val);
6089
6090 if (unlikely(val & NAPIF_STATE_MISSED)) {
6091 __napi_schedule(n);
6092 return false;
6093 }
6094
6095 if (timeout)
6096 hrtimer_start(&n->timer, ns_to_ktime(timeout),
6097 HRTIMER_MODE_REL_PINNED);
6098 return ret;
6099 }
6100 EXPORT_SYMBOL(napi_complete_done);
6101
6102 /* must be called under rcu_read_lock(), as we dont take a reference */
6103 static struct napi_struct *napi_by_id(unsigned int napi_id)
6104 {
6105 unsigned int hash = napi_id % HASH_SIZE(napi_hash);
6106 struct napi_struct *napi;
6107
6108 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
6109 if (napi->napi_id == napi_id)
6110 return napi;
6111
6112 return NULL;
6113 }
6114
6115 #if defined(CONFIG_NET_RX_BUSY_POLL)
6116
6117 static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule)
6118 {
6119 if (!skip_schedule) {
6120 gro_normal_list(napi);
6121 __napi_schedule(napi);
6122 return;
6123 }
6124
6125 if (napi->gro_bitmask) {
6126 /* flush too old packets
6127 * If HZ < 1000, flush all packets.
6128 */
6129 napi_gro_flush(napi, HZ >= 1000);
6130 }
6131
6132 gro_normal_list(napi);
6133 clear_bit(NAPI_STATE_SCHED, &napi->state);
6134 }
6135
6136 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, bool prefer_busy_poll,
6137 u16 budget)
6138 {
6139 bool skip_schedule = false;
6140 unsigned long timeout;
6141 int rc;
6142
6143 /* Busy polling means there is a high chance device driver hard irq
6144 * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
6145 * set in napi_schedule_prep().
6146 * Since we are about to call napi->poll() once more, we can safely
6147 * clear NAPI_STATE_MISSED.
6148 *
6149 * Note: x86 could use a single "lock and ..." instruction
6150 * to perform these two clear_bit()
6151 */
6152 clear_bit(NAPI_STATE_MISSED, &napi->state);
6153 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
6154
6155 local_bh_disable();
6156
6157 if (prefer_busy_poll) {
6158 napi->defer_hard_irqs_count = READ_ONCE(napi->dev->napi_defer_hard_irqs);
6159 timeout = READ_ONCE(napi->dev->gro_flush_timeout);
6160 if (napi->defer_hard_irqs_count && timeout) {
6161 hrtimer_start(&napi->timer, ns_to_ktime(timeout), HRTIMER_MODE_REL_PINNED);
6162 skip_schedule = true;
6163 }
6164 }
6165
6166 /* All we really want here is to re-enable device interrupts.
6167 * Ideally, a new ndo_busy_poll_stop() could avoid another round.
6168 */
6169 rc = napi->poll(napi, budget);
6170 /* We can't gro_normal_list() here, because napi->poll() might have
6171 * rearmed the napi (napi_complete_done()) in which case it could
6172 * already be running on another CPU.
6173 */
6174 trace_napi_poll(napi, rc, budget);
6175 netpoll_poll_unlock(have_poll_lock);
6176 if (rc == budget)
6177 __busy_poll_stop(napi, skip_schedule);
6178 local_bh_enable();
6179 }
6180
6181 void napi_busy_loop(unsigned int napi_id,
6182 bool (*loop_end)(void *, unsigned long),
6183 void *loop_end_arg, bool prefer_busy_poll, u16 budget)
6184 {
6185 unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
6186 int (*napi_poll)(struct napi_struct *napi, int budget);
6187 void *have_poll_lock = NULL;
6188 struct napi_struct *napi;
6189
6190 restart:
6191 napi_poll = NULL;
6192
6193 rcu_read_lock();
6194
6195 napi = napi_by_id(napi_id);
6196 if (!napi)
6197 goto out;
6198
6199 preempt_disable();
6200 for (;;) {
6201 int work = 0;
6202
6203 local_bh_disable();
6204 if (!napi_poll) {
6205 unsigned long val = READ_ONCE(napi->state);
6206
6207 /* If multiple threads are competing for this napi,
6208 * we avoid dirtying napi->state as much as we can.
6209 */
6210 if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
6211 NAPIF_STATE_IN_BUSY_POLL)) {
6212 if (prefer_busy_poll)
6213 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6214 goto count;
6215 }
6216 if (cmpxchg(&napi->state, val,
6217 val | NAPIF_STATE_IN_BUSY_POLL |
6218 NAPIF_STATE_SCHED) != val) {
6219 if (prefer_busy_poll)
6220 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6221 goto count;
6222 }
6223 have_poll_lock = netpoll_poll_lock(napi);
6224 napi_poll = napi->poll;
6225 }
6226 work = napi_poll(napi, budget);
6227 trace_napi_poll(napi, work, budget);
6228 gro_normal_list(napi);
6229 count:
6230 if (work > 0)
6231 __NET_ADD_STATS(dev_net(napi->dev),
6232 LINUX_MIB_BUSYPOLLRXPACKETS, work);
6233 local_bh_enable();
6234
6235 if (!loop_end || loop_end(loop_end_arg, start_time))
6236 break;
6237
6238 if (unlikely(need_resched())) {
6239 if (napi_poll)
6240 busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
6241 preempt_enable();
6242 rcu_read_unlock();
6243 cond_resched();
6244 if (loop_end(loop_end_arg, start_time))
6245 return;
6246 goto restart;
6247 }
6248 cpu_relax();
6249 }
6250 if (napi_poll)
6251 busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
<