~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/core/dev.c

Version: ~ [ linux-5.8-rc4 ] ~ [ linux-5.7.7 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.50 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.131 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.187 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.229 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.229 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *      NET3    Protocol independent device support routines.
  3  *
  4  *              This program is free software; you can redistribute it and/or
  5  *              modify it under the terms of the GNU General Public License
  6  *              as published by the Free Software Foundation; either version
  7  *              2 of the License, or (at your option) any later version.
  8  *
  9  *      Derived from the non IP parts of dev.c 1.0.19
 10  *              Authors:        Ross Biro
 11  *                              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 12  *                              Mark Evans, <evansmp@uhura.aston.ac.uk>
 13  *
 14  *      Additional Authors:
 15  *              Florian la Roche <rzsfl@rz.uni-sb.de>
 16  *              Alan Cox <gw4pts@gw4pts.ampr.org>
 17  *              David Hinds <dahinds@users.sourceforge.net>
 18  *              Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
 19  *              Adam Sulmicki <adam@cfar.umd.edu>
 20  *              Pekka Riikonen <priikone@poesidon.pspt.fi>
 21  *
 22  *      Changes:
 23  *              D.J. Barrow     :       Fixed bug where dev->refcnt gets set
 24  *                                      to 2 if register_netdev gets called
 25  *                                      before net_dev_init & also removed a
 26  *                                      few lines of code in the process.
 27  *              Alan Cox        :       device private ioctl copies fields back.
 28  *              Alan Cox        :       Transmit queue code does relevant
 29  *                                      stunts to keep the queue safe.
 30  *              Alan Cox        :       Fixed double lock.
 31  *              Alan Cox        :       Fixed promisc NULL pointer trap
 32  *              ????????        :       Support the full private ioctl range
 33  *              Alan Cox        :       Moved ioctl permission check into
 34  *                                      drivers
 35  *              Tim Kordas      :       SIOCADDMULTI/SIOCDELMULTI
 36  *              Alan Cox        :       100 backlog just doesn't cut it when
 37  *                                      you start doing multicast video 8)
 38  *              Alan Cox        :       Rewrote net_bh and list manager.
 39  *              Alan Cox        :       Fix ETH_P_ALL echoback lengths.
 40  *              Alan Cox        :       Took out transmit every packet pass
 41  *                                      Saved a few bytes in the ioctl handler
 42  *              Alan Cox        :       Network driver sets packet type before
 43  *                                      calling netif_rx. Saves a function
 44  *                                      call a packet.
 45  *              Alan Cox        :       Hashed net_bh()
 46  *              Richard Kooijman:       Timestamp fixes.
 47  *              Alan Cox        :       Wrong field in SIOCGIFDSTADDR
 48  *              Alan Cox        :       Device lock protection.
 49  *              Alan Cox        :       Fixed nasty side effect of device close
 50  *                                      changes.
 51  *              Rudi Cilibrasi  :       Pass the right thing to
 52  *                                      set_mac_address()
 53  *              Dave Miller     :       32bit quantity for the device lock to
 54  *                                      make it work out on a Sparc.
 55  *              Bjorn Ekwall    :       Added KERNELD hack.
 56  *              Alan Cox        :       Cleaned up the backlog initialise.
 57  *              Craig Metz      :       SIOCGIFCONF fix if space for under
 58  *                                      1 device.
 59  *          Thomas Bogendoerfer :       Return ENODEV for dev_open, if there
 60  *                                      is no device open function.
 61  *              Andi Kleen      :       Fix error reporting for SIOCGIFCONF
 62  *          Michael Chastain    :       Fix signed/unsigned for SIOCGIFCONF
 63  *              Cyrus Durgin    :       Cleaned for KMOD
 64  *              Adam Sulmicki   :       Bug Fix : Network Device Unload
 65  *                                      A network device unload needs to purge
 66  *                                      the backlog queue.
 67  *      Paul Rusty Russell      :       SIOCSIFNAME
 68  *              Pekka Riikonen  :       Netdev boot-time settings code
 69  *              Andrew Morton   :       Make unregister_netdevice wait
 70  *                                      indefinitely on dev->refcnt
 71  *              J Hadi Salim    :       - Backlog queue sampling
 72  *                                      - netif_rx() feedback
 73  */
 74 
 75 #include <linux/uaccess.h>
 76 #include <linux/bitops.h>
 77 #include <linux/capability.h>
 78 #include <linux/cpu.h>
 79 #include <linux/types.h>
 80 #include <linux/kernel.h>
 81 #include <linux/hash.h>
 82 #include <linux/slab.h>
 83 #include <linux/sched.h>
 84 #include <linux/sched/mm.h>
 85 #include <linux/mutex.h>
 86 #include <linux/string.h>
 87 #include <linux/mm.h>
 88 #include <linux/socket.h>
 89 #include <linux/sockios.h>
 90 #include <linux/errno.h>
 91 #include <linux/interrupt.h>
 92 #include <linux/if_ether.h>
 93 #include <linux/netdevice.h>
 94 #include <linux/etherdevice.h>
 95 #include <linux/ethtool.h>
 96 #include <linux/skbuff.h>
 97 #include <linux/bpf.h>
 98 #include <linux/bpf_trace.h>
 99 #include <net/net_namespace.h>
100 #include <net/sock.h>
101 #include <net/busy_poll.h>
102 #include <linux/rtnetlink.h>
103 #include <linux/stat.h>
104 #include <net/dst.h>
105 #include <net/dst_metadata.h>
106 #include <net/pkt_sched.h>
107 #include <net/pkt_cls.h>
108 #include <net/checksum.h>
109 #include <net/xfrm.h>
110 #include <linux/highmem.h>
111 #include <linux/init.h>
112 #include <linux/module.h>
113 #include <linux/netpoll.h>
114 #include <linux/rcupdate.h>
115 #include <linux/delay.h>
116 #include <net/iw_handler.h>
117 #include <asm/current.h>
118 #include <linux/audit.h>
119 #include <linux/dmaengine.h>
120 #include <linux/err.h>
121 #include <linux/ctype.h>
122 #include <linux/if_arp.h>
123 #include <linux/if_vlan.h>
124 #include <linux/ip.h>
125 #include <net/ip.h>
126 #include <net/mpls.h>
127 #include <linux/ipv6.h>
128 #include <linux/in.h>
129 #include <linux/jhash.h>
130 #include <linux/random.h>
131 #include <trace/events/napi.h>
132 #include <trace/events/net.h>
133 #include <trace/events/skb.h>
134 #include <linux/pci.h>
135 #include <linux/inetdevice.h>
136 #include <linux/cpu_rmap.h>
137 #include <linux/static_key.h>
138 #include <linux/hashtable.h>
139 #include <linux/vmalloc.h>
140 #include <linux/if_macvlan.h>
141 #include <linux/errqueue.h>
142 #include <linux/hrtimer.h>
143 #include <linux/netfilter_ingress.h>
144 #include <linux/crash_dump.h>
145 #include <linux/sctp.h>
146 #include <net/udp_tunnel.h>
147 #include <linux/net_namespace.h>
148 #include <linux/indirect_call_wrapper.h>
149 
150 #include "net-sysfs.h"
151 
152 #define MAX_GRO_SKBS 8
153 
154 /* This should be increased if a protocol with a bigger head is added. */
155 #define GRO_MAX_HEAD (MAX_HEADER + 128)
156 
157 static DEFINE_SPINLOCK(ptype_lock);
158 static DEFINE_SPINLOCK(offload_lock);
159 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
160 struct list_head ptype_all __read_mostly;       /* Taps */
161 static struct list_head offload_base __read_mostly;
162 
163 static int netif_rx_internal(struct sk_buff *skb);
164 static int call_netdevice_notifiers_info(unsigned long val,
165                                          struct netdev_notifier_info *info);
166 static int call_netdevice_notifiers_extack(unsigned long val,
167                                            struct net_device *dev,
168                                            struct netlink_ext_ack *extack);
169 static struct napi_struct *napi_by_id(unsigned int napi_id);
170 
171 /*
172  * The @dev_base_head list is protected by @dev_base_lock and the rtnl
173  * semaphore.
174  *
175  * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
176  *
177  * Writers must hold the rtnl semaphore while they loop through the
178  * dev_base_head list, and hold dev_base_lock for writing when they do the
179  * actual updates.  This allows pure readers to access the list even
180  * while a writer is preparing to update it.
181  *
182  * To put it another way, dev_base_lock is held for writing only to
183  * protect against pure readers; the rtnl semaphore provides the
184  * protection against other writers.
185  *
186  * See, for example usages, register_netdevice() and
187  * unregister_netdevice(), which must be called with the rtnl
188  * semaphore held.
189  */
190 DEFINE_RWLOCK(dev_base_lock);
191 EXPORT_SYMBOL(dev_base_lock);
192 
193 static DEFINE_MUTEX(ifalias_mutex);
194 
195 /* protects napi_hash addition/deletion and napi_gen_id */
196 static DEFINE_SPINLOCK(napi_hash_lock);
197 
198 static unsigned int napi_gen_id = NR_CPUS;
199 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
200 
201 static seqcount_t devnet_rename_seq;
202 
203 static inline void dev_base_seq_inc(struct net *net)
204 {
205         while (++net->dev_base_seq == 0)
206                 ;
207 }
208 
209 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
210 {
211         unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
212 
213         return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
214 }
215 
216 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
217 {
218         return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
219 }
220 
221 static inline void rps_lock(struct softnet_data *sd)
222 {
223 #ifdef CONFIG_RPS
224         spin_lock(&sd->input_pkt_queue.lock);
225 #endif
226 }
227 
228 static inline void rps_unlock(struct softnet_data *sd)
229 {
230 #ifdef CONFIG_RPS
231         spin_unlock(&sd->input_pkt_queue.lock);
232 #endif
233 }
234 
235 /* Device list insertion */
236 static void list_netdevice(struct net_device *dev)
237 {
238         struct net *net = dev_net(dev);
239 
240         ASSERT_RTNL();
241 
242         write_lock_bh(&dev_base_lock);
243         list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
244         hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
245         hlist_add_head_rcu(&dev->index_hlist,
246                            dev_index_hash(net, dev->ifindex));
247         write_unlock_bh(&dev_base_lock);
248 
249         dev_base_seq_inc(net);
250 }
251 
252 /* Device list removal
253  * caller must respect a RCU grace period before freeing/reusing dev
254  */
255 static void unlist_netdevice(struct net_device *dev)
256 {
257         ASSERT_RTNL();
258 
259         /* Unlink dev from the device chain */
260         write_lock_bh(&dev_base_lock);
261         list_del_rcu(&dev->dev_list);
262         hlist_del_rcu(&dev->name_hlist);
263         hlist_del_rcu(&dev->index_hlist);
264         write_unlock_bh(&dev_base_lock);
265 
266         dev_base_seq_inc(dev_net(dev));
267 }
268 
269 /*
270  *      Our notifier list
271  */
272 
273 static RAW_NOTIFIER_HEAD(netdev_chain);
274 
275 /*
276  *      Device drivers call our routines to queue packets here. We empty the
277  *      queue in the local softnet handler.
278  */
279 
280 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
281 EXPORT_PER_CPU_SYMBOL(softnet_data);
282 
283 #ifdef CONFIG_LOCKDEP
284 /*
285  * register_netdevice() inits txq->_xmit_lock and sets lockdep class
286  * according to dev->type
287  */
288 static const unsigned short netdev_lock_type[] = {
289          ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
290          ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
291          ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
292          ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
293          ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
294          ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
295          ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
296          ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
297          ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
298          ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
299          ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
300          ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
301          ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
302          ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
303          ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
304 
305 static const char *const netdev_lock_name[] = {
306         "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
307         "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
308         "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
309         "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
310         "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
311         "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
312         "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
313         "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
314         "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
315         "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
316         "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
317         "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
318         "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
319         "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
320         "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
321 
322 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
323 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
324 
325 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
326 {
327         int i;
328 
329         for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
330                 if (netdev_lock_type[i] == dev_type)
331                         return i;
332         /* the last key is used by default */
333         return ARRAY_SIZE(netdev_lock_type) - 1;
334 }
335 
336 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
337                                                  unsigned short dev_type)
338 {
339         int i;
340 
341         i = netdev_lock_pos(dev_type);
342         lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
343                                    netdev_lock_name[i]);
344 }
345 
346 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
347 {
348         int i;
349 
350         i = netdev_lock_pos(dev->type);
351         lockdep_set_class_and_name(&dev->addr_list_lock,
352                                    &netdev_addr_lock_key[i],
353                                    netdev_lock_name[i]);
354 }
355 #else
356 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
357                                                  unsigned short dev_type)
358 {
359 }
360 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
361 {
362 }
363 #endif
364 
365 /*******************************************************************************
366  *
367  *              Protocol management and registration routines
368  *
369  *******************************************************************************/
370 
371 
372 /*
373  *      Add a protocol ID to the list. Now that the input handler is
374  *      smarter we can dispense with all the messy stuff that used to be
375  *      here.
376  *
377  *      BEWARE!!! Protocol handlers, mangling input packets,
378  *      MUST BE last in hash buckets and checking protocol handlers
379  *      MUST start from promiscuous ptype_all chain in net_bh.
380  *      It is true now, do not change it.
381  *      Explanation follows: if protocol handler, mangling packet, will
382  *      be the first on list, it is not able to sense, that packet
383  *      is cloned and should be copied-on-write, so that it will
384  *      change it and subsequent readers will get broken packet.
385  *                                                      --ANK (980803)
386  */
387 
388 static inline struct list_head *ptype_head(const struct packet_type *pt)
389 {
390         if (pt->type == htons(ETH_P_ALL))
391                 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
392         else
393                 return pt->dev ? &pt->dev->ptype_specific :
394                                  &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
395 }
396 
397 /**
398  *      dev_add_pack - add packet handler
399  *      @pt: packet type declaration
400  *
401  *      Add a protocol handler to the networking stack. The passed &packet_type
402  *      is linked into kernel lists and may not be freed until it has been
403  *      removed from the kernel lists.
404  *
405  *      This call does not sleep therefore it can not
406  *      guarantee all CPU's that are in middle of receiving packets
407  *      will see the new packet type (until the next received packet).
408  */
409 
410 void dev_add_pack(struct packet_type *pt)
411 {
412         struct list_head *head = ptype_head(pt);
413 
414         spin_lock(&ptype_lock);
415         list_add_rcu(&pt->list, head);
416         spin_unlock(&ptype_lock);
417 }
418 EXPORT_SYMBOL(dev_add_pack);
419 
420 /**
421  *      __dev_remove_pack        - remove packet handler
422  *      @pt: packet type declaration
423  *
424  *      Remove a protocol handler that was previously added to the kernel
425  *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
426  *      from the kernel lists and can be freed or reused once this function
427  *      returns.
428  *
429  *      The packet type might still be in use by receivers
430  *      and must not be freed until after all the CPU's have gone
431  *      through a quiescent state.
432  */
433 void __dev_remove_pack(struct packet_type *pt)
434 {
435         struct list_head *head = ptype_head(pt);
436         struct packet_type *pt1;
437 
438         spin_lock(&ptype_lock);
439 
440         list_for_each_entry(pt1, head, list) {
441                 if (pt == pt1) {
442                         list_del_rcu(&pt->list);
443                         goto out;
444                 }
445         }
446 
447         pr_warn("dev_remove_pack: %p not found\n", pt);
448 out:
449         spin_unlock(&ptype_lock);
450 }
451 EXPORT_SYMBOL(__dev_remove_pack);
452 
453 /**
454  *      dev_remove_pack  - remove packet handler
455  *      @pt: packet type declaration
456  *
457  *      Remove a protocol handler that was previously added to the kernel
458  *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
459  *      from the kernel lists and can be freed or reused once this function
460  *      returns.
461  *
462  *      This call sleeps to guarantee that no CPU is looking at the packet
463  *      type after return.
464  */
465 void dev_remove_pack(struct packet_type *pt)
466 {
467         __dev_remove_pack(pt);
468 
469         synchronize_net();
470 }
471 EXPORT_SYMBOL(dev_remove_pack);
472 
473 
474 /**
475  *      dev_add_offload - register offload handlers
476  *      @po: protocol offload declaration
477  *
478  *      Add protocol offload handlers to the networking stack. The passed
479  *      &proto_offload is linked into kernel lists and may not be freed until
480  *      it has been removed from the kernel lists.
481  *
482  *      This call does not sleep therefore it can not
483  *      guarantee all CPU's that are in middle of receiving packets
484  *      will see the new offload handlers (until the next received packet).
485  */
486 void dev_add_offload(struct packet_offload *po)
487 {
488         struct packet_offload *elem;
489 
490         spin_lock(&offload_lock);
491         list_for_each_entry(elem, &offload_base, list) {
492                 if (po->priority < elem->priority)
493                         break;
494         }
495         list_add_rcu(&po->list, elem->list.prev);
496         spin_unlock(&offload_lock);
497 }
498 EXPORT_SYMBOL(dev_add_offload);
499 
500 /**
501  *      __dev_remove_offload     - remove offload handler
502  *      @po: packet offload declaration
503  *
504  *      Remove a protocol offload handler that was previously added to the
505  *      kernel offload handlers by dev_add_offload(). The passed &offload_type
506  *      is removed from the kernel lists and can be freed or reused once this
507  *      function returns.
508  *
509  *      The packet type might still be in use by receivers
510  *      and must not be freed until after all the CPU's have gone
511  *      through a quiescent state.
512  */
513 static void __dev_remove_offload(struct packet_offload *po)
514 {
515         struct list_head *head = &offload_base;
516         struct packet_offload *po1;
517 
518         spin_lock(&offload_lock);
519 
520         list_for_each_entry(po1, head, list) {
521                 if (po == po1) {
522                         list_del_rcu(&po->list);
523                         goto out;
524                 }
525         }
526 
527         pr_warn("dev_remove_offload: %p not found\n", po);
528 out:
529         spin_unlock(&offload_lock);
530 }
531 
532 /**
533  *      dev_remove_offload       - remove packet offload handler
534  *      @po: packet offload declaration
535  *
536  *      Remove a packet offload handler that was previously added to the kernel
537  *      offload handlers by dev_add_offload(). The passed &offload_type is
538  *      removed from the kernel lists and can be freed or reused once this
539  *      function returns.
540  *
541  *      This call sleeps to guarantee that no CPU is looking at the packet
542  *      type after return.
543  */
544 void dev_remove_offload(struct packet_offload *po)
545 {
546         __dev_remove_offload(po);
547 
548         synchronize_net();
549 }
550 EXPORT_SYMBOL(dev_remove_offload);
551 
552 /******************************************************************************
553  *
554  *                    Device Boot-time Settings Routines
555  *
556  ******************************************************************************/
557 
558 /* Boot time configuration table */
559 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
560 
561 /**
562  *      netdev_boot_setup_add   - add new setup entry
563  *      @name: name of the device
564  *      @map: configured settings for the device
565  *
566  *      Adds new setup entry to the dev_boot_setup list.  The function
567  *      returns 0 on error and 1 on success.  This is a generic routine to
568  *      all netdevices.
569  */
570 static int netdev_boot_setup_add(char *name, struct ifmap *map)
571 {
572         struct netdev_boot_setup *s;
573         int i;
574 
575         s = dev_boot_setup;
576         for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
577                 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
578                         memset(s[i].name, 0, sizeof(s[i].name));
579                         strlcpy(s[i].name, name, IFNAMSIZ);
580                         memcpy(&s[i].map, map, sizeof(s[i].map));
581                         break;
582                 }
583         }
584 
585         return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
586 }
587 
588 /**
589  * netdev_boot_setup_check      - check boot time settings
590  * @dev: the netdevice
591  *
592  * Check boot time settings for the device.
593  * The found settings are set for the device to be used
594  * later in the device probing.
595  * Returns 0 if no settings found, 1 if they are.
596  */
597 int netdev_boot_setup_check(struct net_device *dev)
598 {
599         struct netdev_boot_setup *s = dev_boot_setup;
600         int i;
601 
602         for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
603                 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
604                     !strcmp(dev->name, s[i].name)) {
605                         dev->irq = s[i].map.irq;
606                         dev->base_addr = s[i].map.base_addr;
607                         dev->mem_start = s[i].map.mem_start;
608                         dev->mem_end = s[i].map.mem_end;
609                         return 1;
610                 }
611         }
612         return 0;
613 }
614 EXPORT_SYMBOL(netdev_boot_setup_check);
615 
616 
617 /**
618  * netdev_boot_base     - get address from boot time settings
619  * @prefix: prefix for network device
620  * @unit: id for network device
621  *
622  * Check boot time settings for the base address of device.
623  * The found settings are set for the device to be used
624  * later in the device probing.
625  * Returns 0 if no settings found.
626  */
627 unsigned long netdev_boot_base(const char *prefix, int unit)
628 {
629         const struct netdev_boot_setup *s = dev_boot_setup;
630         char name[IFNAMSIZ];
631         int i;
632 
633         sprintf(name, "%s%d", prefix, unit);
634 
635         /*
636          * If device already registered then return base of 1
637          * to indicate not to probe for this interface
638          */
639         if (__dev_get_by_name(&init_net, name))
640                 return 1;
641 
642         for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
643                 if (!strcmp(name, s[i].name))
644                         return s[i].map.base_addr;
645         return 0;
646 }
647 
648 /*
649  * Saves at boot time configured settings for any netdevice.
650  */
651 int __init netdev_boot_setup(char *str)
652 {
653         int ints[5];
654         struct ifmap map;
655 
656         str = get_options(str, ARRAY_SIZE(ints), ints);
657         if (!str || !*str)
658                 return 0;
659 
660         /* Save settings */
661         memset(&map, 0, sizeof(map));
662         if (ints[0] > 0)
663                 map.irq = ints[1];
664         if (ints[0] > 1)
665                 map.base_addr = ints[2];
666         if (ints[0] > 2)
667                 map.mem_start = ints[3];
668         if (ints[0] > 3)
669                 map.mem_end = ints[4];
670 
671         /* Add new entry to the list */
672         return netdev_boot_setup_add(str, &map);
673 }
674 
675 __setup("netdev=", netdev_boot_setup);
676 
677 /*******************************************************************************
678  *
679  *                          Device Interface Subroutines
680  *
681  *******************************************************************************/
682 
683 /**
684  *      dev_get_iflink  - get 'iflink' value of a interface
685  *      @dev: targeted interface
686  *
687  *      Indicates the ifindex the interface is linked to.
688  *      Physical interfaces have the same 'ifindex' and 'iflink' values.
689  */
690 
691 int dev_get_iflink(const struct net_device *dev)
692 {
693         if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
694                 return dev->netdev_ops->ndo_get_iflink(dev);
695 
696         return dev->ifindex;
697 }
698 EXPORT_SYMBOL(dev_get_iflink);
699 
700 /**
701  *      dev_fill_metadata_dst - Retrieve tunnel egress information.
702  *      @dev: targeted interface
703  *      @skb: The packet.
704  *
705  *      For better visibility of tunnel traffic OVS needs to retrieve
706  *      egress tunnel information for a packet. Following API allows
707  *      user to get this info.
708  */
709 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
710 {
711         struct ip_tunnel_info *info;
712 
713         if (!dev->netdev_ops  || !dev->netdev_ops->ndo_fill_metadata_dst)
714                 return -EINVAL;
715 
716         info = skb_tunnel_info_unclone(skb);
717         if (!info)
718                 return -ENOMEM;
719         if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
720                 return -EINVAL;
721 
722         return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
723 }
724 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
725 
726 /**
727  *      __dev_get_by_name       - find a device by its name
728  *      @net: the applicable net namespace
729  *      @name: name to find
730  *
731  *      Find an interface by name. Must be called under RTNL semaphore
732  *      or @dev_base_lock. If the name is found a pointer to the device
733  *      is returned. If the name is not found then %NULL is returned. The
734  *      reference counters are not incremented so the caller must be
735  *      careful with locks.
736  */
737 
738 struct net_device *__dev_get_by_name(struct net *net, const char *name)
739 {
740         struct net_device *dev;
741         struct hlist_head *head = dev_name_hash(net, name);
742 
743         hlist_for_each_entry(dev, head, name_hlist)
744                 if (!strncmp(dev->name, name, IFNAMSIZ))
745                         return dev;
746 
747         return NULL;
748 }
749 EXPORT_SYMBOL(__dev_get_by_name);
750 
751 /**
752  * dev_get_by_name_rcu  - find a device by its name
753  * @net: the applicable net namespace
754  * @name: name to find
755  *
756  * Find an interface by name.
757  * If the name is found a pointer to the device is returned.
758  * If the name is not found then %NULL is returned.
759  * The reference counters are not incremented so the caller must be
760  * careful with locks. The caller must hold RCU lock.
761  */
762 
763 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
764 {
765         struct net_device *dev;
766         struct hlist_head *head = dev_name_hash(net, name);
767 
768         hlist_for_each_entry_rcu(dev, head, name_hlist)
769                 if (!strncmp(dev->name, name, IFNAMSIZ))
770                         return dev;
771 
772         return NULL;
773 }
774 EXPORT_SYMBOL(dev_get_by_name_rcu);
775 
776 /**
777  *      dev_get_by_name         - find a device by its name
778  *      @net: the applicable net namespace
779  *      @name: name to find
780  *
781  *      Find an interface by name. This can be called from any
782  *      context and does its own locking. The returned handle has
783  *      the usage count incremented and the caller must use dev_put() to
784  *      release it when it is no longer needed. %NULL is returned if no
785  *      matching device is found.
786  */
787 
788 struct net_device *dev_get_by_name(struct net *net, const char *name)
789 {
790         struct net_device *dev;
791 
792         rcu_read_lock();
793         dev = dev_get_by_name_rcu(net, name);
794         if (dev)
795                 dev_hold(dev);
796         rcu_read_unlock();
797         return dev;
798 }
799 EXPORT_SYMBOL(dev_get_by_name);
800 
801 /**
802  *      __dev_get_by_index - find a device by its ifindex
803  *      @net: the applicable net namespace
804  *      @ifindex: index of device
805  *
806  *      Search for an interface by index. Returns %NULL if the device
807  *      is not found or a pointer to the device. The device has not
808  *      had its reference counter increased so the caller must be careful
809  *      about locking. The caller must hold either the RTNL semaphore
810  *      or @dev_base_lock.
811  */
812 
813 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
814 {
815         struct net_device *dev;
816         struct hlist_head *head = dev_index_hash(net, ifindex);
817 
818         hlist_for_each_entry(dev, head, index_hlist)
819                 if (dev->ifindex == ifindex)
820                         return dev;
821 
822         return NULL;
823 }
824 EXPORT_SYMBOL(__dev_get_by_index);
825 
826 /**
827  *      dev_get_by_index_rcu - find a device by its ifindex
828  *      @net: the applicable net namespace
829  *      @ifindex: index of device
830  *
831  *      Search for an interface by index. Returns %NULL if the device
832  *      is not found or a pointer to the device. The device has not
833  *      had its reference counter increased so the caller must be careful
834  *      about locking. The caller must hold RCU lock.
835  */
836 
837 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
838 {
839         struct net_device *dev;
840         struct hlist_head *head = dev_index_hash(net, ifindex);
841 
842         hlist_for_each_entry_rcu(dev, head, index_hlist)
843                 if (dev->ifindex == ifindex)
844                         return dev;
845 
846         return NULL;
847 }
848 EXPORT_SYMBOL(dev_get_by_index_rcu);
849 
850 
851 /**
852  *      dev_get_by_index - find a device by its ifindex
853  *      @net: the applicable net namespace
854  *      @ifindex: index of device
855  *
856  *      Search for an interface by index. Returns NULL if the device
857  *      is not found or a pointer to the device. The device returned has
858  *      had a reference added and the pointer is safe until the user calls
859  *      dev_put to indicate they have finished with it.
860  */
861 
862 struct net_device *dev_get_by_index(struct net *net, int ifindex)
863 {
864         struct net_device *dev;
865 
866         rcu_read_lock();
867         dev = dev_get_by_index_rcu(net, ifindex);
868         if (dev)
869                 dev_hold(dev);
870         rcu_read_unlock();
871         return dev;
872 }
873 EXPORT_SYMBOL(dev_get_by_index);
874 
875 /**
876  *      dev_get_by_napi_id - find a device by napi_id
877  *      @napi_id: ID of the NAPI struct
878  *
879  *      Search for an interface by NAPI ID. Returns %NULL if the device
880  *      is not found or a pointer to the device. The device has not had
881  *      its reference counter increased so the caller must be careful
882  *      about locking. The caller must hold RCU lock.
883  */
884 
885 struct net_device *dev_get_by_napi_id(unsigned int napi_id)
886 {
887         struct napi_struct *napi;
888 
889         WARN_ON_ONCE(!rcu_read_lock_held());
890 
891         if (napi_id < MIN_NAPI_ID)
892                 return NULL;
893 
894         napi = napi_by_id(napi_id);
895 
896         return napi ? napi->dev : NULL;
897 }
898 EXPORT_SYMBOL(dev_get_by_napi_id);
899 
900 /**
901  *      netdev_get_name - get a netdevice name, knowing its ifindex.
902  *      @net: network namespace
903  *      @name: a pointer to the buffer where the name will be stored.
904  *      @ifindex: the ifindex of the interface to get the name from.
905  *
906  *      The use of raw_seqcount_begin() and cond_resched() before
907  *      retrying is required as we want to give the writers a chance
908  *      to complete when CONFIG_PREEMPT is not set.
909  */
910 int netdev_get_name(struct net *net, char *name, int ifindex)
911 {
912         struct net_device *dev;
913         unsigned int seq;
914 
915 retry:
916         seq = raw_seqcount_begin(&devnet_rename_seq);
917         rcu_read_lock();
918         dev = dev_get_by_index_rcu(net, ifindex);
919         if (!dev) {
920                 rcu_read_unlock();
921                 return -ENODEV;
922         }
923 
924         strcpy(name, dev->name);
925         rcu_read_unlock();
926         if (read_seqcount_retry(&devnet_rename_seq, seq)) {
927                 cond_resched();
928                 goto retry;
929         }
930 
931         return 0;
932 }
933 
934 /**
935  *      dev_getbyhwaddr_rcu - find a device by its hardware address
936  *      @net: the applicable net namespace
937  *      @type: media type of device
938  *      @ha: hardware address
939  *
940  *      Search for an interface by MAC address. Returns NULL if the device
941  *      is not found or a pointer to the device.
942  *      The caller must hold RCU or RTNL.
943  *      The returned device has not had its ref count increased
944  *      and the caller must therefore be careful about locking
945  *
946  */
947 
948 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
949                                        const char *ha)
950 {
951         struct net_device *dev;
952 
953         for_each_netdev_rcu(net, dev)
954                 if (dev->type == type &&
955                     !memcmp(dev->dev_addr, ha, dev->addr_len))
956                         return dev;
957 
958         return NULL;
959 }
960 EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
961 
962 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
963 {
964         struct net_device *dev;
965 
966         ASSERT_RTNL();
967         for_each_netdev(net, dev)
968                 if (dev->type == type)
969                         return dev;
970 
971         return NULL;
972 }
973 EXPORT_SYMBOL(__dev_getfirstbyhwtype);
974 
975 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
976 {
977         struct net_device *dev, *ret = NULL;
978 
979         rcu_read_lock();
980         for_each_netdev_rcu(net, dev)
981                 if (dev->type == type) {
982                         dev_hold(dev);
983                         ret = dev;
984                         break;
985                 }
986         rcu_read_unlock();
987         return ret;
988 }
989 EXPORT_SYMBOL(dev_getfirstbyhwtype);
990 
991 /**
992  *      __dev_get_by_flags - find any device with given flags
993  *      @net: the applicable net namespace
994  *      @if_flags: IFF_* values
995  *      @mask: bitmask of bits in if_flags to check
996  *
997  *      Search for any interface with the given flags. Returns NULL if a device
998  *      is not found or a pointer to the device. Must be called inside
999  *      rtnl_lock(), and result refcount is unchanged.
1000  */
1001 
1002 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
1003                                       unsigned short mask)
1004 {
1005         struct net_device *dev, *ret;
1006 
1007         ASSERT_RTNL();
1008 
1009         ret = NULL;
1010         for_each_netdev(net, dev) {
1011                 if (((dev->flags ^ if_flags) & mask) == 0) {
1012                         ret = dev;
1013                         break;
1014                 }
1015         }
1016         return ret;
1017 }
1018 EXPORT_SYMBOL(__dev_get_by_flags);
1019 
1020 /**
1021  *      dev_valid_name - check if name is okay for network device
1022  *      @name: name string
1023  *
1024  *      Network device names need to be valid file names to
1025  *      to allow sysfs to work.  We also disallow any kind of
1026  *      whitespace.
1027  */
1028 bool dev_valid_name(const char *name)
1029 {
1030         if (*name == '\0')
1031                 return false;
1032         if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
1033                 return false;
1034         if (!strcmp(name, ".") || !strcmp(name, ".."))
1035                 return false;
1036 
1037         while (*name) {
1038                 if (*name == '/' || *name == ':' || isspace(*name))
1039                         return false;
1040                 name++;
1041         }
1042         return true;
1043 }
1044 EXPORT_SYMBOL(dev_valid_name);
1045 
1046 /**
1047  *      __dev_alloc_name - allocate a name for a device
1048  *      @net: network namespace to allocate the device name in
1049  *      @name: name format string
1050  *      @buf:  scratch buffer and result name string
1051  *
1052  *      Passed a format string - eg "lt%d" it will try and find a suitable
1053  *      id. It scans list of devices to build up a free map, then chooses
1054  *      the first empty slot. The caller must hold the dev_base or rtnl lock
1055  *      while allocating the name and adding the device in order to avoid
1056  *      duplicates.
1057  *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1058  *      Returns the number of the unit assigned or a negative errno code.
1059  */
1060 
1061 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1062 {
1063         int i = 0;
1064         const char *p;
1065         const int max_netdevices = 8*PAGE_SIZE;
1066         unsigned long *inuse;
1067         struct net_device *d;
1068 
1069         if (!dev_valid_name(name))
1070                 return -EINVAL;
1071 
1072         p = strchr(name, '%');
1073         if (p) {
1074                 /*
1075                  * Verify the string as this thing may have come from
1076                  * the user.  There must be either one "%d" and no other "%"
1077                  * characters.
1078                  */
1079                 if (p[1] != 'd' || strchr(p + 2, '%'))
1080                         return -EINVAL;
1081 
1082                 /* Use one page as a bit array of possible slots */
1083                 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1084                 if (!inuse)
1085                         return -ENOMEM;
1086 
1087                 for_each_netdev(net, d) {
1088                         if (!sscanf(d->name, name, &i))
1089                                 continue;
1090                         if (i < 0 || i >= max_netdevices)
1091                                 continue;
1092 
1093                         /*  avoid cases where sscanf is not exact inverse of printf */
1094                         snprintf(buf, IFNAMSIZ, name, i);
1095                         if (!strncmp(buf, d->name, IFNAMSIZ))
1096                                 set_bit(i, inuse);
1097                 }
1098 
1099                 i = find_first_zero_bit(inuse, max_netdevices);
1100                 free_page((unsigned long) inuse);
1101         }
1102 
1103         snprintf(buf, IFNAMSIZ, name, i);
1104         if (!__dev_get_by_name(net, buf))
1105                 return i;
1106 
1107         /* It is possible to run out of possible slots
1108          * when the name is long and there isn't enough space left
1109          * for the digits, or if all bits are used.
1110          */
1111         return -ENFILE;
1112 }
1113 
1114 static int dev_alloc_name_ns(struct net *net,
1115                              struct net_device *dev,
1116                              const char *name)
1117 {
1118         char buf[IFNAMSIZ];
1119         int ret;
1120 
1121         BUG_ON(!net);
1122         ret = __dev_alloc_name(net, name, buf);
1123         if (ret >= 0)
1124                 strlcpy(dev->name, buf, IFNAMSIZ);
1125         return ret;
1126 }
1127 
1128 /**
1129  *      dev_alloc_name - allocate a name for a device
1130  *      @dev: device
1131  *      @name: name format string
1132  *
1133  *      Passed a format string - eg "lt%d" it will try and find a suitable
1134  *      id. It scans list of devices to build up a free map, then chooses
1135  *      the first empty slot. The caller must hold the dev_base or rtnl lock
1136  *      while allocating the name and adding the device in order to avoid
1137  *      duplicates.
1138  *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1139  *      Returns the number of the unit assigned or a negative errno code.
1140  */
1141 
1142 int dev_alloc_name(struct net_device *dev, const char *name)
1143 {
1144         return dev_alloc_name_ns(dev_net(dev), dev, name);
1145 }
1146 EXPORT_SYMBOL(dev_alloc_name);
1147 
1148 int dev_get_valid_name(struct net *net, struct net_device *dev,
1149                        const char *name)
1150 {
1151         BUG_ON(!net);
1152 
1153         if (!dev_valid_name(name))
1154                 return -EINVAL;
1155 
1156         if (strchr(name, '%'))
1157                 return dev_alloc_name_ns(net, dev, name);
1158         else if (__dev_get_by_name(net, name))
1159                 return -EEXIST;
1160         else if (dev->name != name)
1161                 strlcpy(dev->name, name, IFNAMSIZ);
1162 
1163         return 0;
1164 }
1165 EXPORT_SYMBOL(dev_get_valid_name);
1166 
1167 /**
1168  *      dev_change_name - change name of a device
1169  *      @dev: device
1170  *      @newname: name (or format string) must be at least IFNAMSIZ
1171  *
1172  *      Change name of a device, can pass format strings "eth%d".
1173  *      for wildcarding.
1174  */
1175 int dev_change_name(struct net_device *dev, const char *newname)
1176 {
1177         unsigned char old_assign_type;
1178         char oldname[IFNAMSIZ];
1179         int err = 0;
1180         int ret;
1181         struct net *net;
1182 
1183         ASSERT_RTNL();
1184         BUG_ON(!dev_net(dev));
1185 
1186         net = dev_net(dev);
1187 
1188         /* Some auto-enslaved devices e.g. failover slaves are
1189          * special, as userspace might rename the device after
1190          * the interface had been brought up and running since
1191          * the point kernel initiated auto-enslavement. Allow
1192          * live name change even when these slave devices are
1193          * up and running.
1194          *
1195          * Typically, users of these auto-enslaving devices
1196          * don't actually care about slave name change, as
1197          * they are supposed to operate on master interface
1198          * directly.
1199          */
1200         if (dev->flags & IFF_UP &&
1201             likely(!(dev->priv_flags & IFF_LIVE_RENAME_OK)))
1202                 return -EBUSY;
1203 
1204         write_seqcount_begin(&devnet_rename_seq);
1205 
1206         if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1207                 write_seqcount_end(&devnet_rename_seq);
1208                 return 0;
1209         }
1210 
1211         memcpy(oldname, dev->name, IFNAMSIZ);
1212 
1213         err = dev_get_valid_name(net, dev, newname);
1214         if (err < 0) {
1215                 write_seqcount_end(&devnet_rename_seq);
1216                 return err;
1217         }
1218 
1219         if (oldname[0] && !strchr(oldname, '%'))
1220                 netdev_info(dev, "renamed from %s\n", oldname);
1221 
1222         old_assign_type = dev->name_assign_type;
1223         dev->name_assign_type = NET_NAME_RENAMED;
1224 
1225 rollback:
1226         ret = device_rename(&dev->dev, dev->name);
1227         if (ret) {
1228                 memcpy(dev->name, oldname, IFNAMSIZ);
1229                 dev->name_assign_type = old_assign_type;
1230                 write_seqcount_end(&devnet_rename_seq);
1231                 return ret;
1232         }
1233 
1234         write_seqcount_end(&devnet_rename_seq);
1235 
1236         netdev_adjacent_rename_links(dev, oldname);
1237 
1238         write_lock_bh(&dev_base_lock);
1239         hlist_del_rcu(&dev->name_hlist);
1240         write_unlock_bh(&dev_base_lock);
1241 
1242         synchronize_rcu();
1243 
1244         write_lock_bh(&dev_base_lock);
1245         hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1246         write_unlock_bh(&dev_base_lock);
1247 
1248         ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1249         ret = notifier_to_errno(ret);
1250 
1251         if (ret) {
1252                 /* err >= 0 after dev_alloc_name() or stores the first errno */
1253                 if (err >= 0) {
1254                         err = ret;
1255                         write_seqcount_begin(&devnet_rename_seq);
1256                         memcpy(dev->name, oldname, IFNAMSIZ);
1257                         memcpy(oldname, newname, IFNAMSIZ);
1258                         dev->name_assign_type = old_assign_type;
1259                         old_assign_type = NET_NAME_RENAMED;
1260                         goto rollback;
1261                 } else {
1262                         pr_err("%s: name change rollback failed: %d\n",
1263                                dev->name, ret);
1264                 }
1265         }
1266 
1267         return err;
1268 }
1269 
1270 /**
1271  *      dev_set_alias - change ifalias of a device
1272  *      @dev: device
1273  *      @alias: name up to IFALIASZ
1274  *      @len: limit of bytes to copy from info
1275  *
1276  *      Set ifalias for a device,
1277  */
1278 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1279 {
1280         struct dev_ifalias *new_alias = NULL;
1281 
1282         if (len >= IFALIASZ)
1283                 return -EINVAL;
1284 
1285         if (len) {
1286                 new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL);
1287                 if (!new_alias)
1288                         return -ENOMEM;
1289 
1290                 memcpy(new_alias->ifalias, alias, len);
1291                 new_alias->ifalias[len] = 0;
1292         }
1293 
1294         mutex_lock(&ifalias_mutex);
1295         rcu_swap_protected(dev->ifalias, new_alias,
1296                            mutex_is_locked(&ifalias_mutex));
1297         mutex_unlock(&ifalias_mutex);
1298 
1299         if (new_alias)
1300                 kfree_rcu(new_alias, rcuhead);
1301 
1302         return len;
1303 }
1304 EXPORT_SYMBOL(dev_set_alias);
1305 
1306 /**
1307  *      dev_get_alias - get ifalias of a device
1308  *      @dev: device
1309  *      @name: buffer to store name of ifalias
1310  *      @len: size of buffer
1311  *
1312  *      get ifalias for a device.  Caller must make sure dev cannot go
1313  *      away,  e.g. rcu read lock or own a reference count to device.
1314  */
1315 int dev_get_alias(const struct net_device *dev, char *name, size_t len)
1316 {
1317         const struct dev_ifalias *alias;
1318         int ret = 0;
1319 
1320         rcu_read_lock();
1321         alias = rcu_dereference(dev->ifalias);
1322         if (alias)
1323                 ret = snprintf(name, len, "%s", alias->ifalias);
1324         rcu_read_unlock();
1325 
1326         return ret;
1327 }
1328 
1329 /**
1330  *      netdev_features_change - device changes features
1331  *      @dev: device to cause notification
1332  *
1333  *      Called to indicate a device has changed features.
1334  */
1335 void netdev_features_change(struct net_device *dev)
1336 {
1337         call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1338 }
1339 EXPORT_SYMBOL(netdev_features_change);
1340 
1341 /**
1342  *      netdev_state_change - device changes state
1343  *      @dev: device to cause notification
1344  *
1345  *      Called to indicate a device has changed state. This function calls
1346  *      the notifier chains for netdev_chain and sends a NEWLINK message
1347  *      to the routing socket.
1348  */
1349 void netdev_state_change(struct net_device *dev)
1350 {
1351         if (dev->flags & IFF_UP) {
1352                 struct netdev_notifier_change_info change_info = {
1353                         .info.dev = dev,
1354                 };
1355 
1356                 call_netdevice_notifiers_info(NETDEV_CHANGE,
1357                                               &change_info.info);
1358                 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1359         }
1360 }
1361 EXPORT_SYMBOL(netdev_state_change);
1362 
1363 /**
1364  * netdev_notify_peers - notify network peers about existence of @dev
1365  * @dev: network device
1366  *
1367  * Generate traffic such that interested network peers are aware of
1368  * @dev, such as by generating a gratuitous ARP. This may be used when
1369  * a device wants to inform the rest of the network about some sort of
1370  * reconfiguration such as a failover event or virtual machine
1371  * migration.
1372  */
1373 void netdev_notify_peers(struct net_device *dev)
1374 {
1375         rtnl_lock();
1376         call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1377         call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1378         rtnl_unlock();
1379 }
1380 EXPORT_SYMBOL(netdev_notify_peers);
1381 
1382 static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1383 {
1384         const struct net_device_ops *ops = dev->netdev_ops;
1385         int ret;
1386 
1387         ASSERT_RTNL();
1388 
1389         if (!netif_device_present(dev))
1390                 return -ENODEV;
1391 
1392         /* Block netpoll from trying to do any rx path servicing.
1393          * If we don't do this there is a chance ndo_poll_controller
1394          * or ndo_poll may be running while we open the device
1395          */
1396         netpoll_poll_disable(dev);
1397 
1398         ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);
1399         ret = notifier_to_errno(ret);
1400         if (ret)
1401                 return ret;
1402 
1403         set_bit(__LINK_STATE_START, &dev->state);
1404 
1405         if (ops->ndo_validate_addr)
1406                 ret = ops->ndo_validate_addr(dev);
1407 
1408         if (!ret && ops->ndo_open)
1409                 ret = ops->ndo_open(dev);
1410 
1411         netpoll_poll_enable(dev);
1412 
1413         if (ret)
1414                 clear_bit(__LINK_STATE_START, &dev->state);
1415         else {
1416                 dev->flags |= IFF_UP;
1417                 dev_set_rx_mode(dev);
1418                 dev_activate(dev);
1419                 add_device_randomness(dev->dev_addr, dev->addr_len);
1420         }
1421 
1422         return ret;
1423 }
1424 
1425 /**
1426  *      dev_open        - prepare an interface for use.
1427  *      @dev: device to open
1428  *      @extack: netlink extended ack
1429  *
1430  *      Takes a device from down to up state. The device's private open
1431  *      function is invoked and then the multicast lists are loaded. Finally
1432  *      the device is moved into the up state and a %NETDEV_UP message is
1433  *      sent to the netdev notifier chain.
1434  *
1435  *      Calling this function on an active interface is a nop. On a failure
1436  *      a negative errno code is returned.
1437  */
1438 int dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1439 {
1440         int ret;
1441 
1442         if (dev->flags & IFF_UP)
1443                 return 0;
1444 
1445         ret = __dev_open(dev, extack);
1446         if (ret < 0)
1447                 return ret;
1448 
1449         rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1450         call_netdevice_notifiers(NETDEV_UP, dev);
1451 
1452         return ret;
1453 }
1454 EXPORT_SYMBOL(dev_open);
1455 
1456 static void __dev_close_many(struct list_head *head)
1457 {
1458         struct net_device *dev;
1459 
1460         ASSERT_RTNL();
1461         might_sleep();
1462 
1463         list_for_each_entry(dev, head, close_list) {
1464                 /* Temporarily disable netpoll until the interface is down */
1465                 netpoll_poll_disable(dev);
1466 
1467                 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1468 
1469                 clear_bit(__LINK_STATE_START, &dev->state);
1470 
1471                 /* Synchronize to scheduled poll. We cannot touch poll list, it
1472                  * can be even on different cpu. So just clear netif_running().
1473                  *
1474                  * dev->stop() will invoke napi_disable() on all of it's
1475                  * napi_struct instances on this device.
1476                  */
1477                 smp_mb__after_atomic(); /* Commit netif_running(). */
1478         }
1479 
1480         dev_deactivate_many(head);
1481 
1482         list_for_each_entry(dev, head, close_list) {
1483                 const struct net_device_ops *ops = dev->netdev_ops;
1484 
1485                 /*
1486                  *      Call the device specific close. This cannot fail.
1487                  *      Only if device is UP
1488                  *
1489                  *      We allow it to be called even after a DETACH hot-plug
1490                  *      event.
1491                  */
1492                 if (ops->ndo_stop)
1493                         ops->ndo_stop(dev);
1494 
1495                 dev->flags &= ~IFF_UP;
1496                 netpoll_poll_enable(dev);
1497         }
1498 }
1499 
1500 static void __dev_close(struct net_device *dev)
1501 {
1502         LIST_HEAD(single);
1503 
1504         list_add(&dev->close_list, &single);
1505         __dev_close_many(&single);
1506         list_del(&single);
1507 }
1508 
1509 void dev_close_many(struct list_head *head, bool unlink)
1510 {
1511         struct net_device *dev, *tmp;
1512 
1513         /* Remove the devices that don't need to be closed */
1514         list_for_each_entry_safe(dev, tmp, head, close_list)
1515                 if (!(dev->flags & IFF_UP))
1516                         list_del_init(&dev->close_list);
1517 
1518         __dev_close_many(head);
1519 
1520         list_for_each_entry_safe(dev, tmp, head, close_list) {
1521                 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1522                 call_netdevice_notifiers(NETDEV_DOWN, dev);
1523                 if (unlink)
1524                         list_del_init(&dev->close_list);
1525         }
1526 }
1527 EXPORT_SYMBOL(dev_close_many);
1528 
1529 /**
1530  *      dev_close - shutdown an interface.
1531  *      @dev: device to shutdown
1532  *
1533  *      This function moves an active device into down state. A
1534  *      %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1535  *      is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1536  *      chain.
1537  */
1538 void dev_close(struct net_device *dev)
1539 {
1540         if (dev->flags & IFF_UP) {
1541                 LIST_HEAD(single);
1542 
1543                 list_add(&dev->close_list, &single);
1544                 dev_close_many(&single, true);
1545                 list_del(&single);
1546         }
1547 }
1548 EXPORT_SYMBOL(dev_close);
1549 
1550 
1551 /**
1552  *      dev_disable_lro - disable Large Receive Offload on a device
1553  *      @dev: device
1554  *
1555  *      Disable Large Receive Offload (LRO) on a net device.  Must be
1556  *      called under RTNL.  This is needed if received packets may be
1557  *      forwarded to another interface.
1558  */
1559 void dev_disable_lro(struct net_device *dev)
1560 {
1561         struct net_device *lower_dev;
1562         struct list_head *iter;
1563 
1564         dev->wanted_features &= ~NETIF_F_LRO;
1565         netdev_update_features(dev);
1566 
1567         if (unlikely(dev->features & NETIF_F_LRO))
1568                 netdev_WARN(dev, "failed to disable LRO!\n");
1569 
1570         netdev_for_each_lower_dev(dev, lower_dev, iter)
1571                 dev_disable_lro(lower_dev);
1572 }
1573 EXPORT_SYMBOL(dev_disable_lro);
1574 
1575 /**
1576  *      dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1577  *      @dev: device
1578  *
1579  *      Disable HW Generic Receive Offload (GRO_HW) on a net device.  Must be
1580  *      called under RTNL.  This is needed if Generic XDP is installed on
1581  *      the device.
1582  */
1583 static void dev_disable_gro_hw(struct net_device *dev)
1584 {
1585         dev->wanted_features &= ~NETIF_F_GRO_HW;
1586         netdev_update_features(dev);
1587 
1588         if (unlikely(dev->features & NETIF_F_GRO_HW))
1589                 netdev_WARN(dev, "failed to disable GRO_HW!\n");
1590 }
1591 
1592 const char *netdev_cmd_to_name(enum netdev_cmd cmd)
1593 {
1594 #define N(val)                                          \
1595         case NETDEV_##val:                              \
1596                 return "NETDEV_" __stringify(val);
1597         switch (cmd) {
1598         N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
1599         N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
1600         N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
1601         N(POST_INIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN) N(CHANGEUPPER)
1602         N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA) N(BONDING_INFO)
1603         N(PRECHANGEUPPER) N(CHANGELOWERSTATE) N(UDP_TUNNEL_PUSH_INFO)
1604         N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
1605         N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
1606         N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
1607         N(PRE_CHANGEADDR)
1608         }
1609 #undef N
1610         return "UNKNOWN_NETDEV_EVENT";
1611 }
1612 EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
1613 
1614 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1615                                    struct net_device *dev)
1616 {
1617         struct netdev_notifier_info info = {
1618                 .dev = dev,
1619         };
1620 
1621         return nb->notifier_call(nb, val, &info);
1622 }
1623 
1624 static int dev_boot_phase = 1;
1625 
1626 /**
1627  * register_netdevice_notifier - register a network notifier block
1628  * @nb: notifier
1629  *
1630  * Register a notifier to be called when network device events occur.
1631  * The notifier passed is linked into the kernel structures and must
1632  * not be reused until it has been unregistered. A negative errno code
1633  * is returned on a failure.
1634  *
1635  * When registered all registration and up events are replayed
1636  * to the new notifier to allow device to have a race free
1637  * view of the network device list.
1638  */
1639 
1640 int register_netdevice_notifier(struct notifier_block *nb)
1641 {
1642         struct net_device *dev;
1643         struct net_device *last;
1644         struct net *net;
1645         int err;
1646 
1647         /* Close race with setup_net() and cleanup_net() */
1648         down_write(&pernet_ops_rwsem);
1649         rtnl_lock();
1650         err = raw_notifier_chain_register(&netdev_chain, nb);
1651         if (err)
1652                 goto unlock;
1653         if (dev_boot_phase)
1654                 goto unlock;
1655         for_each_net(net) {
1656                 for_each_netdev(net, dev) {
1657                         err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1658                         err = notifier_to_errno(err);
1659                         if (err)
1660                                 goto rollback;
1661 
1662                         if (!(dev->flags & IFF_UP))
1663                                 continue;
1664 
1665                         call_netdevice_notifier(nb, NETDEV_UP, dev);
1666                 }
1667         }
1668 
1669 unlock:
1670         rtnl_unlock();
1671         up_write(&pernet_ops_rwsem);
1672         return err;
1673 
1674 rollback:
1675         last = dev;
1676         for_each_net(net) {
1677                 for_each_netdev(net, dev) {
1678                         if (dev == last)
1679                                 goto outroll;
1680 
1681                         if (dev->flags & IFF_UP) {
1682                                 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1683                                                         dev);
1684                                 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1685                         }
1686                         call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1687                 }
1688         }
1689 
1690 outroll:
1691         raw_notifier_chain_unregister(&netdev_chain, nb);
1692         goto unlock;
1693 }
1694 EXPORT_SYMBOL(register_netdevice_notifier);
1695 
1696 /**
1697  * unregister_netdevice_notifier - unregister a network notifier block
1698  * @nb: notifier
1699  *
1700  * Unregister a notifier previously registered by
1701  * register_netdevice_notifier(). The notifier is unlinked into the
1702  * kernel structures and may then be reused. A negative errno code
1703  * is returned on a failure.
1704  *
1705  * After unregistering unregister and down device events are synthesized
1706  * for all devices on the device list to the removed notifier to remove
1707  * the need for special case cleanup code.
1708  */
1709 
1710 int unregister_netdevice_notifier(struct notifier_block *nb)
1711 {
1712         struct net_device *dev;
1713         struct net *net;
1714         int err;
1715 
1716         /* Close race with setup_net() and cleanup_net() */
1717         down_write(&pernet_ops_rwsem);
1718         rtnl_lock();
1719         err = raw_notifier_chain_unregister(&netdev_chain, nb);
1720         if (err)
1721                 goto unlock;
1722 
1723         for_each_net(net) {
1724                 for_each_netdev(net, dev) {
1725                         if (dev->flags & IFF_UP) {
1726                                 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1727                                                         dev);
1728                                 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1729                         }
1730                         call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1731                 }
1732         }
1733 unlock:
1734         rtnl_unlock();
1735         up_write(&pernet_ops_rwsem);
1736         return err;
1737 }
1738 EXPORT_SYMBOL(unregister_netdevice_notifier);
1739 
1740 /**
1741  *      call_netdevice_notifiers_info - call all network notifier blocks
1742  *      @val: value passed unmodified to notifier function
1743  *      @info: notifier information data
1744  *
1745  *      Call all network notifier blocks.  Parameters and return value
1746  *      are as for raw_notifier_call_chain().
1747  */
1748 
1749 static int call_netdevice_notifiers_info(unsigned long val,
1750                                          struct netdev_notifier_info *info)
1751 {
1752         ASSERT_RTNL();
1753         return raw_notifier_call_chain(&netdev_chain, val, info);
1754 }
1755 
1756 static int call_netdevice_notifiers_extack(unsigned long val,
1757                                            struct net_device *dev,
1758                                            struct netlink_ext_ack *extack)
1759 {
1760         struct netdev_notifier_info info = {
1761                 .dev = dev,
1762                 .extack = extack,
1763         };
1764 
1765         return call_netdevice_notifiers_info(val, &info);
1766 }
1767 
1768 /**
1769  *      call_netdevice_notifiers - call all network notifier blocks
1770  *      @val: value passed unmodified to notifier function
1771  *      @dev: net_device pointer passed unmodified to notifier function
1772  *
1773  *      Call all network notifier blocks.  Parameters and return value
1774  *      are as for raw_notifier_call_chain().
1775  */
1776 
1777 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1778 {
1779         return call_netdevice_notifiers_extack(val, dev, NULL);
1780 }
1781 EXPORT_SYMBOL(call_netdevice_notifiers);
1782 
1783 /**
1784  *      call_netdevice_notifiers_mtu - call all network notifier blocks
1785  *      @val: value passed unmodified to notifier function
1786  *      @dev: net_device pointer passed unmodified to notifier function
1787  *      @arg: additional u32 argument passed to the notifier function
1788  *
1789  *      Call all network notifier blocks.  Parameters and return value
1790  *      are as for raw_notifier_call_chain().
1791  */
1792 static int call_netdevice_notifiers_mtu(unsigned long val,
1793                                         struct net_device *dev, u32 arg)
1794 {
1795         struct netdev_notifier_info_ext info = {
1796                 .info.dev = dev,
1797                 .ext.mtu = arg,
1798         };
1799 
1800         BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
1801 
1802         return call_netdevice_notifiers_info(val, &info.info);
1803 }
1804 
1805 #ifdef CONFIG_NET_INGRESS
1806 static DEFINE_STATIC_KEY_FALSE(ingress_needed_key);
1807 
1808 void net_inc_ingress_queue(void)
1809 {
1810         static_branch_inc(&ingress_needed_key);
1811 }
1812 EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1813 
1814 void net_dec_ingress_queue(void)
1815 {
1816         static_branch_dec(&ingress_needed_key);
1817 }
1818 EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1819 #endif
1820 
1821 #ifdef CONFIG_NET_EGRESS
1822 static DEFINE_STATIC_KEY_FALSE(egress_needed_key);
1823 
1824 void net_inc_egress_queue(void)
1825 {
1826         static_branch_inc(&egress_needed_key);
1827 }
1828 EXPORT_SYMBOL_GPL(net_inc_egress_queue);
1829 
1830 void net_dec_egress_queue(void)
1831 {
1832         static_branch_dec(&egress_needed_key);
1833 }
1834 EXPORT_SYMBOL_GPL(net_dec_egress_queue);
1835 #endif
1836 
1837 static DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
1838 #ifdef CONFIG_JUMP_LABEL
1839 static atomic_t netstamp_needed_deferred;
1840 static atomic_t netstamp_wanted;
1841 static void netstamp_clear(struct work_struct *work)
1842 {
1843         int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1844         int wanted;
1845 
1846         wanted = atomic_add_return(deferred, &netstamp_wanted);
1847         if (wanted > 0)
1848                 static_branch_enable(&netstamp_needed_key);
1849         else
1850                 static_branch_disable(&netstamp_needed_key);
1851 }
1852 static DECLARE_WORK(netstamp_work, netstamp_clear);
1853 #endif
1854 
1855 void net_enable_timestamp(void)
1856 {
1857 #ifdef CONFIG_JUMP_LABEL
1858         int wanted;
1859 
1860         while (1) {
1861                 wanted = atomic_read(&netstamp_wanted);
1862                 if (wanted <= 0)
1863                         break;
1864                 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
1865                         return;
1866         }
1867         atomic_inc(&netstamp_needed_deferred);
1868         schedule_work(&netstamp_work);
1869 #else
1870         static_branch_inc(&netstamp_needed_key);
1871 #endif
1872 }
1873 EXPORT_SYMBOL(net_enable_timestamp);
1874 
1875 void net_disable_timestamp(void)
1876 {
1877 #ifdef CONFIG_JUMP_LABEL
1878         int wanted;
1879 
1880         while (1) {
1881                 wanted = atomic_read(&netstamp_wanted);
1882                 if (wanted <= 1)
1883                         break;
1884                 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
1885                         return;
1886         }
1887         atomic_dec(&netstamp_needed_deferred);
1888         schedule_work(&netstamp_work);
1889 #else
1890         static_branch_dec(&netstamp_needed_key);
1891 #endif
1892 }
1893 EXPORT_SYMBOL(net_disable_timestamp);
1894 
1895 static inline void net_timestamp_set(struct sk_buff *skb)
1896 {
1897         skb->tstamp = 0;
1898         if (static_branch_unlikely(&netstamp_needed_key))
1899                 __net_timestamp(skb);
1900 }
1901 
1902 #define net_timestamp_check(COND, SKB)                          \
1903         if (static_branch_unlikely(&netstamp_needed_key)) {     \
1904                 if ((COND) && !(SKB)->tstamp)                   \
1905                         __net_timestamp(SKB);                   \
1906         }                                                       \
1907 
1908 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
1909 {
1910         unsigned int len;
1911 
1912         if (!(dev->flags & IFF_UP))
1913                 return false;
1914 
1915         len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1916         if (skb->len <= len)
1917                 return true;
1918 
1919         /* if TSO is enabled, we don't care about the length as the packet
1920          * could be forwarded without being segmented before
1921          */
1922         if (skb_is_gso(skb))
1923                 return true;
1924 
1925         return false;
1926 }
1927 EXPORT_SYMBOL_GPL(is_skb_forwardable);
1928 
1929 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1930 {
1931         int ret = ____dev_forward_skb(dev, skb);
1932 
1933         if (likely(!ret)) {
1934                 skb->protocol = eth_type_trans(skb, dev);
1935                 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
1936         }
1937 
1938         return ret;
1939 }
1940 EXPORT_SYMBOL_GPL(__dev_forward_skb);
1941 
1942 /**
1943  * dev_forward_skb - loopback an skb to another netif
1944  *
1945  * @dev: destination network device
1946  * @skb: buffer to forward
1947  *
1948  * return values:
1949  *      NET_RX_SUCCESS  (no congestion)
1950  *      NET_RX_DROP     (packet was dropped, but freed)
1951  *
1952  * dev_forward_skb can be used for injecting an skb from the
1953  * start_xmit function of one device into the receive queue
1954  * of another device.
1955  *
1956  * The receiving device may be in another namespace, so
1957  * we have to clear all information in the skb that could
1958  * impact namespace isolation.
1959  */
1960 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1961 {
1962         return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
1963 }
1964 EXPORT_SYMBOL_GPL(dev_forward_skb);
1965 
1966 static inline int deliver_skb(struct sk_buff *skb,
1967                               struct packet_type *pt_prev,
1968                               struct net_device *orig_dev)
1969 {
1970         if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
1971                 return -ENOMEM;
1972         refcount_inc(&skb->users);
1973         return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1974 }
1975 
1976 static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1977                                           struct packet_type **pt,
1978                                           struct net_device *orig_dev,
1979                                           __be16 type,
1980                                           struct list_head *ptype_list)
1981 {
1982         struct packet_type *ptype, *pt_prev = *pt;
1983 
1984         list_for_each_entry_rcu(ptype, ptype_list, list) {
1985                 if (ptype->type != type)
1986                         continue;
1987                 if (pt_prev)
1988                         deliver_skb(skb, pt_prev, orig_dev);
1989                 pt_prev = ptype;
1990         }
1991         *pt = pt_prev;
1992 }
1993 
1994 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1995 {
1996         if (!ptype->af_packet_priv || !skb->sk)
1997                 return false;
1998 
1999         if (ptype->id_match)
2000                 return ptype->id_match(ptype, skb->sk);
2001         else if ((struct sock *)ptype->af_packet_priv == skb->sk)
2002                 return true;
2003 
2004         return false;
2005 }
2006 
2007 /**
2008  * dev_nit_active - return true if any network interface taps are in use
2009  *
2010  * @dev: network device to check for the presence of taps
2011  */
2012 bool dev_nit_active(struct net_device *dev)
2013 {
2014         return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all);
2015 }
2016 EXPORT_SYMBOL_GPL(dev_nit_active);
2017 
2018 /*
2019  *      Support routine. Sends outgoing frames to any network
2020  *      taps currently in use.
2021  */
2022 
2023 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
2024 {
2025         struct packet_type *ptype;
2026         struct sk_buff *skb2 = NULL;
2027         struct packet_type *pt_prev = NULL;
2028         struct list_head *ptype_list = &ptype_all;
2029 
2030         rcu_read_lock();
2031 again:
2032         list_for_each_entry_rcu(ptype, ptype_list, list) {
2033                 if (ptype->ignore_outgoing)
2034                         continue;
2035 
2036                 /* Never send packets back to the socket
2037                  * they originated from - MvS (miquels@drinkel.ow.org)
2038                  */
2039                 if (skb_loop_sk(ptype, skb))
2040                         continue;
2041 
2042                 if (pt_prev) {
2043                         deliver_skb(skb2, pt_prev, skb->dev);
2044                         pt_prev = ptype;
2045                         continue;
2046                 }
2047 
2048                 /* need to clone skb, done only once */
2049                 skb2 = skb_clone(skb, GFP_ATOMIC);
2050                 if (!skb2)
2051                         goto out_unlock;
2052 
2053                 net_timestamp_set(skb2);
2054 
2055                 /* skb->nh should be correctly
2056                  * set by sender, so that the second statement is
2057                  * just protection against buggy protocols.
2058                  */
2059                 skb_reset_mac_header(skb2);
2060 
2061                 if (skb_network_header(skb2) < skb2->data ||
2062                     skb_network_header(skb2) > skb_tail_pointer(skb2)) {
2063                         net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
2064                                              ntohs(skb2->protocol),
2065                                              dev->name);
2066                         skb_reset_network_header(skb2);
2067                 }
2068 
2069                 skb2->transport_header = skb2->network_header;
2070                 skb2->pkt_type = PACKET_OUTGOING;
2071                 pt_prev = ptype;
2072         }
2073 
2074         if (ptype_list == &ptype_all) {
2075                 ptype_list = &dev->ptype_all;
2076                 goto again;
2077         }
2078 out_unlock:
2079         if (pt_prev) {
2080                 if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
2081                         pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
2082                 else
2083                         kfree_skb(skb2);
2084         }
2085         rcu_read_unlock();
2086 }
2087 EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
2088 
2089 /**
2090  * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
2091  * @dev: Network device
2092  * @txq: number of queues available
2093  *
2094  * If real_num_tx_queues is changed the tc mappings may no longer be
2095  * valid. To resolve this verify the tc mapping remains valid and if
2096  * not NULL the mapping. With no priorities mapping to this
2097  * offset/count pair it will no longer be used. In the worst case TC0
2098  * is invalid nothing can be done so disable priority mappings. If is
2099  * expected that drivers will fix this mapping if they can before
2100  * calling netif_set_real_num_tx_queues.
2101  */
2102 static void netif_setup_tc(struct net_device *dev, unsigned int txq)
2103 {
2104         int i;
2105         struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2106 
2107         /* If TC0 is invalidated disable TC mapping */
2108         if (tc->offset + tc->count > txq) {
2109                 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
2110                 dev->num_tc = 0;
2111                 return;
2112         }
2113 
2114         /* Invalidated prio to tc mappings set to TC0 */
2115         for (i = 1; i < TC_BITMASK + 1; i++) {
2116                 int q = netdev_get_prio_tc_map(dev, i);
2117 
2118                 tc = &dev->tc_to_txq[q];
2119                 if (tc->offset + tc->count > txq) {
2120                         pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2121                                 i, q);
2122                         netdev_set_prio_tc_map(dev, i, 0);
2123                 }
2124         }
2125 }
2126 
2127 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2128 {
2129         if (dev->num_tc) {
2130                 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2131                 int i;
2132 
2133                 /* walk through the TCs and see if it falls into any of them */
2134                 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2135                         if ((txq - tc->offset) < tc->count)
2136                                 return i;
2137                 }
2138 
2139                 /* didn't find it, just return -1 to indicate no match */
2140                 return -1;
2141         }
2142 
2143         return 0;
2144 }
2145 EXPORT_SYMBOL(netdev_txq_to_tc);
2146 
2147 #ifdef CONFIG_XPS
2148 struct static_key xps_needed __read_mostly;
2149 EXPORT_SYMBOL(xps_needed);
2150 struct static_key xps_rxqs_needed __read_mostly;
2151 EXPORT_SYMBOL(xps_rxqs_needed);
2152 static DEFINE_MUTEX(xps_map_mutex);
2153 #define xmap_dereference(P)             \
2154         rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2155 
2156 static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2157                              int tci, u16 index)
2158 {
2159         struct xps_map *map = NULL;
2160         int pos;
2161 
2162         if (dev_maps)
2163                 map = xmap_dereference(dev_maps->attr_map[tci]);
2164         if (!map)
2165                 return false;
2166 
2167         for (pos = map->len; pos--;) {
2168                 if (map->queues[pos] != index)
2169                         continue;
2170 
2171                 if (map->len > 1) {
2172                         map->queues[pos] = map->queues[--map->len];
2173                         break;
2174                 }
2175 
2176                 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2177                 kfree_rcu(map, rcu);
2178                 return false;
2179         }
2180 
2181         return true;
2182 }
2183 
2184 static bool remove_xps_queue_cpu(struct net_device *dev,
2185                                  struct xps_dev_maps *dev_maps,
2186                                  int cpu, u16 offset, u16 count)
2187 {
2188         int num_tc = dev->num_tc ? : 1;
2189         bool active = false;
2190         int tci;
2191 
2192         for (tci = cpu * num_tc; num_tc--; tci++) {
2193                 int i, j;
2194 
2195                 for (i = count, j = offset; i--; j++) {
2196                         if (!remove_xps_queue(dev_maps, tci, j))
2197                                 break;
2198                 }
2199 
2200                 active |= i < 0;
2201         }
2202 
2203         return active;
2204 }
2205 
2206 static void reset_xps_maps(struct net_device *dev,
2207                            struct xps_dev_maps *dev_maps,
2208                            bool is_rxqs_map)
2209 {
2210         if (is_rxqs_map) {
2211                 static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
2212                 RCU_INIT_POINTER(dev->xps_rxqs_map, NULL);
2213         } else {
2214                 RCU_INIT_POINTER(dev->xps_cpus_map, NULL);
2215         }
2216         static_key_slow_dec_cpuslocked(&xps_needed);
2217         kfree_rcu(dev_maps, rcu);
2218 }
2219 
2220 static void clean_xps_maps(struct net_device *dev, const unsigned long *mask,
2221                            struct xps_dev_maps *dev_maps, unsigned int nr_ids,
2222                            u16 offset, u16 count, bool is_rxqs_map)
2223 {
2224         bool active = false;
2225         int i, j;
2226 
2227         for (j = -1; j = netif_attrmask_next(j, mask, nr_ids),
2228              j < nr_ids;)
2229                 active |= remove_xps_queue_cpu(dev, dev_maps, j, offset,
2230                                                count);
2231         if (!active)
2232                 reset_xps_maps(dev, dev_maps, is_rxqs_map);
2233 
2234         if (!is_rxqs_map) {
2235                 for (i = offset + (count - 1); count--; i--) {
2236                         netdev_queue_numa_node_write(
2237                                 netdev_get_tx_queue(dev, i),
2238                                 NUMA_NO_NODE);
2239                 }
2240         }
2241 }
2242 
2243 static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2244                                    u16 count)
2245 {
2246         const unsigned long *possible_mask = NULL;
2247         struct xps_dev_maps *dev_maps;
2248         unsigned int nr_ids;
2249 
2250         if (!static_key_false(&xps_needed))
2251                 return;
2252 
2253         cpus_read_lock();
2254         mutex_lock(&xps_map_mutex);
2255 
2256         if (static_key_false(&xps_rxqs_needed)) {
2257                 dev_maps = xmap_dereference(dev->xps_rxqs_map);
2258                 if (dev_maps) {
2259                         nr_ids = dev->num_rx_queues;
2260                         clean_xps_maps(dev, possible_mask, dev_maps, nr_ids,
2261                                        offset, count, true);
2262                 }
2263         }
2264 
2265         dev_maps = xmap_dereference(dev->xps_cpus_map);
2266         if (!dev_maps)
2267                 goto out_no_maps;
2268 
2269         if (num_possible_cpus() > 1)
2270                 possible_mask = cpumask_bits(cpu_possible_mask);
2271         nr_ids = nr_cpu_ids;
2272         clean_xps_maps(dev, possible_mask, dev_maps, nr_ids, offset, count,
2273                        false);
2274 
2275 out_no_maps:
2276         mutex_unlock(&xps_map_mutex);
2277         cpus_read_unlock();
2278 }
2279 
2280 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2281 {
2282         netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2283 }
2284 
2285 static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index,
2286                                       u16 index, bool is_rxqs_map)
2287 {
2288         struct xps_map *new_map;
2289         int alloc_len = XPS_MIN_MAP_ALLOC;
2290         int i, pos;
2291 
2292         for (pos = 0; map && pos < map->len; pos++) {
2293                 if (map->queues[pos] != index)
2294                         continue;
2295                 return map;
2296         }
2297 
2298         /* Need to add tx-queue to this CPU's/rx-queue's existing map */
2299         if (map) {
2300                 if (pos < map->alloc_len)
2301                         return map;
2302 
2303                 alloc_len = map->alloc_len * 2;
2304         }
2305 
2306         /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's
2307          *  map
2308          */
2309         if (is_rxqs_map)
2310                 new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL);
2311         else
2312                 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2313                                        cpu_to_node(attr_index));
2314         if (!new_map)
2315                 return NULL;
2316 
2317         for (i = 0; i < pos; i++)
2318                 new_map->queues[i] = map->queues[i];
2319         new_map->alloc_len = alloc_len;
2320         new_map->len = pos;
2321 
2322         return new_map;
2323 }
2324 
2325 /* Must be called under cpus_read_lock */
2326 int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
2327                           u16 index, bool is_rxqs_map)
2328 {
2329         const unsigned long *online_mask = NULL, *possible_mask = NULL;
2330         struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
2331         int i, j, tci, numa_node_id = -2;
2332         int maps_sz, num_tc = 1, tc = 0;
2333         struct xps_map *map, *new_map;
2334         bool active = false;
2335         unsigned int nr_ids;
2336 
2337         if (dev->num_tc) {
2338                 /* Do not allow XPS on subordinate device directly */
2339                 num_tc = dev->num_tc;
2340                 if (num_tc < 0)
2341                         return -EINVAL;
2342 
2343                 /* If queue belongs to subordinate dev use its map */
2344                 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
2345 
2346                 tc = netdev_txq_to_tc(dev, index);
2347                 if (tc < 0)
2348                         return -EINVAL;
2349         }
2350 
2351         mutex_lock(&xps_map_mutex);
2352         if (is_rxqs_map) {
2353                 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
2354                 dev_maps = xmap_dereference(dev->xps_rxqs_map);
2355                 nr_ids = dev->num_rx_queues;
2356         } else {
2357                 maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
2358                 if (num_possible_cpus() > 1) {
2359                         online_mask = cpumask_bits(cpu_online_mask);
2360                         possible_mask = cpumask_bits(cpu_possible_mask);
2361                 }
2362                 dev_maps = xmap_dereference(dev->xps_cpus_map);
2363                 nr_ids = nr_cpu_ids;
2364         }
2365 
2366         if (maps_sz < L1_CACHE_BYTES)
2367                 maps_sz = L1_CACHE_BYTES;
2368 
2369         /* allocate memory for queue storage */
2370         for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
2371              j < nr_ids;) {
2372                 if (!new_dev_maps)
2373                         new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2374                 if (!new_dev_maps) {
2375                         mutex_unlock(&xps_map_mutex);
2376                         return -ENOMEM;
2377                 }
2378 
2379                 tci = j * num_tc + tc;
2380                 map = dev_maps ? xmap_dereference(dev_maps->attr_map[tci]) :
2381                                  NULL;
2382 
2383                 map = expand_xps_map(map, j, index, is_rxqs_map);
2384                 if (!map)
2385                         goto error;
2386 
2387                 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2388         }
2389 
2390         if (!new_dev_maps)
2391                 goto out_no_new_maps;
2392 
2393         if (!dev_maps) {
2394                 /* Increment static keys at most once per type */
2395                 static_key_slow_inc_cpuslocked(&xps_needed);
2396                 if (is_rxqs_map)
2397                         static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
2398         }
2399 
2400         for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2401              j < nr_ids;) {
2402                 /* copy maps belonging to foreign traffic classes */
2403                 for (i = tc, tci = j * num_tc; dev_maps && i--; tci++) {
2404                         /* fill in the new device map from the old device map */
2405                         map = xmap_dereference(dev_maps->attr_map[tci]);
2406                         RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2407                 }
2408 
2409                 /* We need to explicitly update tci as prevous loop
2410                  * could break out early if dev_maps is NULL.
2411                  */
2412                 tci = j * num_tc + tc;
2413 
2414                 if (netif_attr_test_mask(j, mask, nr_ids) &&
2415                     netif_attr_test_online(j, online_mask, nr_ids)) {
2416                         /* add tx-queue to CPU/rx-queue maps */
2417                         int pos = 0;
2418 
2419                         map = xmap_dereference(new_dev_maps->attr_map[tci]);
2420                         while ((pos < map->len) && (map->queues[pos] != index))
2421                                 pos++;
2422 
2423                         if (pos == map->len)
2424                                 map->queues[map->len++] = index;
2425 #ifdef CONFIG_NUMA
2426                         if (!is_rxqs_map) {
2427                                 if (numa_node_id == -2)
2428                                         numa_node_id = cpu_to_node(j);
2429                                 else if (numa_node_id != cpu_to_node(j))
2430                                         numa_node_id = -1;
2431                         }
2432 #endif
2433                 } else if (dev_maps) {
2434                         /* fill in the new device map from the old device map */
2435                         map = xmap_dereference(dev_maps->attr_map[tci]);
2436                         RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2437                 }
2438 
2439                 /* copy maps belonging to foreign traffic classes */
2440                 for (i = num_tc - tc, tci++; dev_maps && --i; tci++) {
2441                         /* fill in the new device map from the old device map */
2442                         map = xmap_dereference(dev_maps->attr_map[tci]);
2443                         RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2444                 }
2445         }
2446 
2447         if (is_rxqs_map)
2448                 rcu_assign_pointer(dev->xps_rxqs_map, new_dev_maps);
2449         else
2450                 rcu_assign_pointer(dev->xps_cpus_map, new_dev_maps);
2451 
2452         /* Cleanup old maps */
2453         if (!dev_maps)
2454                 goto out_no_old_maps;
2455 
2456         for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2457              j < nr_ids;) {
2458                 for (i = num_tc, tci = j * num_tc; i--; tci++) {
2459                         new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2460                         map = xmap_dereference(dev_maps->attr_map[tci]);
2461                         if (map && map != new_map)
2462                                 kfree_rcu(map, rcu);
2463                 }
2464         }
2465 
2466         kfree_rcu(dev_maps, rcu);
2467 
2468 out_no_old_maps:
2469         dev_maps = new_dev_maps;
2470         active = true;
2471 
2472 out_no_new_maps:
2473         if (!is_rxqs_map) {
2474                 /* update Tx queue numa node */
2475                 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2476                                              (numa_node_id >= 0) ?
2477                                              numa_node_id : NUMA_NO_NODE);
2478         }
2479 
2480         if (!dev_maps)
2481                 goto out_no_maps;
2482 
2483         /* removes tx-queue from unused CPUs/rx-queues */
2484         for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2485              j < nr_ids;) {
2486                 for (i = tc, tci = j * num_tc; i--; tci++)
2487                         active |= remove_xps_queue(dev_maps, tci, index);
2488                 if (!netif_attr_test_mask(j, mask, nr_ids) ||
2489                     !netif_attr_test_online(j, online_mask, nr_ids))
2490                         active |= remove_xps_queue(dev_maps, tci, index);
2491                 for (i = num_tc - tc, tci++; --i; tci++)
2492                         active |= remove_xps_queue(dev_maps, tci, index);
2493         }
2494 
2495         /* free map if not active */
2496         if (!active)
2497                 reset_xps_maps(dev, dev_maps, is_rxqs_map);
2498 
2499 out_no_maps:
2500         mutex_unlock(&xps_map_mutex);
2501 
2502         return 0;
2503 error:
2504         /* remove any maps that we added */
2505         for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2506              j < nr_ids;) {
2507                 for (i = num_tc, tci = j * num_tc; i--; tci++) {
2508                         new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2509                         map = dev_maps ?
2510                               xmap_dereference(dev_maps->attr_map[tci]) :
2511                               NULL;
2512                         if (new_map && new_map != map)
2513                                 kfree(new_map);
2514                 }
2515         }
2516 
2517         mutex_unlock(&xps_map_mutex);
2518 
2519         kfree(new_dev_maps);
2520         return -ENOMEM;
2521 }
2522 EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
2523 
2524 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2525                         u16 index)
2526 {
2527         int ret;
2528 
2529         cpus_read_lock();
2530         ret =  __netif_set_xps_queue(dev, cpumask_bits(mask), index, false);
2531         cpus_read_unlock();
2532 
2533         return ret;
2534 }
2535 EXPORT_SYMBOL(netif_set_xps_queue);
2536 
2537 #endif
2538 static void netdev_unbind_all_sb_channels(struct net_device *dev)
2539 {
2540         struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2541 
2542         /* Unbind any subordinate channels */
2543         while (txq-- != &dev->_tx[0]) {
2544                 if (txq->sb_dev)
2545                         netdev_unbind_sb_channel(dev, txq->sb_dev);
2546         }
2547 }
2548 
2549 void netdev_reset_tc(struct net_device *dev)
2550 {
2551 #ifdef CONFIG_XPS
2552         netif_reset_xps_queues_gt(dev, 0);
2553 #endif
2554         netdev_unbind_all_sb_channels(dev);
2555 
2556         /* Reset TC configuration of device */
2557         dev->num_tc = 0;
2558         memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2559         memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2560 }
2561 EXPORT_SYMBOL(netdev_reset_tc);
2562 
2563 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2564 {
2565         if (tc >= dev->num_tc)
2566                 return -EINVAL;
2567 
2568 #ifdef CONFIG_XPS
2569         netif_reset_xps_queues(dev, offset, count);
2570 #endif
2571         dev->tc_to_txq[tc].count = count;
2572         dev->tc_to_txq[tc].offset = offset;
2573         return 0;
2574 }
2575 EXPORT_SYMBOL(netdev_set_tc_queue);
2576 
2577 int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2578 {
2579         if (num_tc > TC_MAX_QUEUE)
2580                 return -EINVAL;
2581 
2582 #ifdef CONFIG_XPS
2583         netif_reset_xps_queues_gt(dev, 0);
2584 #endif
2585         netdev_unbind_all_sb_channels(dev);
2586 
2587         dev->num_tc = num_tc;
2588         return 0;
2589 }
2590 EXPORT_SYMBOL(netdev_set_num_tc);
2591 
2592 void netdev_unbind_sb_channel(struct net_device *dev,
2593                               struct net_device *sb_dev)
2594 {
2595         struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2596 
2597 #ifdef CONFIG_XPS
2598         netif_reset_xps_queues_gt(sb_dev, 0);
2599 #endif
2600         memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq));
2601         memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map));
2602 
2603         while (txq-- != &dev->_tx[0]) {
2604                 if (txq->sb_dev == sb_dev)
2605                         txq->sb_dev = NULL;
2606         }
2607 }
2608 EXPORT_SYMBOL(netdev_unbind_sb_channel);
2609 
2610 int netdev_bind_sb_channel_queue(struct net_device *dev,
2611                                  struct net_device *sb_dev,
2612                                  u8 tc, u16 count, u16 offset)
2613 {
2614         /* Make certain the sb_dev and dev are already configured */
2615         if (sb_dev->num_tc >= 0 || tc >= dev->num_tc)
2616                 return -EINVAL;
2617 
2618         /* We cannot hand out queues we don't have */
2619         if ((offset + count) > dev->real_num_tx_queues)
2620                 return -EINVAL;
2621 
2622         /* Record the mapping */
2623         sb_dev->tc_to_txq[tc].count = count;
2624         sb_dev->tc_to_txq[tc].offset = offset;
2625 
2626         /* Provide a way for Tx queue to find the tc_to_txq map or
2627          * XPS map for itself.
2628          */
2629         while (count--)
2630                 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev;
2631 
2632         return 0;
2633 }
2634 EXPORT_SYMBOL(netdev_bind_sb_channel_queue);
2635 
2636 int netdev_set_sb_channel(struct net_device *dev, u16 channel)
2637 {
2638         /* Do not use a multiqueue device to represent a subordinate channel */
2639         if (netif_is_multiqueue(dev))
2640                 return -ENODEV;
2641 
2642         /* We allow channels 1 - 32767 to be used for subordinate channels.
2643          * Channel 0 is meant to be "native" mode and used only to represent
2644          * the main root device. We allow writing 0 to reset the device back
2645          * to normal mode after being used as a subordinate channel.
2646          */
2647         if (channel > S16_MAX)
2648                 return -EINVAL;
2649 
2650         dev->num_tc = -channel;
2651 
2652         return 0;
2653 }
2654 EXPORT_SYMBOL(netdev_set_sb_channel);
2655 
2656 /*
2657  * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2658  * greater than real_num_tx_queues stale skbs on the qdisc must be flushed.
2659  */
2660 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2661 {
2662         bool disabling;
2663         int rc;
2664 
2665         disabling = txq < dev->real_num_tx_queues;
2666 
2667         if (txq < 1 || txq > dev->num_tx_queues)
2668                 return -EINVAL;
2669 
2670         if (dev->reg_state == NETREG_REGISTERED ||
2671             dev->reg_state == NETREG_UNREGISTERING) {
2672                 ASSERT_RTNL();
2673 
2674                 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2675                                                   txq);
2676                 if (rc)
2677                         return rc;
2678 
2679                 if (dev->num_tc)
2680                         netif_setup_tc(dev, txq);
2681 
2682                 dev->real_num_tx_queues = txq;
2683 
2684                 if (disabling) {
2685                         synchronize_net();
2686                         qdisc_reset_all_tx_gt(dev, txq);
2687 #ifdef CONFIG_XPS
2688                         netif_reset_xps_queues_gt(dev, txq);
2689 #endif
2690                 }
2691         } else {
2692                 dev->real_num_tx_queues = txq;
2693         }
2694 
2695         return 0;
2696 }
2697 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2698 
2699 #ifdef CONFIG_SYSFS
2700 /**
2701  *      netif_set_real_num_rx_queues - set actual number of RX queues used
2702  *      @dev: Network device
2703  *      @rxq: Actual number of RX queues
2704  *
2705  *      This must be called either with the rtnl_lock held or before
2706  *      registration of the net device.  Returns 0 on success, or a
2707  *      negative error code.  If called before registration, it always
2708  *      succeeds.
2709  */
2710 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2711 {
2712         int rc;
2713 
2714         if (rxq < 1 || rxq > dev->num_rx_queues)
2715                 return -EINVAL;
2716 
2717         if (dev->reg_state == NETREG_REGISTERED) {
2718                 ASSERT_RTNL();
2719 
2720                 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2721                                                   rxq);
2722                 if (rc)
2723                         return rc;
2724         }
2725 
2726         dev->real_num_rx_queues = rxq;
2727         return 0;
2728 }
2729 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2730 #endif
2731 
2732 /**
2733  * netif_get_num_default_rss_queues - default number of RSS queues
2734  *
2735  * This routine should set an upper limit on the number of RSS queues
2736  * used by default by multiqueue devices.
2737  */
2738 int netif_get_num_default_rss_queues(void)
2739 {
2740         return is_kdump_kernel() ?
2741                 1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2742 }
2743 EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2744 
2745 static void __netif_reschedule(struct Qdisc *q)
2746 {
2747         struct softnet_data *sd;
2748         unsigned long flags;
2749 
2750         local_irq_save(flags);
2751         sd = this_cpu_ptr(&softnet_data);
2752         q->next_sched = NULL;
2753         *sd->output_queue_tailp = q;
2754         sd->output_queue_tailp = &q->next_sched;
2755         raise_softirq_irqoff(NET_TX_SOFTIRQ);
2756         local_irq_restore(flags);
2757 }
2758 
2759 void __netif_schedule(struct Qdisc *q)
2760 {
2761         if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2762                 __netif_reschedule(q);
2763 }
2764 EXPORT_SYMBOL(__netif_schedule);
2765 
2766 struct dev_kfree_skb_cb {
2767         enum skb_free_reason reason;
2768 };
2769 
2770 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
2771 {
2772         return (struct dev_kfree_skb_cb *)skb->cb;
2773 }
2774 
2775 void netif_schedule_queue(struct netdev_queue *txq)
2776 {
2777         rcu_read_lock();
2778         if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2779                 struct Qdisc *q = rcu_dereference(txq->qdisc);
2780 
2781                 __netif_schedule(q);
2782         }
2783         rcu_read_unlock();
2784 }
2785 EXPORT_SYMBOL(netif_schedule_queue);
2786 
2787 void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2788 {
2789         if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2790                 struct Qdisc *q;
2791 
2792                 rcu_read_lock();
2793                 q = rcu_dereference(dev_queue->qdisc);
2794                 __netif_schedule(q);
2795                 rcu_read_unlock();
2796         }
2797 }
2798 EXPORT_SYMBOL(netif_tx_wake_queue);
2799 
2800 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2801 {
2802         unsigned long flags;
2803 
2804         if (unlikely(!skb))
2805                 return;
2806 
2807         if (likely(refcount_read(&skb->users) == 1)) {
2808                 smp_rmb();
2809                 refcount_set(&skb->users, 0);
2810         } else if (likely(!refcount_dec_and_test(&skb->users))) {
2811                 return;
2812         }
2813         get_kfree_skb_cb(skb)->reason = reason;
2814         local_irq_save(flags);
2815         skb->next = __this_cpu_read(softnet_data.completion_queue);
2816         __this_cpu_write(softnet_data.completion_queue, skb);
2817         raise_softirq_irqoff(NET_TX_SOFTIRQ);
2818         local_irq_restore(flags);
2819 }
2820 EXPORT_SYMBOL(__dev_kfree_skb_irq);
2821 
2822 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
2823 {
2824         if (in_irq() || irqs_disabled())
2825                 __dev_kfree_skb_irq(skb, reason);
2826         else
2827                 dev_kfree_skb(skb);
2828 }
2829 EXPORT_SYMBOL(__dev_kfree_skb_any);
2830 
2831 
2832 /**
2833  * netif_device_detach - mark device as removed
2834  * @dev: network device
2835  *
2836  * Mark device as removed from system and therefore no longer available.
2837  */
2838 void netif_device_detach(struct net_device *dev)
2839 {
2840         if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2841             netif_running(dev)) {
2842                 netif_tx_stop_all_queues(dev);
2843         }
2844 }
2845 EXPORT_SYMBOL(netif_device_detach);
2846 
2847 /**
2848  * netif_device_attach - mark device as attached
2849  * @dev: network device
2850  *
2851  * Mark device as attached from system and restart if needed.
2852  */
2853 void netif_device_attach(struct net_device *dev)
2854 {
2855         if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2856             netif_running(dev)) {
2857                 netif_tx_wake_all_queues(dev);
2858                 __netdev_watchdog_up(dev);
2859         }
2860 }
2861 EXPORT_SYMBOL(netif_device_attach);
2862 
2863 /*
2864  * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2865  * to be used as a distribution range.
2866  */
2867 static u16 skb_tx_hash(const struct net_device *dev,
2868                        const struct net_device *sb_dev,
2869                        struct sk_buff *skb)
2870 {
2871         u32 hash;
2872         u16 qoffset = 0;
2873         u16 qcount = dev->real_num_tx_queues;
2874 
2875         if (dev->num_tc) {
2876                 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2877 
2878                 qoffset = sb_dev->tc_to_txq[tc].offset;
2879                 qcount = sb_dev->tc_to_txq[tc].count;
2880         }
2881 
2882         if (skb_rx_queue_recorded(skb)) {
2883                 hash = skb_get_rx_queue(skb);
2884                 while (unlikely(hash >= qcount))
2885                         hash -= qcount;
2886                 return hash + qoffset;
2887         }
2888 
2889         return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2890 }
2891 
2892 static void skb_warn_bad_offload(const struct sk_buff *skb)
2893 {
2894         static const netdev_features_t null_features;
2895         struct net_device *dev = skb->dev;
2896         const char *name = "";
2897 
2898         if (!net_ratelimit())
2899                 return;
2900 
2901         if (dev) {
2902                 if (dev->dev.parent)
2903                         name = dev_driver_string(dev->dev.parent);
2904                 else
2905                         name = netdev_name(dev);
2906         }
2907         WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2908              "gso_type=%d ip_summed=%d\n",
2909              name, dev ? &dev->features : &null_features,
2910              skb->sk ? &skb->sk->sk_route_caps : &null_features,
2911              skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2912              skb_shinfo(skb)->gso_type, skb->ip_summed);
2913 }
2914 
2915 /*
2916  * Invalidate hardware checksum when packet is to be mangled, and
2917  * complete checksum manually on outgoing path.
2918  */
2919 int skb_checksum_help(struct sk_buff *skb)
2920 {
2921         __wsum csum;
2922         int ret = 0, offset;
2923 
2924         if (skb->ip_summed == CHECKSUM_COMPLETE)
2925                 goto out_set_summed;
2926 
2927         if (unlikely(skb_shinfo(skb)->gso_size)) {
2928                 skb_warn_bad_offload(skb);
2929                 return -EINVAL;
2930         }
2931 
2932         /* Before computing a checksum, we should make sure no frag could
2933          * be modified by an external entity : checksum could be wrong.
2934          */
2935         if (skb_has_shared_frag(skb)) {
2936                 ret = __skb_linearize(skb);
2937                 if (ret)
2938                         goto out;
2939         }
2940 
2941         offset = skb_checksum_start_offset(skb);
2942         BUG_ON(offset >= skb_headlen(skb));
2943         csum = skb_checksum(skb, offset, skb->len - offset, 0);
2944 
2945         offset += skb->csum_offset;
2946         BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2947 
2948         if (skb_cloned(skb) &&
2949             !skb_clone_writable(skb, offset + sizeof(__sum16))) {
2950                 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2951                 if (ret)
2952                         goto out;
2953         }
2954 
2955         *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
2956 out_set_summed:
2957         skb->ip_summed = CHECKSUM_NONE;
2958 out:
2959         return ret;
2960 }
2961 EXPORT_SYMBOL(skb_checksum_help);
2962 
2963 int skb_crc32c_csum_help(struct sk_buff *skb)
2964 {
2965         __le32 crc32c_csum;
2966         int ret = 0, offset, start;
2967 
2968         if (skb->ip_summed != CHECKSUM_PARTIAL)
2969                 goto out;
2970 
2971         if (unlikely(skb_is_gso(skb)))
2972                 goto out;
2973 
2974         /* Before computing a checksum, we should make sure no frag could
2975          * be modified by an external entity : checksum could be wrong.
2976          */
2977         if (unlikely(skb_has_shared_frag(skb))) {
2978                 ret = __skb_linearize(skb);
2979                 if (ret)
2980                         goto out;
2981         }
2982         start = skb_checksum_start_offset(skb);
2983         offset = start + offsetof(struct sctphdr, checksum);
2984         if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
2985                 ret = -EINVAL;
2986                 goto out;
2987         }
2988         if (skb_cloned(skb) &&
2989             !skb_clone_writable(skb, offset + sizeof(__le32))) {
2990                 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2991                 if (ret)
2992                         goto out;
2993         }
2994         crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
2995                                                   skb->len - start, ~(__u32)0,
2996                                                   crc32c_csum_stub));
2997         *(__le32 *)(skb->data + offset) = crc32c_csum;
2998         skb->ip_summed = CHECKSUM_NONE;
2999         skb->csum_not_inet = 0;
3000 out:
3001         return ret;
3002 }
3003 
3004 __be16 skb_network_protocol(struct sk_buff *skb, int *depth)
3005 {
3006         __be16 type = skb->protocol;
3007 
3008         /* Tunnel gso handlers can set protocol to ethernet. */
3009         if (type == htons(ETH_P_TEB)) {
3010                 struct ethhdr *eth;
3011 
3012                 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
3013                         return 0;
3014 
3015                 eth = (struct ethhdr *)skb->data;
3016                 type = eth->h_proto;
3017         }
3018 
3019         return __vlan_get_protocol(skb, type, depth);
3020 }
3021 
3022 /**
3023  *      skb_mac_gso_segment - mac layer segmentation handler.
3024  *      @skb: buffer to segment
3025  *      @features: features for the output path (see dev->features)
3026  */
3027 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
3028                                     netdev_features_t features)
3029 {
3030         struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
3031         struct packet_offload *ptype;
3032         int vlan_depth = skb->mac_len;
3033         __be16 type = skb_network_protocol(skb, &vlan_depth);
3034 
3035         if (unlikely(!type))
3036                 return ERR_PTR(-EINVAL);
3037 
3038         __skb_pull(skb, vlan_depth);
3039 
3040         rcu_read_lock();
3041         list_for_each_entry_rcu(ptype, &offload_base, list) {
3042                 if (ptype->type == type && ptype->callbacks.gso_segment) {
3043                         segs = ptype->callbacks.gso_segment(skb, features);
3044                         break;
3045                 }
3046         }
3047         rcu_read_unlock();
3048 
3049         __skb_push(skb, skb->data - skb_mac_header(skb));
3050 
3051         return segs;
3052 }
3053 EXPORT_SYMBOL(skb_mac_gso_segment);
3054 
3055 
3056 /* openvswitch calls this on rx path, so we need a different check.
3057  */
3058 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
3059 {
3060         if (tx_path)
3061                 return skb->ip_summed != CHECKSUM_PARTIAL &&
3062                        skb->ip_summed != CHECKSUM_UNNECESSARY;
3063 
3064         return skb->ip_summed == CHECKSUM_NONE;
3065 }
3066 
3067 /**
3068  *      __skb_gso_segment - Perform segmentation on skb.
3069  *      @skb: buffer to segment
3070  *      @features: features for the output path (see dev->features)
3071  *      @tx_path: whether it is called in TX path
3072  *
3073  *      This function segments the given skb and returns a list of segments.
3074  *
3075  *      It may return NULL if the skb requires no segmentation.  This is
3076  *      only possible when GSO is used for verifying header integrity.
3077  *
3078  *      Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb.
3079  */
3080 struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
3081                                   netdev_features_t features, bool tx_path)
3082 {
3083         struct sk_buff *segs;
3084 
3085         if (unlikely(skb_needs_check(skb, tx_path))) {
3086                 int err;
3087 
3088                 /* We're going to init ->check field in TCP or UDP header */
3089                 err = skb_cow_head(skb, 0);
3090                 if (err < 0)
3091                         return ERR_PTR(err);
3092         }
3093 
3094         /* Only report GSO partial support if it will enable us to
3095          * support segmentation on this frame without needing additional
3096          * work.
3097          */
3098         if (features & NETIF_F_GSO_PARTIAL) {
3099                 netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
3100                 struct net_device *dev = skb->dev;
3101 
3102                 partial_features |= dev->features & dev->gso_partial_features;
3103                 if (!skb_gso_ok(skb, features | partial_features))
3104                         features &= ~NETIF_F_GSO_PARTIAL;
3105         }
3106 
3107         BUILD_BUG_ON(SKB_SGO_CB_OFFSET +
3108                      sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
3109 
3110         SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
3111         SKB_GSO_CB(skb)->encap_level = 0;
3112 
3113         skb_reset_mac_header(skb);
3114         skb_reset_mac_len(skb);
3115 
3116         segs = skb_mac_gso_segment(skb, features);
3117 
3118         if (unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
3119                 skb_warn_bad_offload(skb);
3120 
3121         return segs;
3122 }
3123 EXPORT_SYMBOL(__skb_gso_segment);
3124 
3125 /* Take action when hardware reception checksum errors are detected. */
3126 #ifdef CONFIG_BUG
3127 void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3128 {
3129         if (net_ratelimit()) {
3130                 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
3131                 if (dev)
3132                         pr_err("dev features: %pNF\n", &dev->features);
3133                 pr_err("skb len=%u data_len=%u pkt_type=%u gso_size=%u gso_type=%u nr_frags=%u ip_summed=%u csum=%x csum_complete_sw=%d csum_valid=%d csum_level=%u\n",
3134                        skb->len, skb->data_len, skb->pkt_type,
3135                        skb_shinfo(skb)->gso_size, skb_shinfo(skb)->gso_type,
3136                        skb_shinfo(skb)->nr_frags, skb->ip_summed, skb->csum,
3137                        skb->csum_complete_sw, skb->csum_valid, skb->csum_level);
3138                 dump_stack();
3139         }
3140 }
3141 EXPORT_SYMBOL(netdev_rx_csum_fault);
3142 #endif
3143 
3144 /* XXX: check that highmem exists at all on the given machine. */
3145 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
3146 {
3147 #ifdef CONFIG_HIGHMEM
3148         int i;
3149 
3150         if (!(dev->features & NETIF_F_HIGHDMA)) {
3151                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3152                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3153 
3154                         if (PageHighMem(skb_frag_page(frag)))
3155                                 return 1;
3156                 }
3157         }
3158 #endif
3159         return 0;
3160 }
3161 
3162 /* If MPLS offload request, verify we are testing hardware MPLS features
3163  * instead of standard features for the netdev.
3164  */
3165 #if IS_ENABLED(CONFIG_NET_MPLS_GSO)
3166 static netdev_features_t net_mpls_features(struct sk_buff *skb,
3167                                            netdev_features_t features,
3168                                            __be16 type)
3169 {
3170         if (eth_p_mpls(type))
3171                 features &= skb->dev->mpls_features;
3172 
3173         return features;
3174 }
3175 #else
3176 static netdev_features_t net_mpls_features(struct sk_buff *skb,
3177                                            netdev_features_t features,
3178                                            __be16 type)
3179 {
3180         return features;
3181 }
3182 #endif
3183 
3184 static netdev_features_t harmonize_features(struct sk_buff *skb,
3185         netdev_features_t features)
3186 {
3187         int tmp;
3188         __be16 type;
3189 
3190         type = skb_network_protocol(skb, &tmp);
3191         features = net_mpls_features(skb, features, type);
3192 
3193         if (skb->ip_summed != CHECKSUM_NONE &&
3194             !can_checksum_protocol(features, type)) {
3195                 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3196         }
3197         if (illegal_highdma(skb->dev, skb))
3198                 features &= ~NETIF_F_SG;
3199 
3200         return features;
3201 }
3202 
3203 netdev_features_t passthru_features_check(struct sk_buff *skb,
3204                                           struct net_device *dev,
3205                                           netdev_features_t features)
3206 {
3207         return features;
3208 }
3209 EXPORT_SYMBOL(passthru_features_check);
3210 
3211 static netdev_features_t dflt_features_check(struct sk_buff *skb,
3212                                              struct net_device *dev,
3213                                              netdev_features_t features)
3214 {
3215         return vlan_features_check(skb, features);
3216 }
3217 
3218 static netdev_features_t gso_features_check(const struct sk_buff *skb,
3219                                             struct net_device *dev,
3220                                             netdev_features_t features)
3221 {
3222         u16 gso_segs = skb_shinfo(skb)->gso_segs;
3223 
3224         if (gso_segs > dev->gso_max_segs)
3225                 return features & ~NETIF_F_GSO_MASK;
3226 
3227         /* Support for GSO partial features requires software
3228          * intervention before we can actually process the packets
3229          * so we need to strip support for any partial features now
3230          * and we can pull them back in after we have partially
3231          * segmented the frame.
3232          */
3233         if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
3234                 features &= ~dev->gso_partial_features;
3235 
3236         /* Make sure to clear the IPv4 ID mangling feature if the
3237          * IPv4 header has the potential to be fragmented.
3238          */
3239         if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
3240                 struct iphdr *iph = skb->encapsulation ?
3241                                     inner_ip_hdr(skb) : ip_hdr(skb);
3242 
3243                 if (!(iph->frag_off & htons(IP_DF)))
3244                         features &= ~NETIF_F_TSO_MANGLEID;
3245         }
3246 
3247         return features;
3248 }
3249 
3250 netdev_features_t netif_skb_features(struct sk_buff *skb)
3251 {
3252         struct net_device *dev = skb->dev;
3253         netdev_features_t features = dev->features;
3254 
3255         if (skb_is_gso(skb))
3256                 features = gso_features_check(skb, dev, features);
3257 
3258         /* If encapsulation offload request, verify we are testing
3259          * hardware encapsulation features instead of standard
3260          * features for the netdev
3261          */
3262         if (skb->encapsulation)
3263                 features &= dev->hw_enc_features;
3264 
3265         if (skb_vlan_tagged(skb))
3266                 features = netdev_intersect_features(features,
3267                                                      dev->vlan_features |
3268                                                      NETIF_F_HW_VLAN_CTAG_TX |
3269                                                      NETIF_F_HW_VLAN_STAG_TX);
3270 
3271         if (dev->netdev_ops->ndo_features_check)
3272                 features &= dev->netdev_ops->ndo_features_check(skb, dev,
3273                                                                 features);
3274         else
3275                 features &= dflt_features_check(skb, dev, features);
3276 
3277         return harmonize_features(skb, features);
3278 }
3279 EXPORT_SYMBOL(netif_skb_features);
3280 
3281 static int xmit_one(struct sk_buff *skb, struct net_device *dev,
3282                     struct netdev_queue *txq, bool more)
3283 {
3284         unsigned int len;
3285         int rc;
3286 
3287         if (dev_nit_active(dev))
3288                 dev_queue_xmit_nit(skb, dev);
3289 
3290         len = skb->len;
3291         trace_net_dev_start_xmit(skb, dev);
3292         rc = netdev_start_xmit(skb, dev, txq, more);
3293         trace_net_dev_xmit(skb, rc, dev, len);
3294 
3295         return rc;
3296 }
3297 
3298 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
3299                                     struct netdev_queue *txq, int *ret)
3300 {
3301         struct sk_buff *skb = first;
3302         int rc = NETDEV_TX_OK;
3303 
3304         while (skb) {
3305                 struct sk_buff *next = skb->next;
3306 
3307                 skb_mark_not_on_list(skb);
3308                 rc = xmit_one(skb, dev, txq, next != NULL);
3309                 if (unlikely(!dev_xmit_complete(rc))) {
3310                         skb->next = next;
3311                         goto out;
3312                 }
3313 
3314                 skb = next;
3315                 if (netif_tx_queue_stopped(txq) && skb) {
3316                         rc = NETDEV_TX_BUSY;
3317                         break;
3318                 }
3319         }
3320 
3321 out:
3322         *ret = rc;
3323         return skb;
3324 }
3325 
3326 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
3327                                           netdev_features_t features)
3328 {
3329         if (skb_vlan_tag_present(skb) &&
3330             !vlan_hw_offload_capable(features, skb->vlan_proto))
3331                 skb = __vlan_hwaccel_push_inside(skb);
3332         return skb;
3333 }
3334 
3335 int skb_csum_hwoffload_help(struct sk_buff *skb,
3336                             const netdev_features_t features)
3337 {
3338         if (unlikely(skb->csum_not_inet))
3339                 return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3340                         skb_crc32c_csum_help(skb);
3341 
3342         return !!(features & NETIF_F_CSUM_MASK) ? 0 : skb_checksum_help(skb);
3343 }
3344 EXPORT_SYMBOL(skb_csum_hwoffload_help);
3345 
3346 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)
3347 {
3348         netdev_features_t features;
3349 
3350         features = netif_skb_features(skb);
3351         skb = validate_xmit_vlan(skb, features);
3352         if (unlikely(!skb))
3353                 goto out_null;
3354 
3355         skb = sk_validate_xmit_skb(skb, dev);
3356         if (unlikely(!skb))
3357                 goto out_null;
3358 
3359         if (netif_needs_gso(skb, features)) {
3360                 struct sk_buff *segs;
3361 
3362                 segs = skb_gso_segment(skb, features);
3363                 if (IS_ERR(segs)) {
3364                         goto out_kfree_skb;
3365                 } else if (segs) {
3366                         consume_skb(skb);
3367                         skb = segs;
3368                 }
3369         } else {
3370                 if (skb_needs_linearize(skb, features) &&
3371                     __skb_linearize(skb))
3372                         goto out_kfree_skb;
3373 
3374                 /* If packet is not checksummed and device does not
3375                  * support checksumming for this protocol, complete
3376                  * checksumming here.
3377                  */
3378                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3379                         if (skb->encapsulation)
3380                                 skb_set_inner_transport_header(skb,
3381                                                                skb_checksum_start_offset(skb));
3382                         else
3383                                 skb_set_transport_header(skb,
3384                                                          skb_checksum_start_offset(skb));
3385                         if (skb_csum_hwoffload_help(skb, features))
3386                                 goto out_kfree_skb;
3387                 }
3388         }
3389 
3390         skb = validate_xmit_xfrm(skb, features, again);
3391 
3392         return skb;
3393 
3394 out_kfree_skb:
3395         kfree_skb(skb);
3396 out_null:
3397         atomic_long_inc(&dev->tx_dropped);
3398         return NULL;
3399 }
3400 
3401 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again)
3402 {
3403         struct sk_buff *next, *head = NULL, *tail;
3404 
3405         for (; skb != NULL; skb = next) {
3406                 next = skb->next;
3407                 skb_mark_not_on_list(skb);
3408 
3409                 /* in case skb wont be segmented, point to itself */
3410                 skb->prev = skb;
3411 
3412                 skb = validate_xmit_skb(skb, dev, again);
3413                 if (!skb)
3414                         continue;
3415 
3416                 if (!head)
3417                         head = skb;
3418                 else
3419                         tail->next = skb;
3420                 /* If skb was segmented, skb->prev points to
3421                  * the last segment. If not, it still contains skb.
3422                  */
3423                 tail = skb->prev;
3424         }
3425         return head;
3426 }
3427 EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3428 
3429 static void qdisc_pkt_len_init(struct sk_buff *skb)
3430 {
3431         const struct skb_shared_info *shinfo = skb_shinfo(skb);
3432 
3433         qdisc_skb_cb(skb)->pkt_len = skb->len;
3434 
3435         /* To get more precise estimation of bytes sent on wire,
3436          * we add to pkt_len the headers size of all segments
3437          */
3438         if (shinfo->gso_size)  {
3439                 unsigned int hdr_len;
3440                 u16 gso_segs = shinfo->gso_segs;
3441 
3442                 /* mac layer + network layer */
3443                 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3444 
3445                 /* + transport layer */
3446                 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
3447                         const struct tcphdr *th;
3448                         struct tcphdr _tcphdr;
3449 
3450                         th = skb_header_pointer(skb, skb_transport_offset(skb),
3451                                                 sizeof(_tcphdr), &_tcphdr);
3452                         if (likely(th))
3453                                 hdr_len += __tcp_hdrlen(th);
3454                 } else {
3455                         struct udphdr _udphdr;
3456 
3457                         if (skb_header_pointer(skb, skb_transport_offset(skb),
3458                                                sizeof(_udphdr), &_udphdr))
3459                                 hdr_len += sizeof(struct udphdr);
3460                 }
3461 
3462                 if (shinfo->gso_type & SKB_GSO_DODGY)
3463                         gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3464                                                 shinfo->gso_size);
3465 
3466                 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3467         }
3468 }
3469 
3470 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3471                                  struct net_device *dev,
3472                                  struct netdev_queue *txq)
3473 {
3474         spinlock_t *root_lock = qdisc_lock(q);
3475         struct sk_buff *to_free = NULL;
3476         bool contended;
3477         int rc;
3478 
3479         qdisc_calculate_pkt_len(skb, q);
3480 
3481         if (q->flags & TCQ_F_NOLOCK) {
3482                 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3483                         __qdisc_drop(skb, &to_free);
3484                         rc = NET_XMIT_DROP;
3485                 } else {
3486                         rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
3487                         qdisc_run(q);
3488                 }
3489 
3490                 if (unlikely(to_free))
3491                         kfree_skb_list(to_free);
3492                 return rc;
3493         }
3494 
3495         /*
3496          * Heuristic to force contended enqueues to serialize on a
3497          * separate lock before trying to get qdisc main lock.
3498          * This permits qdisc->running owner to get the lock more
3499          * often and dequeue packets faster.
3500          */
3501         contended = qdisc_is_running(q);
3502         if (unlikely(contended))
3503                 spin_lock(&q->busylock);
3504 
3505         spin_lock(root_lock);
3506         if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3507                 __qdisc_drop(skb, &to_free);
3508                 rc = NET_XMIT_DROP;
3509         } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3510                    qdisc_run_begin(q)) {
3511                 /*
3512                  * This is a work-conserving queue; there are no old skbs
3513                  * waiting to be sent out; and the qdisc is not running -
3514                  * xmit the skb directly.
3515                  */
3516 
3517                 qdisc_bstats_update(q, skb);
3518 
3519                 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3520                         if (unlikely(contended)) {
3521                                 spin_unlock(&q->busylock);
3522                                 contended = false;
3523                         }
3524                         __qdisc_run(q);
3525                 }
3526 
3527                 qdisc_run_end(q);
3528                 rc = NET_XMIT_SUCCESS;
3529         } else {
3530                 rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
3531                 if (qdisc_run_begin(q)) {
3532                         if (unlikely(contended)) {
3533                                 spin_unlock(&q->busylock);
3534                                 contended = false;
3535                         }
3536                         __qdisc_run(q);
3537                         qdisc_run_end(q);
3538                 }
3539         }
3540         spin_unlock(root_lock);
3541         if (unlikely(to_free))
3542                 kfree_skb_list(to_free);
3543         if (unlikely(contended))
3544                 spin_unlock(&q->busylock);
3545         return rc;
3546 }
3547 
3548 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3549 static void skb_update_prio(struct sk_buff *skb)
3550 {
3551         const struct netprio_map *map;
3552         const struct sock *sk;
3553         unsigned int prioidx;
3554 
3555         if (skb->priority)
3556                 return;
3557         map = rcu_dereference_bh(skb->dev->priomap);
3558         if (!map)
3559                 return;
3560         sk = skb_to_full_sk(skb);
3561         if (!sk)
3562                 return;
3563 
3564         prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
3565 
3566         if (prioidx < map->priomap_len)
3567                 skb->priority = map->priomap[prioidx];
3568 }
3569 #else
3570 #define skb_update_prio(skb)
3571 #endif
3572 
3573 DEFINE_PER_CPU(int, xmit_recursion);
3574 EXPORT_SYMBOL(xmit_recursion);
3575 
3576 /**
3577  *      dev_loopback_xmit - loop back @skb
3578  *      @net: network namespace this loopback is happening in
3579  *      @sk:  sk needed to be a netfilter okfn
3580  *      @skb: buffer to transmit
3581  */
3582 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3583 {
3584         skb_reset_mac_header(skb);
3585         __skb_pull(skb, skb_network_offset(skb));
3586         skb->pkt_type = PACKET_LOOPBACK;
3587         skb->ip_summed = CHECKSUM_UNNECESSARY;
3588         WARN_ON(!skb_dst(skb));
3589         skb_dst_force(skb);
3590         netif_rx_ni(skb);
3591         return 0;
3592 }
3593 EXPORT_SYMBOL(dev_loopback_xmit);
3594 
3595 #ifdef CONFIG_NET_EGRESS
3596 static struct sk_buff *
3597 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3598 {
3599         struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress);
3600         struct tcf_result cl_res;
3601 
3602         if (!miniq)
3603                 return skb;
3604 
3605         /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
3606         mini_qdisc_bstats_cpu_update(miniq, skb);
3607 
3608         switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) {
3609         case TC_ACT_OK:
3610         case TC_ACT_RECLASSIFY:
3611                 skb->tc_index = TC_H_MIN(cl_res.classid);
3612                 break;
3613         case TC_ACT_SHOT:
3614                 mini_qdisc_qstats_cpu_drop(miniq);
3615                 *ret = NET_XMIT_DROP;
3616                 kfree_skb(skb);
3617                 return NULL;
3618         case TC_ACT_STOLEN:
3619         case TC_ACT_QUEUED:
3620         case TC_ACT_TRAP:
3621                 *ret = NET_XMIT_SUCCESS;
3622                 consume_skb(skb);
3623                 return NULL;
3624         case TC_ACT_REDIRECT:
3625                 /* No need to push/pop skb's mac_header here on egress! */
3626                 skb_do_redirect(skb);
3627                 *ret = NET_XMIT_SUCCESS;
3628                 return NULL;
3629         default:
3630                 break;
3631         }
3632 
3633         return skb;
3634 }
3635 #endif /* CONFIG_NET_EGRESS */
3636 
3637 #ifdef CONFIG_XPS
3638 static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb,
3639                                struct xps_dev_maps *dev_maps, unsigned int tci)
3640 {
3641         struct xps_map *map;
3642         int queue_index = -1;
3643 
3644         if (dev->num_tc) {
3645                 tci *= dev->num_tc;
3646                 tci += netdev_get_prio_tc_map(dev, skb->priority);
3647         }
3648 
3649         map = rcu_dereference(dev_maps->attr_map[tci]);
3650         if (map) {
3651                 if (map->len == 1)
3652                         queue_index = map->queues[0];
3653                 else
3654                         queue_index = map->queues[reciprocal_scale(
3655                                                 skb_get_hash(skb), map->len)];
3656                 if (unlikely(queue_index >= dev->real_num_tx_queues))
3657                         queue_index = -1;
3658         }
3659         return queue_index;
3660 }
3661 #endif
3662 
3663 static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev,
3664                          struct sk_buff *skb)
3665 {
3666 #ifdef CONFIG_XPS
3667         struct xps_dev_maps *dev_maps;
3668         struct sock *sk = skb->sk;
3669         int queue_index = -1;
3670 
3671         if (!static_key_false(&xps_needed))
3672                 return -1;
3673 
3674         rcu_read_lock();
3675         if (!static_key_false(&xps_rxqs_needed))
3676                 goto get_cpus_map;
3677 
3678         dev_maps = rcu_dereference(sb_dev->xps_rxqs_map);
3679         if (dev_maps) {
3680                 int tci = sk_rx_queue_get(sk);
3681 
3682                 if (tci >= 0 && tci < dev->num_rx_queues)
3683                         queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
3684                                                           tci);
3685         }
3686 
3687 get_cpus_map:
3688         if (queue_index < 0) {
3689                 dev_maps = rcu_dereference(sb_dev->xps_cpus_map);
3690                 if (dev_maps) {
3691                         unsigned int tci = skb->sender_cpu - 1;
3692 
3693                         queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
3694                                                           tci);
3695                 }
3696         }
3697         rcu_read_unlock();
3698 
3699         return queue_index;
3700 #else
3701         return -1;
3702 #endif
3703 }
3704 
3705 u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
3706                      struct net_device *sb_dev,
3707                      select_queue_fallback_t fallback)
3708 {
3709         return 0;
3710 }
3711 EXPORT_SYMBOL(dev_pick_tx_zero);
3712 
3713 u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb,
3714                        struct net_device *sb_dev,
3715                        select_queue_fallback_t fallback)
3716 {
3717         return (u16)raw_smp_processor_id() % dev->real_num_tx_queues;
3718 }
3719 EXPORT_SYMBOL(dev_pick_tx_cpu_id);
3720 
3721 static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
3722                             struct net_device *sb_dev)
3723 {
3724         struct sock *sk = skb->sk;
3725         int queue_index = sk_tx_queue_get(sk);
3726 
3727         sb_dev = sb_dev ? : dev;
3728 
3729         if (queue_index < 0 || skb->ooo_okay ||
3730             queue_index >= dev->real_num_tx_queues) {
3731                 int new_index = get_xps_queue(dev, sb_dev, skb);
3732 
3733                 if (new_index < 0)
3734                         new_index = skb_tx_hash(dev, sb_dev, skb);
3735 
3736                 if (queue_index != new_index && sk &&
3737                     sk_fullsock(sk) &&
3738                     rcu_access_pointer(sk->sk_dst_cache))
3739                         sk_tx_queue_set(sk, new_index);
3740 
3741                 queue_index = new_index;
3742         }
3743 
3744         return queue_index;
3745 }
3746 
3747 struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3748                                     struct sk_buff *skb,
3749                                     struct net_device *sb_dev)
3750 {
3751         int queue_index = 0;
3752 
3753 #ifdef CONFIG_XPS
3754         u32 sender_cpu = skb->sender_cpu - 1;
3755 
3756         if (sender_cpu >= (u32)NR_CPUS)
3757                 skb->sender_cpu = raw_smp_processor_id() + 1;
3758 #endif
3759 
3760         if (dev->real_num_tx_queues != 1) {
3761                 const struct net_device_ops *ops = dev->netdev_ops;
3762 
3763                 if (ops->ndo_select_queue)
3764                         queue_index = ops->ndo_select_queue(dev, skb, sb_dev,
3765                                                             __netdev_pick_tx);
3766                 else
3767                         queue_index = __netdev_pick_tx(dev, skb, sb_dev);
3768 
3769                 queue_index = netdev_cap_txqueue(dev, queue_index);
3770         }
3771 
3772         skb_set_queue_mapping(skb, queue_index);
3773         return netdev_get_tx_queue(dev, queue_index);
3774 }
3775 
3776 /**
3777  *      __dev_queue_xmit - transmit a buffer
3778  *      @skb: buffer to transmit
3779  *      @sb_dev: suboordinate device used for L2 forwarding offload
3780  *
3781  *      Queue a buffer for transmission to a network device. The caller must
3782  *      have set the device and priority and built the buffer before calling
3783  *      this function. The function can be called from an interrupt.
3784  *
3785  *      A negative errno code is returned on a failure. A success does not
3786  *      guarantee the frame will be transmitted as it may be dropped due
3787  *      to congestion or traffic shaping.
3788  *
3789  * -----------------------------------------------------------------------------------
3790  *      I notice this method can also return errors from the queue disciplines,
3791  *      including NET_XMIT_DROP, which is a positive value.  So, errors can also
3792  *      be positive.
3793  *
3794  *      Regardless of the return value, the skb is consumed, so it is currently
3795  *      difficult to retry a send to this method.  (You can bump the ref count
3796  *      before sending to hold a reference for retry if you are careful.)
3797  *
3798  *      When calling this method, interrupts MUST be enabled.  This is because
3799  *      the BH enable code must have IRQs enabled so that it will not deadlock.
3800  *          --BLG
3801  */
3802 static int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
3803 {
3804         struct net_device *dev = skb->dev;
3805         struct netdev_queue *txq;
3806         struct Qdisc *q;
3807         int rc = -ENOMEM;
3808         bool again = false;
3809 
3810         skb_reset_mac_header(skb);
3811 
3812         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3813                 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3814 
3815         /* Disable soft irqs for various locks below. Also
3816          * stops preemption for RCU.
3817          */
3818         rcu_read_lock_bh();
3819 
3820         skb_update_prio(skb);
3821 
3822         qdisc_pkt_len_init(skb);
3823 #ifdef CONFIG_NET_CLS_ACT
3824         skb->tc_at_ingress = 0;
3825 # ifdef CONFIG_NET_EGRESS
3826         if (static_branch_unlikely(&egress_needed_key)) {
3827                 skb = sch_handle_egress(skb, &rc, dev);
3828                 if (!skb)
3829                         goto out;
3830         }
3831 # endif
3832 #endif
3833         /* If device/qdisc don't need skb->dst, release it right now while
3834          * its hot in this cpu cache.
3835          */
3836         if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3837                 skb_dst_drop(skb);
3838         else
3839                 skb_dst_force(skb);
3840 
3841         txq = netdev_pick_tx(dev, skb, sb_dev);
3842         q = rcu_dereference_bh(txq->qdisc);
3843 
3844         trace_net_dev_queue(skb);
3845         if (q->enqueue) {
3846                 rc = __dev_xmit_skb(skb, q, dev, txq);
3847                 goto out;
3848         }
3849 
3850         /* The device has no queue. Common case for software devices:
3851          * loopback, all the sorts of tunnels...
3852 
3853          * Really, it is unlikely that netif_tx_lock protection is necessary
3854          * here.  (f.e. loopback and IP tunnels are clean ignoring statistics
3855          * counters.)
3856          * However, it is possible, that they rely on protection
3857          * made by us here.
3858 
3859          * Check this and shot the lock. It is not prone from deadlocks.
3860          *Either shot noqueue qdisc, it is even simpler 8)
3861          */
3862         if (dev->flags & IFF_UP) {
3863                 int cpu = smp_processor_id(); /* ok because BHs are off */
3864 
3865                 if (txq->xmit_lock_owner != cpu) {
3866                         if (unlikely(__this_cpu_read(xmit_recursion) >
3867                                      XMIT_RECURSION_LIMIT))
3868                                 goto recursion_alert;
3869 
3870                         skb = validate_xmit_skb(skb, dev, &again);
3871                         if (!skb)
3872                                 goto out;
3873 
3874                         HARD_TX_LOCK(dev, txq, cpu);
3875 
3876                         if (!netif_xmit_stopped(txq)) {
3877                                 __this_cpu_inc(xmit_recursion);
3878                                 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
3879                                 __this_cpu_dec(xmit_recursion);
3880                                 if (dev_xmit_complete(rc)) {
3881                                         HARD_TX_UNLOCK(dev, txq);
3882                                         goto out;
3883                                 }
3884                         }
3885                         HARD_TX_UNLOCK(dev, txq);
3886                         net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3887                                              dev->name);
3888                 } else {
3889                         /* Recursion is detected! It is possible,
3890                          * unfortunately
3891                          */
3892 recursion_alert:
3893                         net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3894                                              dev->name);
3895                 }
3896         }
3897 
3898         rc = -ENETDOWN;
3899         rcu_read_unlock_bh();
3900 
3901         atomic_long_inc(&dev->tx_dropped);
3902         kfree_skb_list(skb);
3903         return rc;
3904 out:
3905         rcu_read_unlock_bh();
3906         return rc;
3907 }
3908 
3909 int dev_queue_xmit(struct sk_buff *skb)
3910 {
3911         return __dev_queue_xmit(skb, NULL);
3912 }
3913 EXPORT_SYMBOL(dev_queue_xmit);
3914 
3915 int dev_queue_xmit_accel(struct sk_buff *skb, struct net_device *sb_dev)
3916 {
3917         return __dev_queue_xmit(skb, sb_dev);
3918 }
3919 EXPORT_SYMBOL(dev_queue_xmit_accel);
3920 
3921 int dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
3922 {
3923         struct net_device *dev = skb->dev;
3924         struct sk_buff *orig_skb = skb;
3925         struct netdev_queue *txq;
3926         int ret = NETDEV_TX_BUSY;
3927         bool again = false;
3928 
3929         if (unlikely(!netif_running(dev) ||
3930                      !netif_carrier_ok(dev)))
3931                 goto drop;
3932 
3933         skb = validate_xmit_skb_list(skb, dev, &again);
3934         if (skb != orig_skb)
3935                 goto drop;
3936 
3937         skb_set_queue_mapping(skb, queue_id);
3938         txq = skb_get_tx_queue(dev, skb);
3939 
3940         local_bh_disable();
3941 
3942         HARD_TX_LOCK(dev, txq, smp_processor_id());
3943         if (!netif_xmit_frozen_or_drv_stopped(txq))
3944                 ret = netdev_start_xmit(skb, dev, txq, false);
3945         HARD_TX_UNLOCK(dev, txq);
3946 
3947         local_bh_enable();
3948 
3949         if (!dev_xmit_complete(ret))
3950                 kfree_skb(skb);
3951 
3952         return ret;
3953 drop:
3954         atomic_long_inc(&dev->tx_dropped);
3955         kfree_skb_list(skb);
3956         return NET_XMIT_DROP;
3957 }
3958 EXPORT_SYMBOL(dev_direct_xmit);
3959 
3960 /*************************************************************************
3961  *                      Receiver routines
3962  *************************************************************************/
3963 
3964 int netdev_max_backlog __read_mostly = 1000;
3965 EXPORT_SYMBOL(netdev_max_backlog);
3966 
3967 int netdev_tstamp_prequeue __read_mostly = 1;
3968 int netdev_budget __read_mostly = 300;
3969 unsigned int __read_mostly netdev_budget_usecs = 2000;
3970 int weight_p __read_mostly = 64;           /* old backlog weight */
3971 int dev_weight_rx_bias __read_mostly = 1;  /* bias for backlog weight */
3972 int dev_weight_tx_bias __read_mostly = 1;  /* bias for output_queue quota */
3973 int dev_rx_weight __read_mostly = 64;
3974 int dev_tx_weight __read_mostly = 64;
3975 
3976 /* Called with irq disabled */
3977 static inline void ____napi_schedule(struct softnet_data *sd,
3978                                      struct napi_struct *napi)
3979 {
3980         list_add_tail(&napi->poll_list, &sd->poll_list);
3981         __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3982 }
3983 
3984 #ifdef CONFIG_RPS
3985 
3986 /* One global table that all flow-based protocols share. */
3987 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
3988 EXPORT_SYMBOL(rps_sock_flow_table);
3989 u32 rps_cpu_mask __read_mostly;
3990 EXPORT_SYMBOL(rps_cpu_mask);
3991 
3992 struct static_key rps_needed __read_mostly;
3993 EXPORT_SYMBOL(rps_needed);
3994 struct static_key rfs_needed __read_mostly;
3995 EXPORT_SYMBOL(rfs_needed);
3996 
3997 static struct rps_dev_flow *
3998 set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3999             struct rps_dev_flow *rflow, u16 next_cpu)
4000 {
4001         if (next_cpu < nr_cpu_ids) {
4002 #ifdef CONFIG_RFS_ACCEL
4003                 struct netdev_rx_queue *rxqueue;
4004                 struct rps_dev_flow_table *flow_table;
4005                 struct rps_dev_flow *old_rflow;
4006                 u32 flow_id;
4007                 u16 rxq_index;
4008                 int rc;
4009 
4010                 /* Should we steer this flow to a different hardware queue? */
4011                 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
4012                     !(dev->features & NETIF_F_NTUPLE))
4013                         goto out;
4014                 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
4015                 if (rxq_index == skb_get_rx_queue(skb))
4016                         goto out;
4017 
4018                 rxqueue = dev->_rx + rxq_index;
4019                 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4020                 if (!flow_table)
4021                         goto out;
4022                 flow_id = skb_get_hash(skb) & flow_table->mask;
4023                 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
4024                                                         rxq_index, flow_id);
4025                 if (rc < 0)
4026                         goto out;
4027                 old_rflow = rflow;
4028                 rflow = &flow_table->flows[flow_id];
4029                 rflow->filter = rc;
4030                 if (old_rflow->filter == rflow->filter)
4031                         old_rflow->filter = RPS_NO_FILTER;
4032         out:
4033 #endif
4034                 rflow->last_qtail =
4035                         per_cpu(softnet_data, next_cpu).input_queue_head;
4036         }
4037 
4038         rflow->cpu = next_cpu;
4039         return rflow;
4040 }
4041 
4042 /*
4043  * get_rps_cpu is called from netif_receive_skb and returns the target
4044  * CPU from the RPS map of the receiving queue for a given skb.
4045  * rcu_read_lock must be held on entry.
4046  */
4047 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4048                        struct rps_dev_flow **rflowp)
4049 {
4050         const struct rps_sock_flow_table *sock_flow_table;
4051         struct netdev_rx_queue *rxqueue = dev->_rx;
4052         struct rps_dev_flow_table *flow_table;
4053         struct rps_map *map;
4054         int cpu = -1;
4055         u32 tcpu;
4056         u32 hash;
4057 
4058         if (skb_rx_queue_recorded(skb)) {
4059                 u16 index = skb_get_rx_queue(skb);
4060 
4061                 if (unlikely(index >= dev->real_num_rx_queues)) {
4062                         WARN_ONCE(dev->real_num_rx_queues > 1,
4063                                   "%s received packet on queue %u, but number "
4064                                   "of RX queues is %u\n",
4065                                   dev->name, index, dev->real_num_rx_queues);
4066                         goto done;
4067                 }
4068                 rxqueue += index;
4069         }
4070 
4071         /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
4072 
4073         flow_table = rcu_dereference(rxqueue->rps_flow_table);
4074         map = rcu_dereference(rxqueue->rps_map);
4075         if (!flow_table && !map)
4076                 goto done;
4077 
4078         skb_reset_network_header(skb);
4079         hash = skb_get_hash(skb);
4080         if (!hash)
4081                 goto done;
4082 
4083         sock_flow_table = rcu_dereference(rps_sock_flow_table);
4084         if (flow_table && sock_flow_table) {
4085                 struct rps_dev_flow *rflow;
4086                 u32 next_cpu;
4087                 u32 ident;
4088 
4089                 /* First check into global flow table if there is a match */
4090                 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
4091                 if ((ident ^ hash) & ~rps_cpu_mask)
4092                         goto try_rps;
4093 
4094                 next_cpu = ident & rps_cpu_mask;
4095 
4096                 /* OK, now we know there is a match,
4097                  * we can look at the local (per receive queue) flow table
4098                  */
4099                 rflow = &flow_table->flows[hash & flow_table->mask];
4100                 tcpu = rflow->cpu;
4101 
4102                 /*
4103                  * If the desired CPU (where last recvmsg was done) is
4104                  * different from current CPU (one in the rx-queue flow
4105                  * table entry), switch if one of the following holds:
4106                  *   - Current CPU is unset (>= nr_cpu_ids).
4107                  *   - Current CPU is offline.
4108                  *   - The current CPU's queue tail has advanced beyond the
4109                  *     last packet that was enqueued using this table entry.
4110                  *     This guarantees that all previous packets for the flow
4111                  *     have been dequeued, thus preserving in order delivery.
4112                  */
4113                 if (unlikely(tcpu != next_cpu) &&
4114                     (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
4115                      ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
4116                       rflow->last_qtail)) >= 0)) {
4117                         tcpu = next_cpu;
4118                         rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
4119                 }
4120 
4121                 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
4122                         *rflowp = rflow;
4123                         cpu = tcpu;
4124                         goto done;
4125                 }
4126         }
4127 
4128 try_rps:
4129 
4130         if (map) {
4131                 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
4132                 if (cpu_online(tcpu)) {
4133                         cpu = tcpu;
4134                         goto done;
4135                 }
4136         }
4137 
4138 done:
4139         return cpu;
4140 }
4141 
4142 #ifdef CONFIG_RFS_ACCEL
4143 
4144 /**
4145  * rps_may_expire_flow - check whether an RFS hardware filter may be removed
4146  * @dev: Device on which the filter was set
4147  * @rxq_index: RX queue index
4148  * @flow_id: Flow ID passed to ndo_rx_flow_steer()
4149  * @filter_id: Filter ID returned by ndo_rx_flow_steer()
4150  *
4151  * Drivers that implement ndo_rx_flow_steer() should periodically call
4152  * this function for each installed filter and remove the filters for
4153  * which it returns %true.
4154  */
4155 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
4156                          u32 flow_id, u16 filter_id)
4157 {
4158         struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
4159         struct rps_dev_flow_table *flow_table;
4160         struct rps_dev_flow *rflow;
4161         bool expire = true;
4162         unsigned int cpu;
4163 
4164         rcu_read_lock();
4165         flow_table = rcu_dereference(rxqueue->rps_flow_table);
4166         if (flow_table && flow_id <= flow_table->mask) {
4167                 rflow = &flow_table->flows[flow_id];
4168                 cpu = READ_ONCE(rflow->cpu);
4169                 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
4170                     ((int)(per_cpu(softnet_data, cpu).input_queue_head -
4171                            rflow->last_qtail) <
4172                      (int)(10 * flow_table->mask)))
4173                         expire = false;
4174         }
4175         rcu_read_unlock();
4176         return expire;
4177 }
4178 EXPORT_SYMBOL(rps_may_expire_flow);
4179 
4180 #endif /* CONFIG_RFS_ACCEL */
4181 
4182 /* Called from hardirq (IPI) context */
4183 static void rps_trigger_softirq(void *data)
4184 {
4185         struct softnet_data *sd = data;
4186 
4187         ____napi_schedule(sd, &sd->backlog);
4188         sd->received_rps++;
4189 }
4190 
4191 #endif /* CONFIG_RPS */
4192 
4193 /*
4194  * Check if this softnet_data structure is another cpu one
4195  * If yes, queue it to our IPI list and return 1
4196  * If no, return 0
4197  */
4198 static int rps_ipi_queued(struct softnet_data *sd)
4199 {
4200 #ifdef CONFIG_RPS
4201         struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
4202 
4203         if (sd != mysd) {
4204                 sd->rps_ipi_next = mysd->rps_ipi_list;
4205                 mysd->rps_ipi_list = sd;
4206 
4207                 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4208                 return 1;
4209         }
4210 #endif /* CONFIG_RPS */
4211         return 0;
4212 }
4213 
4214 #ifdef CONFIG_NET_FLOW_LIMIT
4215 int netdev_flow_limit_table_len __read_mostly = (1 << 12);
4216 #endif
4217 
4218 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
4219 {
4220 #ifdef CONFIG_NET_FLOW_LIMIT
4221         struct sd_flow_limit *fl;
4222         struct softnet_data *sd;
4223         unsigned int old_flow, new_flow;
4224 
4225         if (qlen < (netdev_max_backlog >> 1))
4226                 return false;
4227 
4228         sd = this_cpu_ptr(&softnet_data);
4229 
4230         rcu_read_lock();
4231         fl = rcu_dereference(sd->flow_limit);
4232         if (fl) {
4233                 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
4234                 old_flow = fl->history[fl->history_head];
4235                 fl->history[fl->history_head] = new_flow;
4236 
4237                 fl->history_head++;
4238                 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
4239 
4240                 if (likely(fl->buckets[old_flow]))
4241                         fl->buckets[old_flow]--;
4242 
4243                 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
4244                         fl->count++;
4245                         rcu_read_unlock();
4246                         return true;
4247                 }
4248         }
4249         rcu_read_unlock();
4250 #endif
4251         return false;
4252 }
4253 
4254 /*
4255  * enqueue_to_backlog is called to queue an skb to a per CPU backlog
4256  * queue (may be a remote CPU queue).
4257  */
4258 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
4259                               unsigned int *qtail)
4260 {
4261         struct softnet_data *sd;
4262         unsigned long flags;
4263         unsigned int qlen;
4264 
4265         sd = &per_cpu(softnet_data, cpu);
4266 
4267         local_irq_save(flags);
4268 
4269         rps_lock(sd);
4270         if (!netif_running(skb->dev))
4271                 goto drop;
4272         qlen = skb_queue_len(&sd->input_pkt_queue);
4273         if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
4274                 if (qlen) {
4275 enqueue:
4276                         __skb_queue_tail(&sd->input_pkt_queue, skb);
4277                         input_queue_tail_incr_save(sd, qtail);
4278                         rps_unlock(sd);
4279                         local_irq_restore(flags);
4280                         return NET_RX_SUCCESS;
4281                 }
4282 
4283                 /* Schedule NAPI for backlog device
4284                  * We can use non atomic operation since we own the queue lock
4285                  */
4286                 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
4287                         if (!rps_ipi_queued(sd))
4288                                 ____napi_schedule(sd, &sd->backlog);
4289                 }
4290                 goto enqueue;
4291         }
4292 
4293 drop:
4294         sd->dropped++;
4295         rps_unlock(sd);
4296 
4297         local_irq_restore(flags);
4298 
4299         atomic_long_inc(&skb->dev->rx_dropped);
4300         kfree_skb(skb);
4301         return NET_RX_DROP;
4302 }
4303 
4304 static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb)
4305 {
4306         struct net_device *dev = skb->dev;
4307         struct netdev_rx_queue *rxqueue;
4308 
4309         rxqueue = dev->_rx;
4310 
4311         if (skb_rx_queue_recorded(skb)) {
4312                 u16 index = skb_get_rx_queue(skb);
4313 
4314                 if (unlikely(index >= dev->real_num_rx_queues)) {
4315                         WARN_ONCE(dev->real_num_rx_queues > 1,
4316                                   "%s received packet on queue %u, but number "
4317                                   "of RX queues is %u\n",
4318                                   dev->name, index, dev->real_num_rx_queues);
4319 
4320                         return rxqueue; /* Return first rxqueue */
4321                 }
4322                 rxqueue += index;
4323         }
4324         return rxqueue;
4325 }
4326 
4327 static u32 netif_receive_generic_xdp(struct sk_buff *skb,
4328                                      struct xdp_buff *xdp,
4329                                      struct bpf_prog *xdp_prog)
4330 {
4331         struct netdev_rx_queue *rxqueue;
4332         void *orig_data, *orig_data_end;
4333         u32 metalen, act = XDP_DROP;
4334         __be16 orig_eth_type;
4335         struct ethhdr *eth;
4336         bool orig_bcast;
4337         int hlen, off;
4338         u32 mac_len;
4339 
4340         /* Reinjected packets coming from act_mirred or similar should
4341          * not get XDP generic processing.
4342          */
4343         if (skb_cloned(skb) || skb_is_tc_redirected(skb))
4344                 return XDP_PASS;
4345 
4346         /* XDP packets must be linear and must have sufficient headroom
4347          * of XDP_PACKET_HEADROOM bytes. This is the guarantee that also
4348          * native XDP provides, thus we need to do it here as well.
4349          */
4350         if (skb_is_nonlinear(skb) ||
4351             skb_headroom(skb) < XDP_PACKET_HEADROOM) {
4352                 int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
4353                 int troom = skb->tail + skb->data_len - skb->end;
4354 
4355                 /* In case we have to go down the path and also linearize,
4356                  * then lets do the pskb_expand_head() work just once here.
4357                  */
4358                 if (pskb_expand_head(skb,
4359                                      hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0,
4360                                      troom > 0 ? troom + 128 : 0, GFP_ATOMIC))
4361                         goto do_drop;
4362                 if (skb_linearize(skb))
4363                         goto do_drop;
4364         }
4365 
4366         /* The XDP program wants to see the packet starting at the MAC
4367          * header.
4368          */
4369         mac_len = skb->data - skb_mac_header(skb);
4370         hlen = skb_headlen(skb) + mac_len;
4371         xdp->data = skb->data - mac_len;
4372         xdp->data_meta = xdp->data;
4373         xdp->data_end = xdp->data + hlen;
4374         xdp->data_hard_start = skb->data - skb_headroom(skb);
4375         orig_data_end = xdp->data_end;
4376         orig_data = xdp->data;
4377         eth = (struct ethhdr *)xdp->data;
4378         orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest);
4379         orig_eth_type = eth->h_proto;
4380 
4381         rxqueue = netif_get_rxqueue(skb);
4382         xdp->rxq = &rxqueue->xdp_rxq;
4383 
4384         act = bpf_prog_run_xdp(xdp_prog, xdp);
4385 
4386         off = xdp->data - orig_data;
4387         if (off > 0)
4388                 __skb_pull(skb, off);
4389         else if (off < 0)
4390                 __skb_push(skb, -off);
4391         skb->mac_header += off;
4392 
4393         /* check if bpf_xdp_adjust_tail was used. it can only "shrink"
4394          * pckt.
4395          */
4396         off = orig_data_end - xdp->data_end;
4397         if (off != 0) {
4398                 skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
4399                 skb->len -= off;
4400 
4401         }
4402 
4403         /* check if XDP changed eth hdr such SKB needs update */
4404         eth = (struct ethhdr *)xdp->data;
4405         if ((orig_eth_type != eth->h_proto) ||
4406             (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) {
4407                 __skb_push(skb, ETH_HLEN);
4408                 skb->protocol = eth_type_trans(skb, skb->dev);
4409         }
4410 
4411         switch (act) {
4412         case XDP_REDIRECT:
4413         case XDP_TX:
4414                 __skb_push(skb, mac_len);
4415                 break;
4416         case XDP_PASS:
4417                 metalen = xdp->data - xdp->data_meta;
4418                 if (metalen)
4419                         skb_metadata_set(skb, metalen);
4420                 break;
4421         default:
4422                 bpf_warn_invalid_xdp_action(act);
4423                 /* fall through */
4424         case XDP_ABORTED:
4425                 trace_xdp_exception(skb->dev, xdp_prog, act);
4426                 /* fall through */
4427         case XDP_DROP:
4428         do_drop:
4429                 kfree_skb(skb);
4430                 break;
4431         }
4432 
4433         return act;
4434 }
4435 
4436 /* When doing generic XDP we have to bypass the qdisc layer and the
4437  * network taps in order to match in-driver-XDP behavior.
4438  */
4439 void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
4440 {
4441         struct net_device *dev = skb->dev;
4442         struct netdev_queue *txq;
4443         bool free_skb = true;
4444         int cpu, rc;
4445 
4446         txq = netdev_pick_tx(dev, skb, NULL);
4447         cpu = smp_processor_id();
4448         HARD_TX_LOCK(dev, txq, cpu);
4449         if (!netif_xmit_stopped(txq)) {
4450                 rc = netdev_start_xmit(skb, dev, txq, 0);
4451                 if (dev_xmit_complete(rc))
4452                         free_skb = false;
4453         }
4454         HARD_TX_UNLOCK(dev, txq);
4455         if (free_skb) {
4456                 trace_xdp_exception(dev, xdp_prog, XDP_TX);
4457                 kfree_skb(skb);
4458         }
4459 }
4460 EXPORT_SYMBOL_GPL(generic_xdp_tx);
4461 
4462 static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
4463 
4464 int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
4465 {
4466         if (xdp_prog) {
4467                 struct xdp_buff xdp;
4468                 u32 act;
4469                 int err;
4470 
4471                 act = netif_receive_generic_xdp(skb, &xdp, xdp_prog);
4472                 if (act != XDP_PASS) {
4473                         switch (act) {
4474                         case XDP_REDIRECT:
4475                                 err = xdp_do_generic_redirect(skb->dev, skb,
4476                                                               &xdp, xdp_prog);
4477                                 if (err)
4478                                         goto out_redir;
4479                                 break;
4480                         case XDP_TX:
4481                                 generic_xdp_tx(skb, xdp_prog);
4482                                 break;
4483                         }
4484                         return XDP_DROP;
4485                 }
4486         }
4487         return XDP_PASS;
4488 out_redir:
4489         kfree_skb(skb);
4490         return XDP_DROP;
4491 }
4492 EXPORT_SYMBOL_GPL(do_xdp_generic);
4493 
4494 static int netif_rx_internal(struct sk_buff *skb)
4495 {
4496         int ret;
4497 
4498         net_timestamp_check(netdev_tstamp_prequeue, skb);
4499 
4500         trace_netif_rx(skb);
4501 
4502         if (static_branch_unlikely(&generic_xdp_needed_key)) {
4503                 int ret;
4504 
4505                 preempt_disable();
4506                 rcu_read_lock();
4507                 ret = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
4508                 rcu_read_unlock();
4509                 preempt_enable();
4510 
4511                 /* Consider XDP consuming the packet a success from
4512                  * the netdev point of view we do not want to count
4513                  * this as an error.
4514                  */
4515                 if (ret != XDP_PASS)
4516                         return NET_RX_SUCCESS;
4517         }
4518 
4519 #ifdef CONFIG_RPS
4520         if (static_key_false(&rps_needed)) {
4521                 struct rps_dev_flow voidflow, *rflow = &voidflow;
4522                 int cpu;
4523 
4524                 preempt_disable();
4525                 rcu_read_lock();
4526 
4527                 cpu = get_rps_cpu(skb->dev, skb, &rflow);
4528                 if (cpu < 0)
4529                         cpu = smp_processor_id();
4530 
4531                 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4532 
4533                 rcu_read_unlock();
4534                 preempt_enable();
4535         } else
4536 #endif
4537         {
4538                 unsigned int qtail;
4539 
4540                 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
4541                 put_cpu();
4542         }
4543         return ret;
4544 }
4545 
4546 /**
4547  *      netif_rx        -       post buffer to the network code
4548  *      @skb: buffer to post
4549  *
4550  *      This function receives a packet from a device driver and queues it for
4551  *      the upper (protocol) levels to process.  It always succeeds. The buffer
4552  *      may be dropped during processing for congestion control or by the
4553  *      protocol layers.
4554  *
4555  *      return values:
4556  *      NET_RX_SUCCESS  (no congestion)
4557  *      NET_RX_DROP     (packet was dropped)
4558  *
4559  */
4560 
4561 int netif_rx(struct sk_buff *skb)
4562 {
4563         int ret;
4564 
4565         trace_netif_rx_entry(skb);
4566 
4567         ret = netif_rx_internal(skb);
4568         trace_netif_rx_exit(ret);
4569 
4570         return ret;
4571 }
4572 EXPORT_SYMBOL(netif_rx);
4573 
4574 int netif_rx_ni(struct sk_buff *skb)
4575 {
4576         int err;
4577 
4578         trace_netif_rx_ni_entry(skb);
4579 
4580         preempt_disable();
4581         err = netif_rx_internal(skb);
4582         if (local_softirq_pending())
4583                 do_softirq();
4584         preempt_enable();
4585         trace_netif_rx_ni_exit(err);
4586 
4587         return err;
4588 }
4589 EXPORT_SYMBOL(netif_rx_ni);
4590 
4591 static __latent_entropy void net_tx_action(struct softirq_action *h)
4592 {
4593         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
4594 
4595         if (sd->completion_queue) {
4596                 struct sk_buff *clist;
4597 
4598                 local_irq_disable();
4599                 clist = sd->completion_queue;
4600                 sd->completion_queue = NULL;
4601                 local_irq_enable();
4602 
4603                 while (clist) {
4604                         struct sk_buff *skb = clist;
4605 
4606                         clist = clist->next;
4607 
4608                         WARN_ON(refcount_read(&skb->users));
4609                         if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
4610                                 trace_consume_skb(skb);
4611                         else
4612                                 trace_kfree_skb(skb, net_tx_action);
4613 
4614                         if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
4615                                 __kfree_skb(skb);
4616                         else
4617                                 __kfree_skb_defer(skb);
4618                 }
4619 
4620                 __kfree_skb_flush();
4621         }
4622 
4623         if (sd->output_queue) {
4624                 struct Qdisc *head;
4625 
4626                 local_irq_disable();
4627                 head = sd->output_queue;
4628                 sd->output_queue = NULL;
4629                 sd->output_queue_tailp = &sd->output_queue;
4630                 local_irq_enable();
4631 
4632                 while (head) {
4633                         struct Qdisc *q = head;
4634                         spinlock_t *root_lock = NULL;
4635 
4636                         head = head->next_sched;
4637 
4638                         if (!(q->flags & TCQ_F_NOLOCK)) {
4639                                 root_lock = qdisc_lock(q);
4640                                 spin_lock(root_lock);
4641                         }
4642                         /* We need to make sure head->next_sched is read
4643                          * before clearing __QDISC_STATE_SCHED
4644                          */
4645                         smp_mb__before_atomic();
4646                         clear_bit(__QDISC_STATE_SCHED, &q->state);
4647                         qdisc_run(q);
4648                         if (root_lock)
4649                                 spin_unlock(root_lock);
4650                 }
4651         }
4652 
4653         xfrm_dev_backlog(sd);
4654 }
4655 
4656 #if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
4657 /* This hook is defined here for ATM LANE */
4658 int (*br_fdb_test_addr_hook)(struct net_device *dev,
4659                              unsigned char *addr) __read_mostly;
4660 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
4661 #endif
4662 
4663 static inline struct sk_buff *
4664 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
4665                    struct net_device *orig_dev)
4666 {
4667 #ifdef CONFIG_NET_CLS_ACT
4668         struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress);
4669         struct tcf_result cl_res;
4670 
4671         /* If there's at least one ingress present somewhere (so
4672          * we get here via enabled static key), remaining devices
4673          * that are not configured with an ingress qdisc will bail
4674          * out here.
4675          */
4676         if (!miniq)
4677                 return skb;
4678 
4679         if (*pt_prev) {
4680                 *ret = deliver_skb(skb, *pt_prev, orig_dev);
4681                 *pt_prev = NULL;
4682         }
4683 
4684         qdisc_skb_cb(skb)->pkt_len = skb->len;
4685         skb->tc_at_ingress = 1;
4686         mini_qdisc_bstats_cpu_update(miniq, skb);
4687 
4688         switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) {
4689         case TC_ACT_OK:
4690         case TC_ACT_RECLASSIFY:
4691                 skb->tc_index = TC_H_MIN(cl_res.classid);
4692                 break;
4693         case TC_ACT_SHOT:
4694                 mini_qdisc_qstats_cpu_drop(miniq);
4695                 kfree_skb(skb);
4696                 return NULL;
4697         case TC_ACT_STOLEN:
4698         case TC_ACT_QUEUED:
4699         case TC_ACT_TRAP:
4700                 consume_skb(skb);
4701                 return NULL;
4702         case TC_ACT_REDIRECT:
4703                 /* skb_mac_header check was done by cls/act_bpf, so
4704                  * we can safely push the L2 header back before
4705                  * redirecting to another netdev
4706                  */
4707                 __skb_push(skb, skb->mac_len);
4708                 skb_do_redirect(skb);
4709                 return NULL;
4710         case TC_ACT_REINSERT:
4711                 /* this does not scrub the packet, and updates stats on error */
4712                 skb_tc_reinsert(skb, &cl_res);
4713                 return NULL;
4714         default:
4715                 break;
4716         }
4717 #endif /* CONFIG_NET_CLS_ACT */
4718         return skb;
4719 }
4720 
4721 /**
4722  *      netdev_is_rx_handler_busy - check if receive handler is registered
4723  *      @dev: device to check
4724  *
4725  *      Check if a receive handler is already registered for a given device.
4726  *      Return true if there one.
4727  *
4728  *      The caller must hold the rtnl_mutex.
4729  */
4730 bool netdev_is_rx_handler_busy(struct net_device *dev)
4731 {
4732         ASSERT_RTNL();
4733         return dev && rtnl_dereference(dev->rx_handler);
4734 }
4735 EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
4736 
4737 /**
4738  *      netdev_rx_handler_register - register receive handler
4739  *      @dev: device to register a handler for
4740  *      @rx_handler: receive handler to register
4741  *      @rx_handler_data: data pointer that is used by rx handler
4742  *
4743  *      Register a receive handler for a device. This handler will then be
4744  *      called from __netif_receive_skb. A negative errno code is returned
4745  *      on a failure.
4746  *
4747  *      The caller must hold the rtnl_mutex.
4748  *
4749  *      For a general description of rx_handler, see enum rx_handler_result.
4750  */
4751 int netdev_rx_handler_register(struct net_device *dev,
4752                                rx_handler_func_t *rx_handler,
4753                                void *rx_handler_data)
4754 {
4755         if (netdev_is_rx_handler_busy(dev))
4756                 return -EBUSY;
4757 
4758         if (dev->priv_flags & IFF_NO_RX_HANDLER)
4759                 return -EINVAL;
4760 
4761         /* Note: rx_handler_data must be set before rx_handler */
4762         rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
4763         rcu_assign_pointer(dev->rx_handler, rx_handler);
4764 
4765         return 0;
4766 }
4767 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
4768 
4769 /**
4770  *      netdev_rx_handler_unregister - unregister receive handler
4771  *      @dev: device to unregister a handler from
4772  *
4773  *      Unregister a receive handler from a device.
4774  *
4775  *      The caller must hold the rtnl_mutex.
4776  */
4777 void netdev_rx_handler_unregister(struct net_device *dev)
4778 {
4779 
4780         ASSERT_RTNL();
4781         RCU_INIT_POINTER(dev->rx_handler, NULL);
4782         /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
4783          * section has a guarantee to see a non NULL rx_handler_data
4784          * as well.
4785          */
4786         synchronize_net();
4787         RCU_INIT_POINTER(dev->rx_handler_data, NULL);
4788 }
4789 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
4790 
4791 /*
4792  * Limit the use of PFMEMALLOC reserves to those protocols that implement
4793  * the special handling of PFMEMALLOC skbs.
4794  */
4795 static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
4796 {
4797         switch (skb->protocol) {
4798         case htons(ETH_P_ARP):
4799         case htons(ETH_P_IP):
4800         case htons(ETH_P_IPV6):
4801         case htons(ETH_P_8021Q):
4802         case htons(ETH_P_8021AD):
4803                 return true;
4804         default:
4805                 return false;
4806         }
4807 }
4808 
4809 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
4810                              int *ret, struct net_device *orig_dev)
4811 {
4812 #ifdef CONFIG_NETFILTER_INGRESS
4813         if (nf_hook_ingress_active(skb)) {
4814                 int ingress_retval;
4815 
4816                 if (*pt_prev) {
4817                         *ret = deliver_skb(skb, *pt_prev, orig_dev);
4818                         *pt_prev = NULL;
4819                 }
4820 
4821                 rcu_read_lock();
4822                 ingress_retval = nf_hook_ingress(skb);
4823                 rcu_read_unlock();
4824                 return ingress_retval;
4825         }
4826 #endif /* CONFIG_NETFILTER_INGRESS */
4827         return 0;
4828 }
4829 
4830 static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc,
4831                                     struct packet_type **ppt_prev)
4832 {
4833         struct packet_type *ptype, *pt_prev;
4834         rx_handler_func_t *rx_handler;
4835         struct net_device *orig_dev;
4836         bool deliver_exact = false;
4837         int ret = NET_RX_DROP;
4838         __be16 type;
4839 
4840         net_timestamp_check(!netdev_tstamp_prequeue, skb);
4841 
4842         trace_netif_receive_skb(skb);
4843 
4844         orig_dev = skb->dev;
4845 
4846         skb_reset_network_header(skb);
4847         if (!skb_transport_header_was_set(skb))
4848                 skb_reset_transport_header(skb);
4849         skb_reset_mac_len(skb);
4850 
4851         pt_prev = NULL;
4852 
4853 another_round:
4854         skb->skb_iif = skb->dev->ifindex;
4855 
4856         __this_cpu_inc(softnet_data.processed);
4857 
4858         if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
4859             skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
4860                 skb = skb_vlan_untag(skb);
4861                 if (unlikely(!skb))
4862                         goto out;
4863         }
4864 
4865         if (skb_skip_tc_classify(skb))
4866                 goto skip_classify;
4867 
4868         if (pfmemalloc)
4869                 goto skip_taps;
4870 
4871         list_for_each_entry_rcu(ptype, &ptype_all, list) {
4872                 if (pt_prev)
4873                         ret = deliver_skb(skb, pt_prev, orig_dev);
4874                 pt_prev = ptype;
4875         }
4876 
4877         list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
4878                 if (pt_prev)
4879                         ret = deliver_skb(skb, pt_prev, orig_dev);
4880                 pt_prev = ptype;
4881         }
4882 
4883 skip_taps:
4884 #ifdef CONFIG_NET_INGRESS
4885         if (static_branch_unlikely(&ingress_needed_key)) {
4886                 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev);
4887                 if (!skb)
4888                         goto out;
4889 
4890                 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
4891                         goto out;
4892         }
4893 #endif
4894         skb_reset_tc(skb);
4895 skip_classify:
4896         if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
4897                 goto drop;
4898 
4899         if (skb_vlan_tag_present(skb)) {
4900                 if (pt_prev) {
4901                         ret = deliver_skb(skb, pt_prev, orig_dev);
4902                         pt_prev = NULL;
4903                 }
4904                 if (vlan_do_receive(&skb))
4905                         goto another_round;
4906                 else if (unlikely(!skb))
4907                         goto out;
4908         }
4909 
4910         rx_handler = rcu_dereference(skb->dev->rx_handler);
4911         if (rx_handler) {
4912                 if (pt_prev) {
4913                         ret = deliver_skb(skb, pt_prev, orig_dev);
4914                         pt_prev = NULL;
4915                 }
4916                 switch (rx_handler(&skb)) {
4917                 case RX_HANDLER_CONSUMED:
4918                         ret = NET_RX_SUCCESS;
4919                         goto out;
4920                 case RX_HANDLER_ANOTHER:
4921                         goto another_round;
4922                 case RX_HANDLER_EXACT:
4923                         deliver_exact = true;
4924                 case RX_HANDLER_PASS:
4925                         break;
4926                 default:
4927                         BUG();
4928                 }
4929         }
4930 
4931         if (unlikely(skb_vlan_tag_present(skb))) {
4932                 if (skb_vlan_tag_get_id(skb))
4933                         skb->pkt_type = PACKET_OTHERHOST;
4934                 /* Note: we might in the future use prio bits
4935                  * and set skb->priority like in vlan_do_receive()
4936                  * For the time being, just ignore Priority Code Point
4937                  */
4938                 __vlan_hwaccel_clear_tag(skb);
4939         }
4940 
4941         type = skb->protocol;
4942 
4943         /* deliver only exact match when indicated */
4944         if (likely(!deliver_exact)) {
4945                 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4946                                        &ptype_base[ntohs(type) &
4947                                                    PTYPE_HASH_MASK]);
4948         }
4949 
4950         deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4951                                &orig_dev->ptype_specific);
4952 
4953         if (unlikely(skb->dev != orig_dev)) {
4954                 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4955                                        &skb->dev->ptype_specific);
4956         }
4957 
4958         if (pt_prev) {
4959                 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
4960                         goto drop;
4961                 *ppt_prev = pt_prev;
4962         } else {
4963 drop:
4964                 if (!deliver_exact)
4965                         atomic_long_inc(&skb->dev->rx_dropped);
4966                 else
4967                         atomic_long_inc(&skb->dev->rx_nohandler);
4968                 kfree_skb(skb);
4969                 /* Jamal, now you will not able to escape explaining
4970                  * me how you were going to use this. :-)
4971                  */
4972                 ret = NET_RX_DROP;
4973         }
4974 
4975 out:
4976         return ret;
4977 }
4978 
4979 static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc)
4980 {
4981         struct net_device *orig_dev = skb->dev;
4982         struct packet_type *pt_prev = NULL;
4983         int ret;
4984 
4985         ret = __netif_receive_skb_core(skb, pfmemalloc, &pt_prev);
4986         if (pt_prev)
4987                 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
4988         return ret;
4989 }
4990 
4991 /**
4992  *      netif_receive_skb_core - special purpose version of netif_receive_skb
4993  *      @skb: buffer to process
4994  *
4995  *      More direct receive version of netif_receive_skb().  It should
4996  *      only be used by callers that have a need to skip RPS and Generic XDP.
4997  *      Caller must also take care of handling if (page_is_)pfmemalloc.
4998  *
4999  *      This function may only be called from softirq context and interrupts
5000  *      should be enabled.
5001  *
5002  *      Return values (usually ignored):
5003  *      NET_RX_SUCCESS: no congestion
5004  *      NET_RX_DROP: packet was dropped
5005  */
5006 int netif_receive_skb_core(struct sk_buff *skb)
5007 {
5008         int ret;
5009 
5010         rcu_read_lock();
5011         ret = __netif_receive_skb_one_core(skb, false);
5012         rcu_read_unlock();
5013 
5014         return ret;
5015 }
5016 EXPORT_SYMBOL(netif_receive_skb_core);
5017 
5018 static inline void __netif_receive_skb_list_ptype(struct list_head *head,
5019                                                   struct packet_type *pt_prev,
5020                                                   struct net_device *orig_dev)
5021 {
5022         struct sk_buff *skb, *next;
5023 
5024         if (!pt_prev)
5025                 return;
5026         if (list_empty(head))
5027                 return;
5028         if (pt_prev->list_func != NULL)
5029                 pt_prev->list_func(head, pt_prev, orig_dev);
5030         else
5031                 list_for_each_entry_safe(skb, next, head, list) {
5032                         skb_list_del_init(skb);
5033                         pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
5034                 }
5035 }
5036 
5037 static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc)
5038 {
5039         /* Fast-path assumptions:
5040          * - There is no RX handler.
5041          * - Only one packet_type matches.
5042          * If either of these fails, we will end up doing some per-packet
5043          * processing in-line, then handling the 'last ptype' for the whole
5044          * sublist.  This can't cause out-of-order delivery to any single ptype,
5045          * because the 'last ptype' must be constant across the sublist, and all
5046          * other ptypes are handled per-packet.
5047          */
5048         /* Current (common) ptype of sublist */
5049         struct packet_type *pt_curr = NULL;
5050         /* Current (common) orig_dev of sublist */
5051         struct net_device *od_curr = NULL;
5052         struct list_head sublist;
5053         struct sk_buff *skb, *next;
5054 
5055         INIT_LIST_HEAD(&sublist);
5056         list_for_each_entry_safe(skb, next, head, list) {
5057                 struct net_device *orig_dev = skb->dev;
5058                 struct packet_type *pt_prev = NULL;
5059 
5060                 skb_list_del_init(skb);
5061                 __netif_receive_skb_core(skb, pfmemalloc, &pt_prev);
5062                 if (!pt_prev)
5063                         continue;
5064                 if (pt_curr != pt_prev || od_curr != orig_dev) {
5065                         /* dispatch old sublist */
5066                         __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5067                         /* start new sublist */
5068                         INIT_LIST_HEAD(&sublist);
5069                         pt_curr = pt_prev;
5070                         od_curr = orig_dev;
5071                 }
5072                 list_add_tail(&skb->list, &sublist);
5073         }
5074 
5075         /* dispatch final sublist */
5076         __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5077 }
5078 
5079 static int __netif_receive_skb(struct sk_buff *skb)
5080 {
5081         int ret;
5082 
5083         if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
5084                 unsigned int noreclaim_flag;
5085 
5086                 /*
5087                  * PFMEMALLOC skbs are special, they should
5088                  * - be delivered to SOCK_MEMALLOC sockets only
5089                  * - stay away from userspace
5090                  * - have bounded memory usage
5091                  *
5092                  * Use PF_MEMALLOC as this saves us from propagating the allocation
5093                  * context down to all allocation sites.
5094                  */
5095                 noreclaim_flag = memalloc_noreclaim_save();
5096                 ret = __netif_receive_skb_one_core(skb, true);
5097                 memalloc_noreclaim_restore(noreclaim_flag);
5098         } else
5099                 ret = __netif_receive_skb_one_core(skb, false);
5100 
5101         return ret;
5102 }
5103 
5104 static void __netif_receive_skb_list(struct list_head *head)
5105 {
5106         unsigned long noreclaim_flag = 0;
5107         struct sk_buff *skb, *next;
5108         bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */
5109 
5110         list_for_each_entry_safe(skb, next, head, list) {
5111                 if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) {
5112                         struct list_head sublist;
5113 
5114                         /* Handle the previous sublist */
5115                         list_cut_before(&sublist, head, &skb->list);
5116                         if (!list_empty(&sublist))
5117                                 __netif_receive_skb_list_core(&sublist, pfmemalloc);
5118                         pfmemalloc = !pfmemalloc;
5119                         /* See comments in __netif_receive_skb */
5120                         if (pfmemalloc)
5121                                 noreclaim_flag = memalloc_noreclaim_save();
5122                         else
5123                                 memalloc_noreclaim_restore(noreclaim_flag);
5124                 }
5125         }
5126         /* Handle the remaining sublist */
5127         if (!list_empty(head))
5128                 __netif_receive_skb_list_core(head, pfmemalloc);
5129         /* Restore pflags */
5130         if (pfmemalloc)
5131                 memalloc_noreclaim_restore(noreclaim_flag);
5132 }
5133 
5134 static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
5135 {
5136         struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
5137         struct bpf_prog *new = xdp->prog;
5138         int ret = 0;
5139 
5140         switch (xdp->command) {
5141         case XDP_SETUP_PROG:
5142                 rcu_assign_pointer(dev->xdp_prog, new);
5143                 if (old)
5144                         bpf_prog_put(old);
5145 
5146                 if (old && !new) {
5147                         static_branch_dec(&generic_xdp_needed_key);
5148                 } else if (new && !old) {
5149                         static_branch_inc(&generic_xdp_needed_key);
5150                         dev_disable_lro(dev);
5151                         dev_disable_gro_hw(dev);
5152                 }
5153                 break;
5154 
5155         case XDP_QUERY_PROG:
5156                 xdp->prog_id = old ? old->aux->id : 0;
5157                 break;
5158 
5159         default:
5160                 ret = -EINVAL;
5161                 break;
5162         }
5163 
5164         return ret;
5165 }
5166 
5167 static int netif_receive_skb_internal(struct sk_buff *skb)
5168 {
5169         int ret;
5170 
5171         net_timestamp_check(netdev_tstamp_prequeue, skb);
5172 
5173         if (skb_defer_rx_timestamp(skb))
5174                 return NET_RX_SUCCESS;
5175 
5176         if (static_branch_unlikely(&generic_xdp_needed_key)) {
5177                 int ret;
5178 
5179                 preempt_disable();
5180                 rcu_read_lock();
5181                 ret = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
5182                 rcu_read_unlock();
5183                 preempt_enable();
5184 
5185                 if (ret != XDP_PASS)
5186                         return NET_RX_DROP;
5187         }
5188 
5189         rcu_read_lock();
5190 #ifdef CONFIG_RPS
5191         if (static_key_false(&rps_needed)) {
5192                 struct rps_dev_flow voidflow, *rflow = &voidflow;
5193                 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5194 
5195                 if (cpu >= 0) {
5196                         ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5197                         rcu_read_unlock();
5198                         return ret;
5199                 }
5200         }
5201 #endif
5202         ret = __netif_receive_skb(skb);
5203         rcu_read_unlock();
5204         return ret;
5205 }
5206 
5207 static void netif_receive_skb_list_internal(struct list_head *head)
5208 {
5209         struct bpf_prog *xdp_prog = NULL;
5210         struct sk_buff *skb, *next;
5211         struct list_head sublist;
5212 
5213         INIT_LIST_HEAD(&sublist);
5214         list_for_each_entry_safe(skb, next, head, list) {
5215                 net_timestamp_check(netdev_tstamp_prequeue, skb);
5216                 skb_list_del_init(skb);
5217                 if (!skb_defer_rx_timestamp(skb))
5218                         list_add_tail(&skb->list, &sublist);
5219         }
5220         list_splice_init(&sublist, head);
5221 
5222         if (static_branch_unlikely(&generic_xdp_needed_key)) {
5223                 preempt_disable();
5224                 rcu_read_lock();
5225                 list_for_each_entry_safe(skb, next, head, list) {
5226                         xdp_prog = rcu_dereference(skb->dev->xdp_prog);
5227                         skb_list_del_init(skb);
5228                         if (do_xdp_generic(xdp_prog, skb) == XDP_PASS)
5229                                 list_add_tail(&skb->list, &sublist);
5230                 }
5231                 rcu_read_unlock();
5232                 preempt_enable();
5233                 /* Put passed packets back on main list */
5234                 list_splice_init(&sublist, head);
5235         }
5236 
5237         rcu_read_lock();
5238 #ifdef CONFIG_RPS
5239         if (static_key_false(&rps_needed)) {
5240                 list_for_each_entry_safe(skb, next, head, list) {
5241                         struct rps_dev_flow voidflow, *rflow = &voidflow;
5242                         int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5243 
5244                         if (cpu >= 0) {
5245                                 /* Will be handled, remove from list */
5246                                 skb_list_del_init(skb);
5247                                 enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5248                         }
5249                 }
5250         }
5251 #endif
5252         __netif_receive_skb_list(head);
5253         rcu_read_unlock();
5254 }
5255 
5256 /**
5257  *      netif_receive_skb - process receive buffer from network
5258  *      @skb: buffer to process
5259  *
5260  *      netif_receive_skb() is the main receive data processing function.
5261  *      It always succeeds. The buffer may be dropped during processing
5262  *      for congestion control or by the protocol layers.
5263  *
5264  *      This function may only be called from softirq context and interrupts
5265  *      should be enabled.
5266  *
5267  *      Return values (usually ignored):
5268  *      NET_RX_SUCCESS: no congestion
5269  *      NET_RX_DROP: packet was dropped
5270  */
5271 int netif_receive_skb(struct sk_buff *skb)
5272 {
5273         int ret;
5274 
5275         trace_netif_receive_skb_entry(skb);
5276 
5277         ret = netif_receive_skb_internal(skb);
5278         trace_netif_receive_skb_exit(ret);
5279 
5280         return ret;
5281 }
5282 EXPORT_SYMBOL(netif_receive_skb);
5283 
5284 /**
5285  *      netif_receive_skb_list - process many receive buffers from network
5286  *      @head: list of skbs to process.
5287  *
5288  *      Since return value of netif_receive_skb() is normally ignored, and
5289  *      wouldn't be meaningful for a list, this function returns void.
5290  *
5291  *      This function may only be called from softirq context and interrupts
5292  *      should be enabled.
5293  */
5294 void netif_receive_skb_list(struct list_head *head)
5295 {
5296         struct sk_buff *skb;
5297 
5298         if (list_empty(head))
5299                 return;
5300         if (trace_netif_receive_skb_list_entry_enabled()) {
5301                 list_for_each_entry(skb, head, list)
5302                         trace_netif_receive_skb_list_entry(skb);
5303         }
5304         netif_receive_skb_list_internal(head);
5305         trace_netif_receive_skb_list_exit(0);
5306 }
5307 EXPORT_SYMBOL(netif_receive_skb_list);
5308 
5309 DEFINE_PER_CPU(struct work_struct, flush_works);
5310 
5311 /* Network device is going away, flush any packets still pending */
5312 static void flush_backlog(struct work_struct *work)
5313 {
5314         struct sk_buff *skb, *tmp;
5315         struct softnet_data *sd;
5316 
5317         local_bh_disable();
5318         sd = this_cpu_ptr(&softnet_data);
5319 
5320         local_irq_disable();
5321         rps_lock(sd);
5322         skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
5323                 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
5324                         __skb_unlink(skb, &sd->input_pkt_queue);
5325                         kfree_skb(skb);
5326                         input_queue_head_incr(sd);
5327                 }
5328         }
5329         rps_unlock(sd);
5330         local_irq_enable();
5331 
5332         skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
5333                 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
5334                         __skb_unlink(skb, &sd->process_queue);
5335                         kfree_skb(skb);
5336                         input_queue_head_incr(sd);
5337                 }
5338         }
5339         local_bh_enable();
5340 }
5341 
5342 static void flush_all_backlogs(void)
5343 {
5344         unsigned int cpu;
5345 
5346         get_online_cpus();
5347 
5348         for_each_online_cpu(cpu)
5349                 queue_work_on(cpu, system_highpri_wq,
5350                               per_cpu_ptr(&flush_works, cpu));
5351 
5352         for_each_online_cpu(cpu)
5353                 flush_work(per_cpu_ptr(&flush_works, cpu));
5354 
5355         put_online_cpus();
5356 }
5357 
5358 INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int));
5359 INDIRECT_CALLABLE_DECLARE(int ipv6_gro_complete(struct sk_buff *, int));
5360 static int napi_gro_complete(struct sk_buff *skb)
5361 {
5362         struct packet_offload *ptype;
5363         __be16 type = skb->protocol;
5364         struct list_head *head = &offload_base;
5365         int err = -ENOENT;
5366 
5367         BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
5368 
5369         if (NAPI_GRO_CB(skb)->count == 1) {
5370                 skb_shinfo(skb)->gso_size = 0;
5371                 goto out;
5372         }
5373 
5374         rcu_read_lock();
5375         list_for_each_entry_rcu(ptype, head, list) {
5376                 if (ptype->type != type || !ptype->callbacks.gro_complete)
5377                         continue;
5378 
5379                 err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete,
5380                                          ipv6_gro_complete, inet_gro_complete,
5381                                          skb, 0);
5382                 break;
5383         }
5384         rcu_read_unlock();
5385 
5386         if (err) {
5387                 WARN_ON(&ptype->list == head);
5388                 kfree_skb(skb);
5389                 return NET_RX_SUCCESS;
5390         }
5391 
5392 out:
5393         return netif_receive_skb_internal(skb);
5394 }
5395 
5396 static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index,
5397                                    bool flush_old)
5398 {
5399         struct list_head *head = &napi->gro_hash[index].list;
5400         struct sk_buff *skb, *p;
5401 
5402         list_for_each_entry_safe_reverse(skb, p, head, list) {
5403                 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
5404                         return;
5405                 skb_list_del_init(skb);
5406                 napi_gro_complete(skb);
5407                 napi->gro_hash[index].count--;
5408         }
5409 
5410         if (!napi->gro_hash[index].count)
5411                 __clear_bit(index, &napi->gro_bitmask);
5412 }
5413 
5414 /* napi->gro_hash[].list contains packets ordered by age.
5415  * youngest packets at the head of it.
5416  * Complete skbs in reverse order to reduce latencies.
5417  */
5418 void napi_gro_flush(struct napi_struct *napi, bool flush_old)
5419 {
5420         unsigned long bitmask = napi->gro_bitmask;
5421         unsigned int i, base = ~0U;
5422 
5423         while ((i = ffs(bitmask)) != 0) {
5424                 bitmask >>= i;
5425                 base += i;
5426                 __napi_gro_flush_chain(napi, base, flush_old);
5427         }
5428 }
5429 EXPORT_SYMBOL(napi_gro_flush);
5430 
5431 static struct list_head *gro_list_prepare(struct napi_struct *napi,
5432                                           struct sk_buff *skb)
5433 {
5434         unsigned int maclen = skb->dev->hard_header_len;
5435         u32 hash = skb_get_hash_raw(skb);
5436         struct list_head *head;
5437         struct sk_buff *p;
5438 
5439         head = &napi->gro_hash[hash & (GRO_HASH_BUCKETS - 1)].list;
5440         list_for_each_entry(p, head, list) {
5441                 unsigned long diffs;
5442 
5443                 NAPI_GRO_CB(p)->flush = 0;
5444 
5445                 if (hash != skb_get_hash_raw(p)) {
5446                         NAPI_GRO_CB(p)->same_flow = 0;
5447                         continue;
5448                 }
5449 
5450                 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
5451                 diffs |= skb_vlan_tag_present(p) ^ skb_vlan_tag_present(skb);
5452                 if (skb_vlan_tag_present(p))
5453                         diffs |= p->vlan_tci ^ skb->vlan_tci;
5454                 diffs |= skb_metadata_dst_cmp(p, skb);
5455                 diffs |= skb_metadata_differs(p, skb);
5456                 if (maclen == ETH_HLEN)
5457                         diffs |= compare_ether_header(skb_mac_header(p),
5458                                                       skb_mac_header(skb));
5459                 else if (!diffs)
5460                         diffs = memcmp(skb_mac_header(p),
5461                                        skb_mac_header(skb),
5462                                        maclen);
5463                 NAPI_GRO_CB(p)->same_flow = !diffs;
5464         }
5465 
5466         return head;
5467 }
5468 
5469 static void skb_gro_reset_offset(struct sk_buff *skb)
5470 {
5471         const struct skb_shared_info *pinfo = skb_shinfo(skb);
5472         const skb_frag_t *frag0 = &pinfo->frags[0];
5473 
5474         NAPI_GRO_CB(skb)->data_offset = 0;
5475         NAPI_GRO_CB(skb)->frag0 = NULL;
5476         NAPI_GRO_CB(skb)->frag0_len = 0;
5477 
5478         if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
5479             pinfo->nr_frags &&
5480             !PageHighMem(skb_frag_page(frag0))) {
5481                 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
5482                 NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
5483                                                     skb_frag_size(frag0),
5484                                                     skb->end - skb->tail);
5485         }
5486 }
5487 
5488 static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
5489 {
5490         struct skb_shared_info *pinfo = skb_shinfo(skb);
5491 
5492         BUG_ON(skb->end - skb->tail < grow);
5493 
5494         memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
5495 
5496         skb->data_len -= grow;
5497         skb->tail += grow;
5498 
5499         pinfo->frags[0].page_offset += grow;
5500         skb_frag_size_sub(&pinfo->frags[0], grow);
5501 
5502         if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
5503                 skb_frag_unref(skb, 0);
5504                 memmove(pinfo->frags, pinfo->frags + 1,
5505                         --pinfo->nr_frags * sizeof(pinfo->frags[0]));
5506         }
5507 }
5508 
5509 static void gro_flush_oldest(struct list_head *head)
5510 {
5511         struct sk_buff *oldest;
5512 
5513         oldest = list_last_entry(head, struct sk_buff, list);
5514 
5515         /* We are called with head length >= MAX_GRO_SKBS, so this is
5516          * impossible.
5517          */
5518         if (WARN_ON_ONCE(!oldest))
5519                 return;
5520 
5521         /* Do not adjust napi->gro_hash[].count, caller is adding a new
5522          * SKB to the chain.
5523          */
5524         skb_list_del_init(oldest);
5525         napi_gro_complete(oldest);
5526 }
5527 
5528 INDIRECT_CALLABLE_DECLARE(struct sk_buff *inet_gro_receive(struct list_head *,
5529                                                            struct sk_buff *));
5530 INDIRECT_CALLABLE_DECLARE(struct sk_buff *ipv6_gro_receive(struct list_head *,
5531                                                            struct sk_buff *));
5532 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
5533 {
5534         u32 hash = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1);
5535         struct list_head *head = &offload_base;
5536         struct packet_offload *ptype;
5537         __be16 type = skb->protocol;
5538         struct list_head *gro_head;
5539         struct sk_buff *pp = NULL;
5540         enum gro_result ret;
5541         int same_flow;
5542         int grow;
5543 
5544         if (netif_elide_gro(skb->dev))
5545                 goto normal;
5546 
5547         gro_head = gro_list_prepare(napi, skb);
5548 
5549         rcu_read_lock();
5550         list_for_each_entry_rcu(ptype, head, list) {
5551                 if (ptype->type != type || !ptype->callbacks.gro_receive)
5552                         continue;
5553 
5554                 skb_set_network_header(skb, skb_gro_offset(skb));
5555                 skb_reset_mac_len(skb);
5556                 NAPI_GRO_CB(skb)->same_flow = 0;
5557                 NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb);
5558                 NAPI_GRO_CB(skb)->free = 0;
5559                 NAPI_GRO_CB(skb)->encap_mark = 0;
5560                 NAPI_GRO_CB(skb)->recursion_counter = 0;
5561                 NAPI_GRO_CB(skb)->is_fou = 0;
5562                 NAPI_GRO_CB(skb)->is_atomic = 1;
5563                 NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
5564 
5565                 /* Setup for GRO checksum validation */
5566                 switch (skb->ip_summed) {
5567                 case CHECKSUM_COMPLETE:
5568                         NAPI_GRO_CB(skb)->csum = skb->csum;
5569                         NAPI_GRO_CB(skb)->csum_valid = 1;
5570                         NAPI_GRO_CB(skb)->csum_cnt = 0;
5571                         break;
5572                 case CHECKSUM_UNNECESSARY:
5573                         NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
5574                         NAPI_GRO_CB(skb)->csum_valid = 0;
5575                         break;
5576                 default:
5577                         NAPI_GRO_CB(skb)->csum_cnt = 0;
5578                         NAPI_GRO_CB(skb)->csum_valid = 0;
5579                 }
5580 
5581                 pp = INDIRECT_CALL_INET(ptype->callbacks.gro_receive,
5582                                         ipv6_gro_receive, inet_gro_receive,
5583                                         gro_head, skb);
5584                 break;
5585         }
5586         rcu_read_unlock();
5587 
5588         if (&ptype->list == head)
5589                 goto normal;
5590 
5591         if (IS_ERR(pp) && PTR_ERR(pp) == -EINPROGRESS) {
5592                 ret = GRO_CONSUMED;
5593                 goto ok;
5594         }
5595 
5596         same_flow = NAPI_GRO_CB(skb)->same_flow;
5597         ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
5598 
5599         if (pp) {
5600                 skb_list_del_init(pp);
5601                 napi_gro_complete(pp);
5602                 napi->gro_hash[hash].count--;
5603         }
5604 
5605         if (same_flow)
5606                 goto ok;
5607 
5608         if (NAPI_GRO_CB(skb)->flush)
5609                 goto normal;
5610 
5611         if (unlikely(napi->gro_hash[hash].count >= MAX_GRO_SKBS)) {
5612                 gro_flush_oldest(gro_head);
5613         } else {
5614                 napi->gro_hash[hash].count++;
5615         }
5616         NAPI_GRO_CB(skb)->count = 1;
5617         NAPI_GRO_CB(skb)->age = jiffies;
5618         NAPI_GRO_CB(skb)->last = skb;
5619         skb_shinfo(skb)->gso_size = skb_gro_len(skb);
5620         list_add(&skb->list, gro_head);
5621         ret = GRO_HELD;
5622 
5623 pull:
5624         grow = skb_gro_offset(skb) - skb_headlen(skb);
5625         if (grow > 0)
5626                 gro_pull_from_frag0(skb, grow);
5627 ok:
5628         if (napi->gro_hash[hash].count) {
5629                 if (!test_bit(hash, &napi->gro_bitmask))
5630                         __set_bit(hash, &napi->gro_bitmask);
5631         } else if (test_bit(hash, &napi->gro_bitmask)) {
5632                 __clear_bit(hash, &napi->gro_bitmask);
5633         }
5634 
5635         return ret;
5636 
5637 normal:
5638         ret = GRO_NORMAL;
5639         goto pull;
5640 }
5641 
5642 struct packet_offload *gro_find_receive_by_type(__be16 type)
5643 {
5644         struct list_head *offload_head = &offload_base;
5645         struct packet_offload *ptype;
5646 
5647         list_for_each_entry_rcu(ptype, offload_head, list) {
5648                 if (ptype->type != type || !ptype->callbacks.gro_receive)
5649                         continue;
5650                 return ptype;
5651         }
5652         return NULL;
5653 }
5654 EXPORT_SYMBOL(gro_find_receive_by_type);
5655 
5656 struct packet_offload *gro_find_complete_by_type(__be16 type)
5657 {
5658         struct list_head *offload_head = &offload_base;
5659         struct packet_offload *ptype;
5660 
5661         list_for_each_entry_rcu(ptype, offload_head, list) {
5662                 if (ptype->type != type || !ptype->callbacks.gro_complete)
5663                         continue;
5664                 return ptype;
5665         }
5666         return NULL;
5667 }
5668 EXPORT_SYMBOL(gro_find_complete_by_type);
5669 
5670 static void napi_skb_free_stolen_head(struct sk_buff *skb)
5671 {
5672         skb_dst_drop(skb);
5673         secpath_reset(skb);
5674         kmem_cache_free(skbuff_head_cache, skb);
5675 }
5676 
5677 static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
5678 {
5679         switch (ret) {
5680         case GRO_NORMAL:
5681                 if (netif_receive_skb_internal(skb))
5682                         ret = GRO_DROP;
5683                 break;
5684 
5685         case GRO_DROP:
5686                 kfree_skb(skb);
5687                 break;
5688 
5689         case GRO_MERGED_FREE:
5690                 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
5691                         napi_skb_free_stolen_head(skb);
5692                 else
5693                         __kfree_skb(skb);
5694                 break;
5695 
5696         case GRO_HELD:
5697         case GRO_MERGED:
5698         case GRO_CONSUMED:
5699                 break;
5700         }
5701 
5702         return ret;
5703 }
5704 
5705 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
5706 {
5707         gro_result_t ret;
5708 
5709         skb_mark_napi_id(skb, napi);
5710         trace_napi_gro_receive_entry(skb);
5711 
5712         skb_gro_reset_offset(skb);
5713 
5714         ret = napi_skb_finish(dev_gro_receive(napi, skb), skb);
5715         trace_napi_gro_receive_exit(ret);
5716 
5717         return ret;
5718 }
5719 EXPORT_SYMBOL(napi_gro_receive);
5720 
5721 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
5722 {
5723         if (unlikely(skb->pfmemalloc)) {
5724                 consume_skb(skb);
5725                 return;
5726         }
5727         __skb_pull(skb, skb_headlen(skb));
5728         /* restore the reserve we had after netdev_alloc_skb_ip_align() */
5729         skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
5730         __vlan_hwaccel_clear_tag(skb);
5731         skb->dev = napi->dev;
5732         skb->skb_iif = 0;
5733 
5734         /* eth_type_trans() assumes pkt_type is PACKET_HOST */
5735         skb->pkt_type = PACKET_HOST;
5736 
5737         skb->encapsulation = 0;
5738         skb_shinfo(skb)->gso_type = 0;
5739         skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
5740         secpath_reset(skb);
5741 
5742         napi->skb = skb;
5743 }
5744 
5745 struct sk_buff *napi_get_frags(struct napi_struct *napi)
5746 {
5747         struct sk_buff *skb = napi->skb;
5748 
5749         if (!skb) {
5750                 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
5751                 if (skb) {
5752                         napi->skb = skb;
5753                         skb_mark_napi_id(skb, napi);
5754                 }
5755         }
5756         return skb;
5757 }
5758 EXPORT_SYMBOL(napi_get_frags);
5759 
5760 static gro_result_t napi_frags_finish(struct napi_struct *napi,
5761                                       struct sk_buff *skb,
5762                                       gro_result_t ret)
5763 {
5764         switch (ret) {
5765         case GRO_NORMAL:
5766         case GRO_HELD:
5767                 __skb_push(skb, ETH_HLEN);
5768                 skb->protocol = eth_type_trans(skb, skb->dev);
5769                 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
5770                         ret = GRO_DROP;
5771                 break;
5772 
5773         case GRO_DROP:
5774                 napi_reuse_skb(napi, skb);
5775                 break;
5776 
5777         case GRO_MERGED_FREE:
5778                 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
5779                         napi_skb_free_stolen_head(skb);
5780                 else
5781                         napi_reuse_skb(napi, skb);
5782                 break;
5783 
5784         case GRO_MERGED:
5785         case GRO_CONSUMED:
5786                 break;
5787         }
5788 
5789         return ret;
5790 }
5791 
5792 /* Upper GRO stack assumes network header starts at gro_offset=0
5793  * Drivers could call both napi_gro_frags() and napi_gro_receive()
5794  * We copy ethernet header into skb->data to have a common layout.
5795  */
5796 static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
5797 {
5798         struct sk_buff *skb = napi->skb;
5799         const struct ethhdr *eth;
5800         unsigned int hlen = sizeof(*eth);
5801 
5802         napi->skb = NULL;
5803 
5804         skb_reset_mac_header(skb);
5805         skb_gro_reset_offset(skb);
5806 
5807         if (unlikely(skb_gro_header_hard(skb, hlen))) {
5808                 eth = skb_gro_header_slow(skb, hlen, 0);
5809                 if (unlikely(!eth)) {
5810                         net_warn_ratelimited("%s: dropping impossible skb from %s\n",
5811                                              __func__, napi->dev->name);
5812                         napi_reuse_skb(napi, skb);
5813                         return NULL;
5814                 }
5815         } else {
5816                 eth = (const struct ethhdr *)skb->data;
5817                 gro_pull_from_frag0(skb, hlen);
5818                 NAPI_GRO_CB(skb)->frag0 += hlen;
5819                 NAPI_GRO_CB(skb)->frag0_len -= hlen;
5820         }
5821         __skb_pull(skb, hlen);
5822 
5823         /*
5824          * This works because the only protocols we care about don't require
5825          * special handling.
5826          * We'll fix it up properly in napi_frags_finish()
5827          */
5828         skb->protocol = eth->h_proto;
5829 
5830         return skb;
5831 }
5832 
5833 gro_result_t napi_gro_frags(struct napi_struct *napi)
5834 {
5835         gro_result_t ret;
5836         struct sk_buff *skb = napi_frags_skb(napi);
5837 
5838         if (!skb)
5839                 return GRO_DROP;
5840 
5841         trace_napi_gro_frags_entry(skb);
5842 
5843         ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
5844         trace_napi_gro_frags_exit(ret);
5845 
5846         return ret;
5847 }
5848 EXPORT_SYMBOL(napi_gro_frags);
5849 
5850 /* Compute the checksum from gro_offset and return the folded value
5851  * after adding in any pseudo checksum.
5852  */
5853 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
5854 {
5855         __wsum wsum;
5856         __sum16 sum;
5857 
5858         wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
5859 
5860         /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
5861         sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
5862         /* See comments in __skb_checksum_complete(). */
5863         if (likely(!sum)) {
5864                 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
5865                     !skb->csum_complete_sw)
5866                         netdev_rx_csum_fault(skb->dev, skb);
5867         }
5868 
5869         NAPI_GRO_CB(skb)->csum = wsum;
5870         NAPI_GRO_CB(skb)->csum_valid = 1;
5871 
5872         return sum;
5873 }
5874 EXPORT_SYMBOL(__skb_gro_checksum_complete);
5875 
5876 static void net_rps_send_ipi(struct softnet_data *remsd)
5877 {
5878 #ifdef CONFIG_RPS
5879         while (remsd) {
5880                 struct softnet_data *next = remsd->rps_ipi_next;
5881 
5882                 if (cpu_online(remsd->cpu))
5883                         smp_call_function_single_async(remsd->cpu, &remsd->csd);
5884                 remsd = next;
5885         }
5886 #endif
5887 }
5888 
5889 /*
5890  * net_rps_action_and_irq_enable sends any pending IPI's for rps.
5891  * Note: called with local irq disabled, but exits with local irq enabled.
5892  */
5893 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
5894 {
5895 #ifdef CONFIG_RPS
5896         struct softnet_data *remsd = sd->rps_ipi_list;
5897 
5898         if (remsd) {
5899                 sd->rps_ipi_list = NULL;
5900 
5901                 local_irq_enable();
5902 
5903                 /* Send pending IPI's to kick RPS processing on remote cpus. */
5904                 net_rps_send_ipi(remsd);
5905         } else
5906 #endif
5907                 local_irq_enable();
5908 }
5909 
5910 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
5911 {
5912 #ifdef CONFIG_RPS
5913         return sd->rps_ipi_list != NULL;
5914 #else
5915         return false;
5916 #endif
5917 }
5918 
5919 static int process_backlog(struct napi_struct *napi, int quota)
5920 {
5921         struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
5922         bool again = true;
5923         int work = 0;
5924 
5925         /* Check if we have pending ipi, its better to send them now,
5926          * not waiting net_rx_action() end.
5927          */
5928         if (sd_has_rps_ipi_waiting(sd)) {
5929                 local_irq_disable();
5930                 net_rps_action_and_irq_enable(sd);
5931         }
5932 
5933         napi->weight = dev_rx_weight;
5934         while (again) {
5935                 struct sk_buff *skb;
5936 
5937                 while ((skb = __skb_dequeue(&sd->process_queue))) {
5938                         rcu_read_lock();
5939                         __netif_receive_skb(skb);
5940                         rcu_read_unlock();
5941                         input_queue_head_incr(sd);
5942                         if (++work >= quota)
5943                                 return work;
5944 
5945                 }
5946 
5947                 local_irq_disable();
5948                 rps_lock(sd);
5949                 if (skb_queue_empty(&sd->input_pkt_queue)) {
5950                         /*
5951                          * Inline a custom version of __napi_complete().
5952                          * only current cpu owns and manipulates this napi,
5953                          * and NAPI_STATE_SCHED is the only possible flag set
5954                          * on backlog.
5955                          * We can use a plain write instead of clear_bit(),
5956                          * and we dont need an smp_mb() memory barrier.
5957                          */
5958                         napi->state = 0;
5959                         again = false;
5960                 } else {
5961                         skb_queue_splice_tail_init(&sd->input_pkt_queue,
5962                                                    &sd->process_queue);
5963                 }
5964                 rps_unlock(sd);
5965                 local_irq_enable();
5966         }
5967 
5968         return work;
5969 }
5970 
5971 /**
5972  * __napi_schedule - schedule for receive
5973  * @n: entry to schedule
5974  *
5975  * The entry's receive function will be scheduled to run.
5976  * Consider using __napi_schedule_irqoff() if hard irqs are masked.
5977  */
5978 void __napi_schedule(struct napi_struct *n)
5979 {
5980         unsigned long flags;
5981 
5982         local_irq_save(flags);
5983         ____napi_schedule(this_cpu_ptr(&softnet_data), n);
5984         local_irq_restore(flags);
5985 }
5986 EXPORT_SYMBOL(__napi_schedule);
5987 
5988 /**
5989  *      napi_schedule_prep - check if napi can be scheduled
5990  *      @n: napi context
5991  *
5992  * Test if NAPI routine is already running, and if not mark
5993  * it as running.  This is used as a condition variable
5994  * insure only one NAPI poll instance runs.  We also make
5995  * sure there is no pending NAPI disable.
5996  */
5997 bool napi_schedule_prep(struct napi_struct *n)
5998 {
5999         unsigned long val, new;
6000 
6001         do {
6002                 val = READ_ONCE(n->state);
6003                 if (unlikely(val & NAPIF_STATE_DISABLE))
6004                         return false;
6005                 new = val | NAPIF_STATE_SCHED;
6006 
6007                 /* Sets STATE_MISSED bit if STATE_SCHED was already set
6008                  * This was suggested by Alexander Duyck, as compiler
6009                  * emits better code than :
6010                  * if (val & NAPIF_STATE_SCHED)
6011                  *     new |= NAPIF_STATE_MISSED;
6012                  */
6013                 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
6014                                                    NAPIF_STATE_MISSED;
6015         } while (cmpxchg(&n->state, val, new) != val);
6016 
6017         return !(val & NAPIF_STATE_SCHED);
6018 }
6019 EXPORT_SYMBOL(napi_schedule_prep);
6020 
6021 /**
6022  * __napi_schedule_irqoff - schedule for receive
6023  * @n: entry to schedule
6024  *
6025  * Variant of __napi_schedule() assuming hard irqs are masked
6026  */
6027 void __napi_schedule_irqoff(struct napi_struct *n)
6028 {
6029         ____napi_schedule(this_cpu_ptr(&softnet_data), n);
6030 }
6031 EXPORT_SYMBOL(__napi_schedule_irqoff);
6032 
6033 bool napi_complete_done(struct napi_struct *n, int work_done)
6034 {
6035         unsigned long flags, val, new;
6036 
6037         /*
6038          * 1) Don't let napi dequeue from the cpu poll list
6039          *    just in case its running on a different cpu.
6040          * 2) If we are busy polling, do nothing here, we have
6041          *    the guarantee we will be called later.
6042          */
6043         if (unlikely(n->state & (NAPIF_STATE_NPSVC |
6044                                  NAPIF_STATE_IN_BUSY_POLL)))
6045                 return false;
6046 
6047         if (n->gro_bitmask) {
6048                 unsigned long timeout = 0;
6049 
6050                 if (work_done)
6051                         timeout = n->dev->gro_flush_timeout;
6052 
6053                 /* When the NAPI instance uses a timeout and keeps postponing
6054                  * it, we need to bound somehow the time packets are kept in
6055                  * the GRO layer
6056                  */
6057                 napi_gro_flush(n, !!timeout);
6058                 if (timeout)
6059                         hrtimer_start(&n->timer, ns_to_ktime(timeout),
6060                                       HRTIMER_MODE_REL_PINNED);
6061         }
6062         if (unlikely(!list_empty(&n->poll_list))) {
6063                 /* If n->poll_list is not empty, we need to mask irqs */
6064                 local_irq_save(flags);
6065                 list_del_init(&n->poll_list);
6066                 local_irq_restore(flags);
6067         }
6068 
6069         do {
6070                 val = READ_ONCE(n->state);
6071 
6072                 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
6073 
6074                 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED);
6075 
6076                 /* If STATE_MISSED was set, leave STATE_SCHED set,
6077                  * because we will call napi->poll() one more time.
6078                  * This C code was suggested by Alexander Duyck to help gcc.
6079                  */
6080                 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
6081                                                     NAPIF_STATE_SCHED;
6082         } while (cmpxchg(&n->state, val, new) != val);
6083 
6084         if (unlikely(val & NAPIF_STATE_MISSED)) {
6085                 __napi_schedule(n);
6086                 return false;
6087         }
6088 
6089         return true;
6090 }
6091 EXPORT_SYMBOL(napi_complete_done);
6092 
6093 /* must be called under rcu_read_lock(), as we dont take a reference */
6094 static struct napi_struct *napi_by_id(unsigned int napi_id)
6095 {
6096         unsigned int hash = napi_id % HASH_SIZE(napi_hash);
6097         struct napi_struct *napi;
6098 
6099         hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
6100                 if (napi->napi_id == napi_id)
6101                         return napi;
6102 
6103         return NULL;
6104 }
6105 
6106 #if defined(CONFIG_NET_RX_BUSY_POLL)
6107 
6108 #define BUSY_POLL_BUDGET 8
6109 
6110 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock)
6111 {
6112         int rc;
6113 
6114         /* Busy polling means there is a high chance device driver hard irq
6115          * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
6116          * set in napi_schedule_prep().
6117          * Since we are about to call napi->poll() once more, we can safely
6118          * clear NAPI_STATE_MISSED.
6119          *
6120          * Note: x86 could use a single "lock and ..." instruction
6121          * to perform these two clear_bit()
6122          */
6123         clear_bit(NAPI_STATE_MISSED, &napi->state);
6124         clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
6125 
6126         local_bh_disable();
6127 
6128         /* All we really want here is to re-enable device interrupts.
6129          * Ideally, a new ndo_busy_poll_stop() could avoid another round.
6130          */
6131         rc = napi->poll(napi, BUSY_POLL_BUDGET);
6132         trace_napi_poll(napi, rc, BUSY_POLL_BUDGET);
6133         netpoll_poll_unlock(have_poll_lock);
6134         if (rc == BUSY_POLL_BUDGET)
6135                 __napi_schedule(napi);
6136         local_bh_enable();
6137 }
6138 
6139 void napi_busy_loop(unsigned int napi_id,
6140                     bool (*loop_end)(void *, unsigned long),
6141                     void *loop_end_arg)
6142 {
6143         unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
6144         int (*napi_poll)(struct napi_struct *napi, int budget);
6145         void *have_poll_lock = NULL;
6146         struct napi_struct *napi;
6147 
6148 restart:
6149         napi_poll = NULL;
6150 
6151         rcu_read_lock();
6152 
6153         napi = napi_by_id(napi_id);
6154         if (!napi)
6155                 goto out;
6156 
6157         preempt_disable();
6158         for (;;) {
6159                 int work = 0;
6160 
6161                 local_bh_disable();
6162                 if (!napi_poll) {
6163                         unsigned long val = READ_ONCE(napi->state);
6164 
6165                         /* If multiple threads are competing for this napi,
6166                          * we avoid dirtying napi->state as much as we can.
6167                          */
6168                         if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
6169                                    NAPIF_STATE_IN_BUSY_POLL))
6170                                 goto count;
6171                         if (cmpxchg(&napi->state, val,
6172                                     val | NAPIF_STATE_IN_BUSY_POLL |
6173                                           NAPIF_STATE_SCHED) != val)
6174                                 goto count;
6175                         have_poll_lock = netpoll_poll_lock(napi);
6176                         napi_poll = napi->poll;
6177                 }
6178                 work = napi_poll(napi, BUSY_POLL_BUDGET);
6179                 trace_napi_poll(napi, work, BUSY_POLL_BUDGET);
6180 count:
6181                 if (work > 0)
6182                         __NET_ADD_STATS(dev_net(napi->dev),
6183                                         LINUX_MIB_BUSYPOLLRXPACKETS, work);
6184                 local_bh_enable();
6185 
6186                 if (!loop_end || loop_end(loop_end_arg, start_time))
6187                         break;
6188 
6189                 if (unlikely(