~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/core/dev.c

Version: ~ [ linux-5.16-rc3 ] ~ [ linux-5.15.5 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.82 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.162 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.218 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.256 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.291 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.293 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *      NET3    Protocol independent device support routines.
  3  *
  4  *              This program is free software; you can redistribute it and/or
  5  *              modify it under the terms of the GNU General Public License
  6  *              as published by the Free Software Foundation; either version
  7  *              2 of the License, or (at your option) any later version.
  8  *
  9  *      Derived from the non IP parts of dev.c 1.0.19
 10  *              Authors:        Ross Biro
 11  *                              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 12  *                              Mark Evans, <evansmp@uhura.aston.ac.uk>
 13  *
 14  *      Additional Authors:
 15  *              Florian la Roche <rzsfl@rz.uni-sb.de>
 16  *              Alan Cox <gw4pts@gw4pts.ampr.org>
 17  *              David Hinds <dahinds@users.sourceforge.net>
 18  *              Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
 19  *              Adam Sulmicki <adam@cfar.umd.edu>
 20  *              Pekka Riikonen <priikone@poesidon.pspt.fi>
 21  *
 22  *      Changes:
 23  *              D.J. Barrow     :       Fixed bug where dev->refcnt gets set
 24  *                                      to 2 if register_netdev gets called
 25  *                                      before net_dev_init & also removed a
 26  *                                      few lines of code in the process.
 27  *              Alan Cox        :       device private ioctl copies fields back.
 28  *              Alan Cox        :       Transmit queue code does relevant
 29  *                                      stunts to keep the queue safe.
 30  *              Alan Cox        :       Fixed double lock.
 31  *              Alan Cox        :       Fixed promisc NULL pointer trap
 32  *              ????????        :       Support the full private ioctl range
 33  *              Alan Cox        :       Moved ioctl permission check into
 34  *                                      drivers
 35  *              Tim Kordas      :       SIOCADDMULTI/SIOCDELMULTI
 36  *              Alan Cox        :       100 backlog just doesn't cut it when
 37  *                                      you start doing multicast video 8)
 38  *              Alan Cox        :       Rewrote net_bh and list manager.
 39  *              Alan Cox        :       Fix ETH_P_ALL echoback lengths.
 40  *              Alan Cox        :       Took out transmit every packet pass
 41  *                                      Saved a few bytes in the ioctl handler
 42  *              Alan Cox        :       Network driver sets packet type before
 43  *                                      calling netif_rx. Saves a function
 44  *                                      call a packet.
 45  *              Alan Cox        :       Hashed net_bh()
 46  *              Richard Kooijman:       Timestamp fixes.
 47  *              Alan Cox        :       Wrong field in SIOCGIFDSTADDR
 48  *              Alan Cox        :       Device lock protection.
 49  *              Alan Cox        :       Fixed nasty side effect of device close
 50  *                                      changes.
 51  *              Rudi Cilibrasi  :       Pass the right thing to
 52  *                                      set_mac_address()
 53  *              Dave Miller     :       32bit quantity for the device lock to
 54  *                                      make it work out on a Sparc.
 55  *              Bjorn Ekwall    :       Added KERNELD hack.
 56  *              Alan Cox        :       Cleaned up the backlog initialise.
 57  *              Craig Metz      :       SIOCGIFCONF fix if space for under
 58  *                                      1 device.
 59  *          Thomas Bogendoerfer :       Return ENODEV for dev_open, if there
 60  *                                      is no device open function.
 61  *              Andi Kleen      :       Fix error reporting for SIOCGIFCONF
 62  *          Michael Chastain    :       Fix signed/unsigned for SIOCGIFCONF
 63  *              Cyrus Durgin    :       Cleaned for KMOD
 64  *              Adam Sulmicki   :       Bug Fix : Network Device Unload
 65  *                                      A network device unload needs to purge
 66  *                                      the backlog queue.
 67  *      Paul Rusty Russell      :       SIOCSIFNAME
 68  *              Pekka Riikonen  :       Netdev boot-time settings code
 69  *              Andrew Morton   :       Make unregister_netdevice wait
 70  *                                      indefinitely on dev->refcnt
 71  *              J Hadi Salim    :       - Backlog queue sampling
 72  *                                      - netif_rx() feedback
 73  */
 74 
 75 #include <linux/uaccess.h>
 76 #include <linux/bitops.h>
 77 #include <linux/capability.h>
 78 #include <linux/cpu.h>
 79 #include <linux/types.h>
 80 #include <linux/kernel.h>
 81 #include <linux/hash.h>
 82 #include <linux/slab.h>
 83 #include <linux/sched.h>
 84 #include <linux/sched/mm.h>
 85 #include <linux/mutex.h>
 86 #include <linux/string.h>
 87 #include <linux/mm.h>
 88 #include <linux/socket.h>
 89 #include <linux/sockios.h>
 90 #include <linux/errno.h>
 91 #include <linux/interrupt.h>
 92 #include <linux/if_ether.h>
 93 #include <linux/netdevice.h>
 94 #include <linux/etherdevice.h>
 95 #include <linux/ethtool.h>
 96 #include <linux/notifier.h>
 97 #include <linux/skbuff.h>
 98 #include <linux/bpf.h>
 99 #include <linux/bpf_trace.h>
100 #include <net/net_namespace.h>
101 #include <net/sock.h>
102 #include <net/busy_poll.h>
103 #include <linux/rtnetlink.h>
104 #include <linux/stat.h>
105 #include <net/dst.h>
106 #include <net/dst_metadata.h>
107 #include <net/pkt_sched.h>
108 #include <net/pkt_cls.h>
109 #include <net/checksum.h>
110 #include <net/xfrm.h>
111 #include <linux/highmem.h>
112 #include <linux/init.h>
113 #include <linux/module.h>
114 #include <linux/netpoll.h>
115 #include <linux/rcupdate.h>
116 #include <linux/delay.h>
117 #include <net/iw_handler.h>
118 #include <asm/current.h>
119 #include <linux/audit.h>
120 #include <linux/dmaengine.h>
121 #include <linux/err.h>
122 #include <linux/ctype.h>
123 #include <linux/if_arp.h>
124 #include <linux/if_vlan.h>
125 #include <linux/ip.h>
126 #include <net/ip.h>
127 #include <net/mpls.h>
128 #include <linux/ipv6.h>
129 #include <linux/in.h>
130 #include <linux/jhash.h>
131 #include <linux/random.h>
132 #include <trace/events/napi.h>
133 #include <trace/events/net.h>
134 #include <trace/events/skb.h>
135 #include <linux/pci.h>
136 #include <linux/inetdevice.h>
137 #include <linux/cpu_rmap.h>
138 #include <linux/static_key.h>
139 #include <linux/hashtable.h>
140 #include <linux/vmalloc.h>
141 #include <linux/if_macvlan.h>
142 #include <linux/errqueue.h>
143 #include <linux/hrtimer.h>
144 #include <linux/netfilter_ingress.h>
145 #include <linux/crash_dump.h>
146 #include <linux/sctp.h>
147 
148 #include "net-sysfs.h"
149 
150 /* Instead of increasing this, you should create a hash table. */
151 #define MAX_GRO_SKBS 8
152 
153 /* This should be increased if a protocol with a bigger head is added. */
154 #define GRO_MAX_HEAD (MAX_HEADER + 128)
155 
156 static DEFINE_SPINLOCK(ptype_lock);
157 static DEFINE_SPINLOCK(offload_lock);
158 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
159 struct list_head ptype_all __read_mostly;       /* Taps */
160 static struct list_head offload_base __read_mostly;
161 
162 static int netif_rx_internal(struct sk_buff *skb);
163 static int call_netdevice_notifiers_info(unsigned long val,
164                                          struct net_device *dev,
165                                          struct netdev_notifier_info *info);
166 static struct napi_struct *napi_by_id(unsigned int napi_id);
167 
168 /*
169  * The @dev_base_head list is protected by @dev_base_lock and the rtnl
170  * semaphore.
171  *
172  * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
173  *
174  * Writers must hold the rtnl semaphore while they loop through the
175  * dev_base_head list, and hold dev_base_lock for writing when they do the
176  * actual updates.  This allows pure readers to access the list even
177  * while a writer is preparing to update it.
178  *
179  * To put it another way, dev_base_lock is held for writing only to
180  * protect against pure readers; the rtnl semaphore provides the
181  * protection against other writers.
182  *
183  * See, for example usages, register_netdevice() and
184  * unregister_netdevice(), which must be called with the rtnl
185  * semaphore held.
186  */
187 DEFINE_RWLOCK(dev_base_lock);
188 EXPORT_SYMBOL(dev_base_lock);
189 
190 /* protects napi_hash addition/deletion and napi_gen_id */
191 static DEFINE_SPINLOCK(napi_hash_lock);
192 
193 static unsigned int napi_gen_id = NR_CPUS;
194 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
195 
196 static seqcount_t devnet_rename_seq;
197 
198 static inline void dev_base_seq_inc(struct net *net)
199 {
200         while (++net->dev_base_seq == 0)
201                 ;
202 }
203 
204 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
205 {
206         unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
207 
208         return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
209 }
210 
211 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
212 {
213         return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
214 }
215 
216 static inline void rps_lock(struct softnet_data *sd)
217 {
218 #ifdef CONFIG_RPS
219         spin_lock(&sd->input_pkt_queue.lock);
220 #endif
221 }
222 
223 static inline void rps_unlock(struct softnet_data *sd)
224 {
225 #ifdef CONFIG_RPS
226         spin_unlock(&sd->input_pkt_queue.lock);
227 #endif
228 }
229 
230 /* Device list insertion */
231 static void list_netdevice(struct net_device *dev)
232 {
233         struct net *net = dev_net(dev);
234 
235         ASSERT_RTNL();
236 
237         write_lock_bh(&dev_base_lock);
238         list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
239         hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
240         hlist_add_head_rcu(&dev->index_hlist,
241                            dev_index_hash(net, dev->ifindex));
242         write_unlock_bh(&dev_base_lock);
243 
244         dev_base_seq_inc(net);
245 }
246 
247 /* Device list removal
248  * caller must respect a RCU grace period before freeing/reusing dev
249  */
250 static void unlist_netdevice(struct net_device *dev)
251 {
252         ASSERT_RTNL();
253 
254         /* Unlink dev from the device chain */
255         write_lock_bh(&dev_base_lock);
256         list_del_rcu(&dev->dev_list);
257         hlist_del_rcu(&dev->name_hlist);
258         hlist_del_rcu(&dev->index_hlist);
259         write_unlock_bh(&dev_base_lock);
260 
261         dev_base_seq_inc(dev_net(dev));
262 }
263 
264 /*
265  *      Our notifier list
266  */
267 
268 static RAW_NOTIFIER_HEAD(netdev_chain);
269 
270 /*
271  *      Device drivers call our routines to queue packets here. We empty the
272  *      queue in the local softnet handler.
273  */
274 
275 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
276 EXPORT_PER_CPU_SYMBOL(softnet_data);
277 
278 #ifdef CONFIG_LOCKDEP
279 /*
280  * register_netdevice() inits txq->_xmit_lock and sets lockdep class
281  * according to dev->type
282  */
283 static const unsigned short netdev_lock_type[] = {
284          ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
285          ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
286          ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
287          ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
288          ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
289          ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
290          ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
291          ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
292          ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
293          ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
294          ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
295          ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
296          ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
297          ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
298          ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
299 
300 static const char *const netdev_lock_name[] = {
301         "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
302         "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
303         "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
304         "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
305         "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
306         "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
307         "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
308         "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
309         "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
310         "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
311         "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
312         "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
313         "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
314         "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
315         "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
316 
317 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
318 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
319 
320 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
321 {
322         int i;
323 
324         for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
325                 if (netdev_lock_type[i] == dev_type)
326                         return i;
327         /* the last key is used by default */
328         return ARRAY_SIZE(netdev_lock_type) - 1;
329 }
330 
331 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
332                                                  unsigned short dev_type)
333 {
334         int i;
335 
336         i = netdev_lock_pos(dev_type);
337         lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
338                                    netdev_lock_name[i]);
339 }
340 
341 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
342 {
343         int i;
344 
345         i = netdev_lock_pos(dev->type);
346         lockdep_set_class_and_name(&dev->addr_list_lock,
347                                    &netdev_addr_lock_key[i],
348                                    netdev_lock_name[i]);
349 }
350 #else
351 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
352                                                  unsigned short dev_type)
353 {
354 }
355 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
356 {
357 }
358 #endif
359 
360 /*******************************************************************************
361  *
362  *              Protocol management and registration routines
363  *
364  *******************************************************************************/
365 
366 
367 /*
368  *      Add a protocol ID to the list. Now that the input handler is
369  *      smarter we can dispense with all the messy stuff that used to be
370  *      here.
371  *
372  *      BEWARE!!! Protocol handlers, mangling input packets,
373  *      MUST BE last in hash buckets and checking protocol handlers
374  *      MUST start from promiscuous ptype_all chain in net_bh.
375  *      It is true now, do not change it.
376  *      Explanation follows: if protocol handler, mangling packet, will
377  *      be the first on list, it is not able to sense, that packet
378  *      is cloned and should be copied-on-write, so that it will
379  *      change it and subsequent readers will get broken packet.
380  *                                                      --ANK (980803)
381  */
382 
383 static inline struct list_head *ptype_head(const struct packet_type *pt)
384 {
385         if (pt->type == htons(ETH_P_ALL))
386                 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
387         else
388                 return pt->dev ? &pt->dev->ptype_specific :
389                                  &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
390 }
391 
392 /**
393  *      dev_add_pack - add packet handler
394  *      @pt: packet type declaration
395  *
396  *      Add a protocol handler to the networking stack. The passed &packet_type
397  *      is linked into kernel lists and may not be freed until it has been
398  *      removed from the kernel lists.
399  *
400  *      This call does not sleep therefore it can not
401  *      guarantee all CPU's that are in middle of receiving packets
402  *      will see the new packet type (until the next received packet).
403  */
404 
405 void dev_add_pack(struct packet_type *pt)
406 {
407         struct list_head *head = ptype_head(pt);
408 
409         spin_lock(&ptype_lock);
410         list_add_rcu(&pt->list, head);
411         spin_unlock(&ptype_lock);
412 }
413 EXPORT_SYMBOL(dev_add_pack);
414 
415 /**
416  *      __dev_remove_pack        - remove packet handler
417  *      @pt: packet type declaration
418  *
419  *      Remove a protocol handler that was previously added to the kernel
420  *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
421  *      from the kernel lists and can be freed or reused once this function
422  *      returns.
423  *
424  *      The packet type might still be in use by receivers
425  *      and must not be freed until after all the CPU's have gone
426  *      through a quiescent state.
427  */
428 void __dev_remove_pack(struct packet_type *pt)
429 {
430         struct list_head *head = ptype_head(pt);
431         struct packet_type *pt1;
432 
433         spin_lock(&ptype_lock);
434 
435         list_for_each_entry(pt1, head, list) {
436                 if (pt == pt1) {
437                         list_del_rcu(&pt->list);
438                         goto out;
439                 }
440         }
441 
442         pr_warn("dev_remove_pack: %p not found\n", pt);
443 out:
444         spin_unlock(&ptype_lock);
445 }
446 EXPORT_SYMBOL(__dev_remove_pack);
447 
448 /**
449  *      dev_remove_pack  - remove packet handler
450  *      @pt: packet type declaration
451  *
452  *      Remove a protocol handler that was previously added to the kernel
453  *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
454  *      from the kernel lists and can be freed or reused once this function
455  *      returns.
456  *
457  *      This call sleeps to guarantee that no CPU is looking at the packet
458  *      type after return.
459  */
460 void dev_remove_pack(struct packet_type *pt)
461 {
462         __dev_remove_pack(pt);
463 
464         synchronize_net();
465 }
466 EXPORT_SYMBOL(dev_remove_pack);
467 
468 
469 /**
470  *      dev_add_offload - register offload handlers
471  *      @po: protocol offload declaration
472  *
473  *      Add protocol offload handlers to the networking stack. The passed
474  *      &proto_offload is linked into kernel lists and may not be freed until
475  *      it has been removed from the kernel lists.
476  *
477  *      This call does not sleep therefore it can not
478  *      guarantee all CPU's that are in middle of receiving packets
479  *      will see the new offload handlers (until the next received packet).
480  */
481 void dev_add_offload(struct packet_offload *po)
482 {
483         struct packet_offload *elem;
484 
485         spin_lock(&offload_lock);
486         list_for_each_entry(elem, &offload_base, list) {
487                 if (po->priority < elem->priority)
488                         break;
489         }
490         list_add_rcu(&po->list, elem->list.prev);
491         spin_unlock(&offload_lock);
492 }
493 EXPORT_SYMBOL(dev_add_offload);
494 
495 /**
496  *      __dev_remove_offload     - remove offload handler
497  *      @po: packet offload declaration
498  *
499  *      Remove a protocol offload handler that was previously added to the
500  *      kernel offload handlers by dev_add_offload(). The passed &offload_type
501  *      is removed from the kernel lists and can be freed or reused once this
502  *      function returns.
503  *
504  *      The packet type might still be in use by receivers
505  *      and must not be freed until after all the CPU's have gone
506  *      through a quiescent state.
507  */
508 static void __dev_remove_offload(struct packet_offload *po)
509 {
510         struct list_head *head = &offload_base;
511         struct packet_offload *po1;
512 
513         spin_lock(&offload_lock);
514 
515         list_for_each_entry(po1, head, list) {
516                 if (po == po1) {
517                         list_del_rcu(&po->list);
518                         goto out;
519                 }
520         }
521 
522         pr_warn("dev_remove_offload: %p not found\n", po);
523 out:
524         spin_unlock(&offload_lock);
525 }
526 
527 /**
528  *      dev_remove_offload       - remove packet offload handler
529  *      @po: packet offload declaration
530  *
531  *      Remove a packet offload handler that was previously added to the kernel
532  *      offload handlers by dev_add_offload(). The passed &offload_type is
533  *      removed from the kernel lists and can be freed or reused once this
534  *      function returns.
535  *
536  *      This call sleeps to guarantee that no CPU is looking at the packet
537  *      type after return.
538  */
539 void dev_remove_offload(struct packet_offload *po)
540 {
541         __dev_remove_offload(po);
542 
543         synchronize_net();
544 }
545 EXPORT_SYMBOL(dev_remove_offload);
546 
547 /******************************************************************************
548  *
549  *                    Device Boot-time Settings Routines
550  *
551  ******************************************************************************/
552 
553 /* Boot time configuration table */
554 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
555 
556 /**
557  *      netdev_boot_setup_add   - add new setup entry
558  *      @name: name of the device
559  *      @map: configured settings for the device
560  *
561  *      Adds new setup entry to the dev_boot_setup list.  The function
562  *      returns 0 on error and 1 on success.  This is a generic routine to
563  *      all netdevices.
564  */
565 static int netdev_boot_setup_add(char *name, struct ifmap *map)
566 {
567         struct netdev_boot_setup *s;
568         int i;
569 
570         s = dev_boot_setup;
571         for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
572                 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
573                         memset(s[i].name, 0, sizeof(s[i].name));
574                         strlcpy(s[i].name, name, IFNAMSIZ);
575                         memcpy(&s[i].map, map, sizeof(s[i].map));
576                         break;
577                 }
578         }
579 
580         return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
581 }
582 
583 /**
584  * netdev_boot_setup_check      - check boot time settings
585  * @dev: the netdevice
586  *
587  * Check boot time settings for the device.
588  * The found settings are set for the device to be used
589  * later in the device probing.
590  * Returns 0 if no settings found, 1 if they are.
591  */
592 int netdev_boot_setup_check(struct net_device *dev)
593 {
594         struct netdev_boot_setup *s = dev_boot_setup;
595         int i;
596 
597         for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
598                 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
599                     !strcmp(dev->name, s[i].name)) {
600                         dev->irq = s[i].map.irq;
601                         dev->base_addr = s[i].map.base_addr;
602                         dev->mem_start = s[i].map.mem_start;
603                         dev->mem_end = s[i].map.mem_end;
604                         return 1;
605                 }
606         }
607         return 0;
608 }
609 EXPORT_SYMBOL(netdev_boot_setup_check);
610 
611 
612 /**
613  * netdev_boot_base     - get address from boot time settings
614  * @prefix: prefix for network device
615  * @unit: id for network device
616  *
617  * Check boot time settings for the base address of device.
618  * The found settings are set for the device to be used
619  * later in the device probing.
620  * Returns 0 if no settings found.
621  */
622 unsigned long netdev_boot_base(const char *prefix, int unit)
623 {
624         const struct netdev_boot_setup *s = dev_boot_setup;
625         char name[IFNAMSIZ];
626         int i;
627 
628         sprintf(name, "%s%d", prefix, unit);
629 
630         /*
631          * If device already registered then return base of 1
632          * to indicate not to probe for this interface
633          */
634         if (__dev_get_by_name(&init_net, name))
635                 return 1;
636 
637         for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
638                 if (!strcmp(name, s[i].name))
639                         return s[i].map.base_addr;
640         return 0;
641 }
642 
643 /*
644  * Saves at boot time configured settings for any netdevice.
645  */
646 int __init netdev_boot_setup(char *str)
647 {
648         int ints[5];
649         struct ifmap map;
650 
651         str = get_options(str, ARRAY_SIZE(ints), ints);
652         if (!str || !*str)
653                 return 0;
654 
655         /* Save settings */
656         memset(&map, 0, sizeof(map));
657         if (ints[0] > 0)
658                 map.irq = ints[1];
659         if (ints[0] > 1)
660                 map.base_addr = ints[2];
661         if (ints[0] > 2)
662                 map.mem_start = ints[3];
663         if (ints[0] > 3)
664                 map.mem_end = ints[4];
665 
666         /* Add new entry to the list */
667         return netdev_boot_setup_add(str, &map);
668 }
669 
670 __setup("netdev=", netdev_boot_setup);
671 
672 /*******************************************************************************
673  *
674  *                          Device Interface Subroutines
675  *
676  *******************************************************************************/
677 
678 /**
679  *      dev_get_iflink  - get 'iflink' value of a interface
680  *      @dev: targeted interface
681  *
682  *      Indicates the ifindex the interface is linked to.
683  *      Physical interfaces have the same 'ifindex' and 'iflink' values.
684  */
685 
686 int dev_get_iflink(const struct net_device *dev)
687 {
688         if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
689                 return dev->netdev_ops->ndo_get_iflink(dev);
690 
691         return dev->ifindex;
692 }
693 EXPORT_SYMBOL(dev_get_iflink);
694 
695 /**
696  *      dev_fill_metadata_dst - Retrieve tunnel egress information.
697  *      @dev: targeted interface
698  *      @skb: The packet.
699  *
700  *      For better visibility of tunnel traffic OVS needs to retrieve
701  *      egress tunnel information for a packet. Following API allows
702  *      user to get this info.
703  */
704 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
705 {
706         struct ip_tunnel_info *info;
707 
708         if (!dev->netdev_ops  || !dev->netdev_ops->ndo_fill_metadata_dst)
709                 return -EINVAL;
710 
711         info = skb_tunnel_info_unclone(skb);
712         if (!info)
713                 return -ENOMEM;
714         if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
715                 return -EINVAL;
716 
717         return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
718 }
719 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
720 
721 /**
722  *      __dev_get_by_name       - find a device by its name
723  *      @net: the applicable net namespace
724  *      @name: name to find
725  *
726  *      Find an interface by name. Must be called under RTNL semaphore
727  *      or @dev_base_lock. If the name is found a pointer to the device
728  *      is returned. If the name is not found then %NULL is returned. The
729  *      reference counters are not incremented so the caller must be
730  *      careful with locks.
731  */
732 
733 struct net_device *__dev_get_by_name(struct net *net, const char *name)
734 {
735         struct net_device *dev;
736         struct hlist_head *head = dev_name_hash(net, name);
737 
738         hlist_for_each_entry(dev, head, name_hlist)
739                 if (!strncmp(dev->name, name, IFNAMSIZ))
740                         return dev;
741 
742         return NULL;
743 }
744 EXPORT_SYMBOL(__dev_get_by_name);
745 
746 /**
747  * dev_get_by_name_rcu  - find a device by its name
748  * @net: the applicable net namespace
749  * @name: name to find
750  *
751  * Find an interface by name.
752  * If the name is found a pointer to the device is returned.
753  * If the name is not found then %NULL is returned.
754  * The reference counters are not incremented so the caller must be
755  * careful with locks. The caller must hold RCU lock.
756  */
757 
758 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
759 {
760         struct net_device *dev;
761         struct hlist_head *head = dev_name_hash(net, name);
762 
763         hlist_for_each_entry_rcu(dev, head, name_hlist)
764                 if (!strncmp(dev->name, name, IFNAMSIZ))
765                         return dev;
766 
767         return NULL;
768 }
769 EXPORT_SYMBOL(dev_get_by_name_rcu);
770 
771 /**
772  *      dev_get_by_name         - find a device by its name
773  *      @net: the applicable net namespace
774  *      @name: name to find
775  *
776  *      Find an interface by name. This can be called from any
777  *      context and does its own locking. The returned handle has
778  *      the usage count incremented and the caller must use dev_put() to
779  *      release it when it is no longer needed. %NULL is returned if no
780  *      matching device is found.
781  */
782 
783 struct net_device *dev_get_by_name(struct net *net, const char *name)
784 {
785         struct net_device *dev;
786 
787         rcu_read_lock();
788         dev = dev_get_by_name_rcu(net, name);
789         if (dev)
790                 dev_hold(dev);
791         rcu_read_unlock();
792         return dev;
793 }
794 EXPORT_SYMBOL(dev_get_by_name);
795 
796 /**
797  *      __dev_get_by_index - find a device by its ifindex
798  *      @net: the applicable net namespace
799  *      @ifindex: index of device
800  *
801  *      Search for an interface by index. Returns %NULL if the device
802  *      is not found or a pointer to the device. The device has not
803  *      had its reference counter increased so the caller must be careful
804  *      about locking. The caller must hold either the RTNL semaphore
805  *      or @dev_base_lock.
806  */
807 
808 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
809 {
810         struct net_device *dev;
811         struct hlist_head *head = dev_index_hash(net, ifindex);
812 
813         hlist_for_each_entry(dev, head, index_hlist)
814                 if (dev->ifindex == ifindex)
815                         return dev;
816 
817         return NULL;
818 }
819 EXPORT_SYMBOL(__dev_get_by_index);
820 
821 /**
822  *      dev_get_by_index_rcu - find a device by its ifindex
823  *      @net: the applicable net namespace
824  *      @ifindex: index of device
825  *
826  *      Search for an interface by index. Returns %NULL if the device
827  *      is not found or a pointer to the device. The device has not
828  *      had its reference counter increased so the caller must be careful
829  *      about locking. The caller must hold RCU lock.
830  */
831 
832 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
833 {
834         struct net_device *dev;
835         struct hlist_head *head = dev_index_hash(net, ifindex);
836 
837         hlist_for_each_entry_rcu(dev, head, index_hlist)
838                 if (dev->ifindex == ifindex)
839                         return dev;
840 
841         return NULL;
842 }
843 EXPORT_SYMBOL(dev_get_by_index_rcu);
844 
845 
846 /**
847  *      dev_get_by_index - find a device by its ifindex
848  *      @net: the applicable net namespace
849  *      @ifindex: index of device
850  *
851  *      Search for an interface by index. Returns NULL if the device
852  *      is not found or a pointer to the device. The device returned has
853  *      had a reference added and the pointer is safe until the user calls
854  *      dev_put to indicate they have finished with it.
855  */
856 
857 struct net_device *dev_get_by_index(struct net *net, int ifindex)
858 {
859         struct net_device *dev;
860 
861         rcu_read_lock();
862         dev = dev_get_by_index_rcu(net, ifindex);
863         if (dev)
864                 dev_hold(dev);
865         rcu_read_unlock();
866         return dev;
867 }
868 EXPORT_SYMBOL(dev_get_by_index);
869 
870 /**
871  *      dev_get_by_napi_id - find a device by napi_id
872  *      @napi_id: ID of the NAPI struct
873  *
874  *      Search for an interface by NAPI ID. Returns %NULL if the device
875  *      is not found or a pointer to the device. The device has not had
876  *      its reference counter increased so the caller must be careful
877  *      about locking. The caller must hold RCU lock.
878  */
879 
880 struct net_device *dev_get_by_napi_id(unsigned int napi_id)
881 {
882         struct napi_struct *napi;
883 
884         WARN_ON_ONCE(!rcu_read_lock_held());
885 
886         if (napi_id < MIN_NAPI_ID)
887                 return NULL;
888 
889         napi = napi_by_id(napi_id);
890 
891         return napi ? napi->dev : NULL;
892 }
893 EXPORT_SYMBOL(dev_get_by_napi_id);
894 
895 /**
896  *      netdev_get_name - get a netdevice name, knowing its ifindex.
897  *      @net: network namespace
898  *      @name: a pointer to the buffer where the name will be stored.
899  *      @ifindex: the ifindex of the interface to get the name from.
900  *
901  *      The use of raw_seqcount_begin() and cond_resched() before
902  *      retrying is required as we want to give the writers a chance
903  *      to complete when CONFIG_PREEMPT is not set.
904  */
905 int netdev_get_name(struct net *net, char *name, int ifindex)
906 {
907         struct net_device *dev;
908         unsigned int seq;
909 
910 retry:
911         seq = raw_seqcount_begin(&devnet_rename_seq);
912         rcu_read_lock();
913         dev = dev_get_by_index_rcu(net, ifindex);
914         if (!dev) {
915                 rcu_read_unlock();
916                 return -ENODEV;
917         }
918 
919         strcpy(name, dev->name);
920         rcu_read_unlock();
921         if (read_seqcount_retry(&devnet_rename_seq, seq)) {
922                 cond_resched();
923                 goto retry;
924         }
925 
926         return 0;
927 }
928 
929 /**
930  *      dev_getbyhwaddr_rcu - find a device by its hardware address
931  *      @net: the applicable net namespace
932  *      @type: media type of device
933  *      @ha: hardware address
934  *
935  *      Search for an interface by MAC address. Returns NULL if the device
936  *      is not found or a pointer to the device.
937  *      The caller must hold RCU or RTNL.
938  *      The returned device has not had its ref count increased
939  *      and the caller must therefore be careful about locking
940  *
941  */
942 
943 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
944                                        const char *ha)
945 {
946         struct net_device *dev;
947 
948         for_each_netdev_rcu(net, dev)
949                 if (dev->type == type &&
950                     !memcmp(dev->dev_addr, ha, dev->addr_len))
951                         return dev;
952 
953         return NULL;
954 }
955 EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
956 
957 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
958 {
959         struct net_device *dev;
960 
961         ASSERT_RTNL();
962         for_each_netdev(net, dev)
963                 if (dev->type == type)
964                         return dev;
965 
966         return NULL;
967 }
968 EXPORT_SYMBOL(__dev_getfirstbyhwtype);
969 
970 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
971 {
972         struct net_device *dev, *ret = NULL;
973 
974         rcu_read_lock();
975         for_each_netdev_rcu(net, dev)
976                 if (dev->type == type) {
977                         dev_hold(dev);
978                         ret = dev;
979                         break;
980                 }
981         rcu_read_unlock();
982         return ret;
983 }
984 EXPORT_SYMBOL(dev_getfirstbyhwtype);
985 
986 /**
987  *      __dev_get_by_flags - find any device with given flags
988  *      @net: the applicable net namespace
989  *      @if_flags: IFF_* values
990  *      @mask: bitmask of bits in if_flags to check
991  *
992  *      Search for any interface with the given flags. Returns NULL if a device
993  *      is not found or a pointer to the device. Must be called inside
994  *      rtnl_lock(), and result refcount is unchanged.
995  */
996 
997 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
998                                       unsigned short mask)
999 {
1000         struct net_device *dev, *ret;
1001 
1002         ASSERT_RTNL();
1003 
1004         ret = NULL;
1005         for_each_netdev(net, dev) {
1006                 if (((dev->flags ^ if_flags) & mask) == 0) {
1007                         ret = dev;
1008                         break;
1009                 }
1010         }
1011         return ret;
1012 }
1013 EXPORT_SYMBOL(__dev_get_by_flags);
1014 
1015 /**
1016  *      dev_valid_name - check if name is okay for network device
1017  *      @name: name string
1018  *
1019  *      Network device names need to be valid file names to
1020  *      to allow sysfs to work.  We also disallow any kind of
1021  *      whitespace.
1022  */
1023 bool dev_valid_name(const char *name)
1024 {
1025         if (*name == '\0')
1026                 return false;
1027         if (strlen(name) >= IFNAMSIZ)
1028                 return false;
1029         if (!strcmp(name, ".") || !strcmp(name, ".."))
1030                 return false;
1031 
1032         while (*name) {
1033                 if (*name == '/' || *name == ':' || isspace(*name))
1034                         return false;
1035                 name++;
1036         }
1037         return true;
1038 }
1039 EXPORT_SYMBOL(dev_valid_name);
1040 
1041 /**
1042  *      __dev_alloc_name - allocate a name for a device
1043  *      @net: network namespace to allocate the device name in
1044  *      @name: name format string
1045  *      @buf:  scratch buffer and result name string
1046  *
1047  *      Passed a format string - eg "lt%d" it will try and find a suitable
1048  *      id. It scans list of devices to build up a free map, then chooses
1049  *      the first empty slot. The caller must hold the dev_base or rtnl lock
1050  *      while allocating the name and adding the device in order to avoid
1051  *      duplicates.
1052  *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1053  *      Returns the number of the unit assigned or a negative errno code.
1054  */
1055 
1056 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1057 {
1058         int i = 0;
1059         const char *p;
1060         const int max_netdevices = 8*PAGE_SIZE;
1061         unsigned long *inuse;
1062         struct net_device *d;
1063 
1064         p = strnchr(name, IFNAMSIZ-1, '%');
1065         if (p) {
1066                 /*
1067                  * Verify the string as this thing may have come from
1068                  * the user.  There must be either one "%d" and no other "%"
1069                  * characters.
1070                  */
1071                 if (p[1] != 'd' || strchr(p + 2, '%'))
1072                         return -EINVAL;
1073 
1074                 /* Use one page as a bit array of possible slots */
1075                 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1076                 if (!inuse)
1077                         return -ENOMEM;
1078 
1079                 for_each_netdev(net, d) {
1080                         if (!sscanf(d->name, name, &i))
1081                                 continue;
1082                         if (i < 0 || i >= max_netdevices)
1083                                 continue;
1084 
1085                         /*  avoid cases where sscanf is not exact inverse of printf */
1086                         snprintf(buf, IFNAMSIZ, name, i);
1087                         if (!strncmp(buf, d->name, IFNAMSIZ))
1088                                 set_bit(i, inuse);
1089                 }
1090 
1091                 i = find_first_zero_bit(inuse, max_netdevices);
1092                 free_page((unsigned long) inuse);
1093         }
1094 
1095         if (buf != name)
1096                 snprintf(buf, IFNAMSIZ, name, i);
1097         if (!__dev_get_by_name(net, buf))
1098                 return i;
1099 
1100         /* It is possible to run out of possible slots
1101          * when the name is long and there isn't enough space left
1102          * for the digits, or if all bits are used.
1103          */
1104         return -ENFILE;
1105 }
1106 
1107 /**
1108  *      dev_alloc_name - allocate a name for a device
1109  *      @dev: device
1110  *      @name: name format string
1111  *
1112  *      Passed a format string - eg "lt%d" it will try and find a suitable
1113  *      id. It scans list of devices to build up a free map, then chooses
1114  *      the first empty slot. The caller must hold the dev_base or rtnl lock
1115  *      while allocating the name and adding the device in order to avoid
1116  *      duplicates.
1117  *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1118  *      Returns the number of the unit assigned or a negative errno code.
1119  */
1120 
1121 int dev_alloc_name(struct net_device *dev, const char *name)
1122 {
1123         char buf[IFNAMSIZ];
1124         struct net *net;
1125         int ret;
1126 
1127         BUG_ON(!dev_net(dev));
1128         net = dev_net(dev);
1129         ret = __dev_alloc_name(net, name, buf);
1130         if (ret >= 0)
1131                 strlcpy(dev->name, buf, IFNAMSIZ);
1132         return ret;
1133 }
1134 EXPORT_SYMBOL(dev_alloc_name);
1135 
1136 static int dev_alloc_name_ns(struct net *net,
1137                              struct net_device *dev,
1138                              const char *name)
1139 {
1140         char buf[IFNAMSIZ];
1141         int ret;
1142 
1143         ret = __dev_alloc_name(net, name, buf);
1144         if (ret >= 0)
1145                 strlcpy(dev->name, buf, IFNAMSIZ);
1146         return ret;
1147 }
1148 
1149 int dev_get_valid_name(struct net *net, struct net_device *dev,
1150                        const char *name)
1151 {
1152         BUG_ON(!net);
1153 
1154         if (!dev_valid_name(name))
1155                 return -EINVAL;
1156 
1157         if (strchr(name, '%'))
1158                 return dev_alloc_name_ns(net, dev, name);
1159         else if (__dev_get_by_name(net, name))
1160                 return -EEXIST;
1161         else if (dev->name != name)
1162                 strlcpy(dev->name, name, IFNAMSIZ);
1163 
1164         return 0;
1165 }
1166 EXPORT_SYMBOL(dev_get_valid_name);
1167 
1168 /**
1169  *      dev_change_name - change name of a device
1170  *      @dev: device
1171  *      @newname: name (or format string) must be at least IFNAMSIZ
1172  *
1173  *      Change name of a device, can pass format strings "eth%d".
1174  *      for wildcarding.
1175  */
1176 int dev_change_name(struct net_device *dev, const char *newname)
1177 {
1178         unsigned char old_assign_type;
1179         char oldname[IFNAMSIZ];
1180         int err = 0;
1181         int ret;
1182         struct net *net;
1183 
1184         ASSERT_RTNL();
1185         BUG_ON(!dev_net(dev));
1186 
1187         net = dev_net(dev);
1188         if (dev->flags & IFF_UP)
1189                 return -EBUSY;
1190 
1191         write_seqcount_begin(&devnet_rename_seq);
1192 
1193         if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1194                 write_seqcount_end(&devnet_rename_seq);
1195                 return 0;
1196         }
1197 
1198         memcpy(oldname, dev->name, IFNAMSIZ);
1199 
1200         err = dev_get_valid_name(net, dev, newname);
1201         if (err < 0) {
1202                 write_seqcount_end(&devnet_rename_seq);
1203                 return err;
1204         }
1205 
1206         if (oldname[0] && !strchr(oldname, '%'))
1207                 netdev_info(dev, "renamed from %s\n", oldname);
1208 
1209         old_assign_type = dev->name_assign_type;
1210         dev->name_assign_type = NET_NAME_RENAMED;
1211 
1212 rollback:
1213         ret = device_rename(&dev->dev, dev->name);
1214         if (ret) {
1215                 memcpy(dev->name, oldname, IFNAMSIZ);
1216                 dev->name_assign_type = old_assign_type;
1217                 write_seqcount_end(&devnet_rename_seq);
1218                 return ret;
1219         }
1220 
1221         write_seqcount_end(&devnet_rename_seq);
1222 
1223         netdev_adjacent_rename_links(dev, oldname);
1224 
1225         write_lock_bh(&dev_base_lock);
1226         hlist_del_rcu(&dev->name_hlist);
1227         write_unlock_bh(&dev_base_lock);
1228 
1229         synchronize_rcu();
1230 
1231         write_lock_bh(&dev_base_lock);
1232         hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1233         write_unlock_bh(&dev_base_lock);
1234 
1235         ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1236         ret = notifier_to_errno(ret);
1237 
1238         if (ret) {
1239                 /* err >= 0 after dev_alloc_name() or stores the first errno */
1240                 if (err >= 0) {
1241                         err = ret;
1242                         write_seqcount_begin(&devnet_rename_seq);
1243                         memcpy(dev->name, oldname, IFNAMSIZ);
1244                         memcpy(oldname, newname, IFNAMSIZ);
1245                         dev->name_assign_type = old_assign_type;
1246                         old_assign_type = NET_NAME_RENAMED;
1247                         goto rollback;
1248                 } else {
1249                         pr_err("%s: name change rollback failed: %d\n",
1250                                dev->name, ret);
1251                 }
1252         }
1253 
1254         return err;
1255 }
1256 
1257 /**
1258  *      dev_set_alias - change ifalias of a device
1259  *      @dev: device
1260  *      @alias: name up to IFALIASZ
1261  *      @len: limit of bytes to copy from info
1262  *
1263  *      Set ifalias for a device,
1264  */
1265 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1266 {
1267         char *new_ifalias;
1268 
1269         ASSERT_RTNL();
1270 
1271         if (len >= IFALIASZ)
1272                 return -EINVAL;
1273 
1274         if (!len) {
1275                 kfree(dev->ifalias);
1276                 dev->ifalias = NULL;
1277                 return 0;
1278         }
1279 
1280         new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1281         if (!new_ifalias)
1282                 return -ENOMEM;
1283         dev->ifalias = new_ifalias;
1284         memcpy(dev->ifalias, alias, len);
1285         dev->ifalias[len] = 0;
1286 
1287         return len;
1288 }
1289 
1290 
1291 /**
1292  *      netdev_features_change - device changes features
1293  *      @dev: device to cause notification
1294  *
1295  *      Called to indicate a device has changed features.
1296  */
1297 void netdev_features_change(struct net_device *dev)
1298 {
1299         call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1300 }
1301 EXPORT_SYMBOL(netdev_features_change);
1302 
1303 /**
1304  *      netdev_state_change - device changes state
1305  *      @dev: device to cause notification
1306  *
1307  *      Called to indicate a device has changed state. This function calls
1308  *      the notifier chains for netdev_chain and sends a NEWLINK message
1309  *      to the routing socket.
1310  */
1311 void netdev_state_change(struct net_device *dev)
1312 {
1313         if (dev->flags & IFF_UP) {
1314                 struct netdev_notifier_change_info change_info;
1315 
1316                 change_info.flags_changed = 0;
1317                 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
1318                                               &change_info.info);
1319                 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1320         }
1321 }
1322 EXPORT_SYMBOL(netdev_state_change);
1323 
1324 /**
1325  * netdev_notify_peers - notify network peers about existence of @dev
1326  * @dev: network device
1327  *
1328  * Generate traffic such that interested network peers are aware of
1329  * @dev, such as by generating a gratuitous ARP. This may be used when
1330  * a device wants to inform the rest of the network about some sort of
1331  * reconfiguration such as a failover event or virtual machine
1332  * migration.
1333  */
1334 void netdev_notify_peers(struct net_device *dev)
1335 {
1336         rtnl_lock();
1337         call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1338         call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1339         rtnl_unlock();
1340 }
1341 EXPORT_SYMBOL(netdev_notify_peers);
1342 
1343 static int __dev_open(struct net_device *dev)
1344 {
1345         const struct net_device_ops *ops = dev->netdev_ops;
1346         int ret;
1347 
1348         ASSERT_RTNL();
1349 
1350         if (!netif_device_present(dev))
1351                 return -ENODEV;
1352 
1353         /* Block netpoll from trying to do any rx path servicing.
1354          * If we don't do this there is a chance ndo_poll_controller
1355          * or ndo_poll may be running while we open the device
1356          */
1357         netpoll_poll_disable(dev);
1358 
1359         ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1360         ret = notifier_to_errno(ret);
1361         if (ret)
1362                 return ret;
1363 
1364         set_bit(__LINK_STATE_START, &dev->state);
1365 
1366         if (ops->ndo_validate_addr)
1367                 ret = ops->ndo_validate_addr(dev);
1368 
1369         if (!ret && ops->ndo_open)
1370                 ret = ops->ndo_open(dev);
1371 
1372         netpoll_poll_enable(dev);
1373 
1374         if (ret)
1375                 clear_bit(__LINK_STATE_START, &dev->state);
1376         else {
1377                 dev->flags |= IFF_UP;
1378                 dev_set_rx_mode(dev);
1379                 dev_activate(dev);
1380                 add_device_randomness(dev->dev_addr, dev->addr_len);
1381         }
1382 
1383         return ret;
1384 }
1385 
1386 /**
1387  *      dev_open        - prepare an interface for use.
1388  *      @dev:   device to open
1389  *
1390  *      Takes a device from down to up state. The device's private open
1391  *      function is invoked and then the multicast lists are loaded. Finally
1392  *      the device is moved into the up state and a %NETDEV_UP message is
1393  *      sent to the netdev notifier chain.
1394  *
1395  *      Calling this function on an active interface is a nop. On a failure
1396  *      a negative errno code is returned.
1397  */
1398 int dev_open(struct net_device *dev)
1399 {
1400         int ret;
1401 
1402         if (dev->flags & IFF_UP)
1403                 return 0;
1404 
1405         ret = __dev_open(dev);
1406         if (ret < 0)
1407                 return ret;
1408 
1409         rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1410         call_netdevice_notifiers(NETDEV_UP, dev);
1411 
1412         return ret;
1413 }
1414 EXPORT_SYMBOL(dev_open);
1415 
1416 static int __dev_close_many(struct list_head *head)
1417 {
1418         struct net_device *dev;
1419 
1420         ASSERT_RTNL();
1421         might_sleep();
1422 
1423         list_for_each_entry(dev, head, close_list) {
1424                 /* Temporarily disable netpoll until the interface is down */
1425                 netpoll_poll_disable(dev);
1426 
1427                 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1428 
1429                 clear_bit(__LINK_STATE_START, &dev->state);
1430 
1431                 /* Synchronize to scheduled poll. We cannot touch poll list, it
1432                  * can be even on different cpu. So just clear netif_running().
1433                  *
1434                  * dev->stop() will invoke napi_disable() on all of it's
1435                  * napi_struct instances on this device.
1436                  */
1437                 smp_mb__after_atomic(); /* Commit netif_running(). */
1438         }
1439 
1440         dev_deactivate_many(head);
1441 
1442         list_for_each_entry(dev, head, close_list) {
1443                 const struct net_device_ops *ops = dev->netdev_ops;
1444 
1445                 /*
1446                  *      Call the device specific close. This cannot fail.
1447                  *      Only if device is UP
1448                  *
1449                  *      We allow it to be called even after a DETACH hot-plug
1450                  *      event.
1451                  */
1452                 if (ops->ndo_stop)
1453                         ops->ndo_stop(dev);
1454 
1455                 dev->flags &= ~IFF_UP;
1456                 netpoll_poll_enable(dev);
1457         }
1458 
1459         return 0;
1460 }
1461 
1462 static int __dev_close(struct net_device *dev)
1463 {
1464         int retval;
1465         LIST_HEAD(single);
1466 
1467         list_add(&dev->close_list, &single);
1468         retval = __dev_close_many(&single);
1469         list_del(&single);
1470 
1471         return retval;
1472 }
1473 
1474 int dev_close_many(struct list_head *head, bool unlink)
1475 {
1476         struct net_device *dev, *tmp;
1477 
1478         /* Remove the devices that don't need to be closed */
1479         list_for_each_entry_safe(dev, tmp, head, close_list)
1480                 if (!(dev->flags & IFF_UP))
1481                         list_del_init(&dev->close_list);
1482 
1483         __dev_close_many(head);
1484 
1485         list_for_each_entry_safe(dev, tmp, head, close_list) {
1486                 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1487                 call_netdevice_notifiers(NETDEV_DOWN, dev);
1488                 if (unlink)
1489                         list_del_init(&dev->close_list);
1490         }
1491 
1492         return 0;
1493 }
1494 EXPORT_SYMBOL(dev_close_many);
1495 
1496 /**
1497  *      dev_close - shutdown an interface.
1498  *      @dev: device to shutdown
1499  *
1500  *      This function moves an active device into down state. A
1501  *      %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1502  *      is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1503  *      chain.
1504  */
1505 int dev_close(struct net_device *dev)
1506 {
1507         if (dev->flags & IFF_UP) {
1508                 LIST_HEAD(single);
1509 
1510                 list_add(&dev->close_list, &single);
1511                 dev_close_many(&single, true);
1512                 list_del(&single);
1513         }
1514         return 0;
1515 }
1516 EXPORT_SYMBOL(dev_close);
1517 
1518 
1519 /**
1520  *      dev_disable_lro - disable Large Receive Offload on a device
1521  *      @dev: device
1522  *
1523  *      Disable Large Receive Offload (LRO) on a net device.  Must be
1524  *      called under RTNL.  This is needed if received packets may be
1525  *      forwarded to another interface.
1526  */
1527 void dev_disable_lro(struct net_device *dev)
1528 {
1529         struct net_device *lower_dev;
1530         struct list_head *iter;
1531 
1532         dev->wanted_features &= ~NETIF_F_LRO;
1533         netdev_update_features(dev);
1534 
1535         if (unlikely(dev->features & NETIF_F_LRO))
1536                 netdev_WARN(dev, "failed to disable LRO!\n");
1537 
1538         netdev_for_each_lower_dev(dev, lower_dev, iter)
1539                 dev_disable_lro(lower_dev);
1540 }
1541 EXPORT_SYMBOL(dev_disable_lro);
1542 
1543 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1544                                    struct net_device *dev)
1545 {
1546         struct netdev_notifier_info info;
1547 
1548         netdev_notifier_info_init(&info, dev);
1549         return nb->notifier_call(nb, val, &info);
1550 }
1551 
1552 static int dev_boot_phase = 1;
1553 
1554 /**
1555  * register_netdevice_notifier - register a network notifier block
1556  * @nb: notifier
1557  *
1558  * Register a notifier to be called when network device events occur.
1559  * The notifier passed is linked into the kernel structures and must
1560  * not be reused until it has been unregistered. A negative errno code
1561  * is returned on a failure.
1562  *
1563  * When registered all registration and up events are replayed
1564  * to the new notifier to allow device to have a race free
1565  * view of the network device list.
1566  */
1567 
1568 int register_netdevice_notifier(struct notifier_block *nb)
1569 {
1570         struct net_device *dev;
1571         struct net_device *last;
1572         struct net *net;
1573         int err;
1574 
1575         rtnl_lock();
1576         err = raw_notifier_chain_register(&netdev_chain, nb);
1577         if (err)
1578                 goto unlock;
1579         if (dev_boot_phase)
1580                 goto unlock;
1581         for_each_net(net) {
1582                 for_each_netdev(net, dev) {
1583                         err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1584                         err = notifier_to_errno(err);
1585                         if (err)
1586                                 goto rollback;
1587 
1588                         if (!(dev->flags & IFF_UP))
1589                                 continue;
1590 
1591                         call_netdevice_notifier(nb, NETDEV_UP, dev);
1592                 }
1593         }
1594 
1595 unlock:
1596         rtnl_unlock();
1597         return err;
1598 
1599 rollback:
1600         last = dev;
1601         for_each_net(net) {
1602                 for_each_netdev(net, dev) {
1603                         if (dev == last)
1604                                 goto outroll;
1605 
1606                         if (dev->flags & IFF_UP) {
1607                                 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1608                                                         dev);
1609                                 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1610                         }
1611                         call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1612                 }
1613         }
1614 
1615 outroll:
1616         raw_notifier_chain_unregister(&netdev_chain, nb);
1617         goto unlock;
1618 }
1619 EXPORT_SYMBOL(register_netdevice_notifier);
1620 
1621 /**
1622  * unregister_netdevice_notifier - unregister a network notifier block
1623  * @nb: notifier
1624  *
1625  * Unregister a notifier previously registered by
1626  * register_netdevice_notifier(). The notifier is unlinked into the
1627  * kernel structures and may then be reused. A negative errno code
1628  * is returned on a failure.
1629  *
1630  * After unregistering unregister and down device events are synthesized
1631  * for all devices on the device list to the removed notifier to remove
1632  * the need for special case cleanup code.
1633  */
1634 
1635 int unregister_netdevice_notifier(struct notifier_block *nb)
1636 {
1637         struct net_device *dev;
1638         struct net *net;
1639         int err;
1640 
1641         rtnl_lock();
1642         err = raw_notifier_chain_unregister(&netdev_chain, nb);
1643         if (err)
1644                 goto unlock;
1645 
1646         for_each_net(net) {
1647                 for_each_netdev(net, dev) {
1648                         if (dev->flags & IFF_UP) {
1649                                 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1650                                                         dev);
1651                                 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1652                         }
1653                         call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1654                 }
1655         }
1656 unlock:
1657         rtnl_unlock();
1658         return err;
1659 }
1660 EXPORT_SYMBOL(unregister_netdevice_notifier);
1661 
1662 /**
1663  *      call_netdevice_notifiers_info - call all network notifier blocks
1664  *      @val: value passed unmodified to notifier function
1665  *      @dev: net_device pointer passed unmodified to notifier function
1666  *      @info: notifier information data
1667  *
1668  *      Call all network notifier blocks.  Parameters and return value
1669  *      are as for raw_notifier_call_chain().
1670  */
1671 
1672 static int call_netdevice_notifiers_info(unsigned long val,
1673                                          struct net_device *dev,
1674                                          struct netdev_notifier_info *info)
1675 {
1676         ASSERT_RTNL();
1677         netdev_notifier_info_init(info, dev);
1678         return raw_notifier_call_chain(&netdev_chain, val, info);
1679 }
1680 
1681 /**
1682  *      call_netdevice_notifiers - call all network notifier blocks
1683  *      @val: value passed unmodified to notifier function
1684  *      @dev: net_device pointer passed unmodified to notifier function
1685  *
1686  *      Call all network notifier blocks.  Parameters and return value
1687  *      are as for raw_notifier_call_chain().
1688  */
1689 
1690 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1691 {
1692         struct netdev_notifier_info info;
1693 
1694         return call_netdevice_notifiers_info(val, dev, &info);
1695 }
1696 EXPORT_SYMBOL(call_netdevice_notifiers);
1697 
1698 #ifdef CONFIG_NET_INGRESS
1699 static struct static_key ingress_needed __read_mostly;
1700 
1701 void net_inc_ingress_queue(void)
1702 {
1703         static_key_slow_inc(&ingress_needed);
1704 }
1705 EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1706 
1707 void net_dec_ingress_queue(void)
1708 {
1709         static_key_slow_dec(&ingress_needed);
1710 }
1711 EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1712 #endif
1713 
1714 #ifdef CONFIG_NET_EGRESS
1715 static struct static_key egress_needed __read_mostly;
1716 
1717 void net_inc_egress_queue(void)
1718 {
1719         static_key_slow_inc(&egress_needed);
1720 }
1721 EXPORT_SYMBOL_GPL(net_inc_egress_queue);
1722 
1723 void net_dec_egress_queue(void)
1724 {
1725         static_key_slow_dec(&egress_needed);
1726 }
1727 EXPORT_SYMBOL_GPL(net_dec_egress_queue);
1728 #endif
1729 
1730 static struct static_key netstamp_needed __read_mostly;
1731 #ifdef HAVE_JUMP_LABEL
1732 static atomic_t netstamp_needed_deferred;
1733 static atomic_t netstamp_wanted;
1734 static void netstamp_clear(struct work_struct *work)
1735 {
1736         int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1737         int wanted;
1738 
1739         wanted = atomic_add_return(deferred, &netstamp_wanted);
1740         if (wanted > 0)
1741                 static_key_enable(&netstamp_needed);
1742         else
1743                 static_key_disable(&netstamp_needed);
1744 }
1745 static DECLARE_WORK(netstamp_work, netstamp_clear);
1746 #endif
1747 
1748 void net_enable_timestamp(void)
1749 {
1750 #ifdef HAVE_JUMP_LABEL
1751         int wanted;
1752 
1753         while (1) {
1754                 wanted = atomic_read(&netstamp_wanted);
1755                 if (wanted <= 0)
1756                         break;
1757                 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
1758                         return;
1759         }
1760         atomic_inc(&netstamp_needed_deferred);
1761         schedule_work(&netstamp_work);
1762 #else
1763         static_key_slow_inc(&netstamp_needed);
1764 #endif
1765 }
1766 EXPORT_SYMBOL(net_enable_timestamp);
1767 
1768 void net_disable_timestamp(void)
1769 {
1770 #ifdef HAVE_JUMP_LABEL
1771         int wanted;
1772 
1773         while (1) {
1774                 wanted = atomic_read(&netstamp_wanted);
1775                 if (wanted <= 1)
1776                         break;
1777                 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
1778                         return;
1779         }
1780         atomic_dec(&netstamp_needed_deferred);
1781         schedule_work(&netstamp_work);
1782 #else
1783         static_key_slow_dec(&netstamp_needed);
1784 #endif
1785 }
1786 EXPORT_SYMBOL(net_disable_timestamp);
1787 
1788 static inline void net_timestamp_set(struct sk_buff *skb)
1789 {
1790         skb->tstamp = 0;
1791         if (static_key_false(&netstamp_needed))
1792                 __net_timestamp(skb);
1793 }
1794 
1795 #define net_timestamp_check(COND, SKB)                  \
1796         if (static_key_false(&netstamp_needed)) {               \
1797                 if ((COND) && !(SKB)->tstamp)   \
1798                         __net_timestamp(SKB);           \
1799         }                                               \
1800 
1801 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
1802 {
1803         unsigned int len;
1804 
1805         if (!(dev->flags & IFF_UP))
1806                 return false;
1807 
1808         len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1809         if (skb->len <= len)
1810                 return true;
1811 
1812         /* if TSO is enabled, we don't care about the length as the packet
1813          * could be forwarded without being segmented before
1814          */
1815         if (skb_is_gso(skb))
1816                 return true;
1817 
1818         return false;
1819 }
1820 EXPORT_SYMBOL_GPL(is_skb_forwardable);
1821 
1822 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1823 {
1824         int ret = ____dev_forward_skb(dev, skb);
1825 
1826         if (likely(!ret)) {
1827                 skb->protocol = eth_type_trans(skb, dev);
1828                 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
1829         }
1830 
1831         return ret;
1832 }
1833 EXPORT_SYMBOL_GPL(__dev_forward_skb);
1834 
1835 /**
1836  * dev_forward_skb - loopback an skb to another netif
1837  *
1838  * @dev: destination network device
1839  * @skb: buffer to forward
1840  *
1841  * return values:
1842  *      NET_RX_SUCCESS  (no congestion)
1843  *      NET_RX_DROP     (packet was dropped, but freed)
1844  *
1845  * dev_forward_skb can be used for injecting an skb from the
1846  * start_xmit function of one device into the receive queue
1847  * of another device.
1848  *
1849  * The receiving device may be in another namespace, so
1850  * we have to clear all information in the skb that could
1851  * impact namespace isolation.
1852  */
1853 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1854 {
1855         return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
1856 }
1857 EXPORT_SYMBOL_GPL(dev_forward_skb);
1858 
1859 static inline int deliver_skb(struct sk_buff *skb,
1860                               struct packet_type *pt_prev,
1861                               struct net_device *orig_dev)
1862 {
1863         if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1864                 return -ENOMEM;
1865         refcount_inc(&skb->users);
1866         return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1867 }
1868 
1869 static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1870                                           struct packet_type **pt,
1871                                           struct net_device *orig_dev,
1872                                           __be16 type,
1873                                           struct list_head *ptype_list)
1874 {
1875         struct packet_type *ptype, *pt_prev = *pt;
1876 
1877         list_for_each_entry_rcu(ptype, ptype_list, list) {
1878                 if (ptype->type != type)
1879                         continue;
1880                 if (pt_prev)
1881                         deliver_skb(skb, pt_prev, orig_dev);
1882                 pt_prev = ptype;
1883         }
1884         *pt = pt_prev;
1885 }
1886 
1887 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1888 {
1889         if (!ptype->af_packet_priv || !skb->sk)
1890                 return false;
1891 
1892         if (ptype->id_match)
1893                 return ptype->id_match(ptype, skb->sk);
1894         else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1895                 return true;
1896 
1897         return false;
1898 }
1899 
1900 /*
1901  *      Support routine. Sends outgoing frames to any network
1902  *      taps currently in use.
1903  */
1904 
1905 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1906 {
1907         struct packet_type *ptype;
1908         struct sk_buff *skb2 = NULL;
1909         struct packet_type *pt_prev = NULL;
1910         struct list_head *ptype_list = &ptype_all;
1911 
1912         rcu_read_lock();
1913 again:
1914         list_for_each_entry_rcu(ptype, ptype_list, list) {
1915                 /* Never send packets back to the socket
1916                  * they originated from - MvS (miquels@drinkel.ow.org)
1917                  */
1918                 if (skb_loop_sk(ptype, skb))
1919                         continue;
1920 
1921                 if (pt_prev) {
1922                         deliver_skb(skb2, pt_prev, skb->dev);
1923                         pt_prev = ptype;
1924                         continue;
1925                 }
1926 
1927                 /* need to clone skb, done only once */
1928                 skb2 = skb_clone(skb, GFP_ATOMIC);
1929                 if (!skb2)
1930                         goto out_unlock;
1931 
1932                 net_timestamp_set(skb2);
1933 
1934                 /* skb->nh should be correctly
1935                  * set by sender, so that the second statement is
1936                  * just protection against buggy protocols.
1937                  */
1938                 skb_reset_mac_header(skb2);
1939 
1940                 if (skb_network_header(skb2) < skb2->data ||
1941                     skb_network_header(skb2) > skb_tail_pointer(skb2)) {
1942                         net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1943                                              ntohs(skb2->protocol),
1944                                              dev->name);
1945                         skb_reset_network_header(skb2);
1946                 }
1947 
1948                 skb2->transport_header = skb2->network_header;
1949                 skb2->pkt_type = PACKET_OUTGOING;
1950                 pt_prev = ptype;
1951         }
1952 
1953         if (ptype_list == &ptype_all) {
1954                 ptype_list = &dev->ptype_all;
1955                 goto again;
1956         }
1957 out_unlock:
1958         if (pt_prev)
1959                 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
1960         rcu_read_unlock();
1961 }
1962 EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
1963 
1964 /**
1965  * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1966  * @dev: Network device
1967  * @txq: number of queues available
1968  *
1969  * If real_num_tx_queues is changed the tc mappings may no longer be
1970  * valid. To resolve this verify the tc mapping remains valid and if
1971  * not NULL the mapping. With no priorities mapping to this
1972  * offset/count pair it will no longer be used. In the worst case TC0
1973  * is invalid nothing can be done so disable priority mappings. If is
1974  * expected that drivers will fix this mapping if they can before
1975  * calling netif_set_real_num_tx_queues.
1976  */
1977 static void netif_setup_tc(struct net_device *dev, unsigned int txq)
1978 {
1979         int i;
1980         struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1981 
1982         /* If TC0 is invalidated disable TC mapping */
1983         if (tc->offset + tc->count > txq) {
1984                 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
1985                 dev->num_tc = 0;
1986                 return;
1987         }
1988 
1989         /* Invalidated prio to tc mappings set to TC0 */
1990         for (i = 1; i < TC_BITMASK + 1; i++) {
1991                 int q = netdev_get_prio_tc_map(dev, i);
1992 
1993                 tc = &dev->tc_to_txq[q];
1994                 if (tc->offset + tc->count > txq) {
1995                         pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1996                                 i, q);
1997                         netdev_set_prio_tc_map(dev, i, 0);
1998                 }
1999         }
2000 }
2001 
2002 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2003 {
2004         if (dev->num_tc) {
2005                 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2006                 int i;
2007 
2008                 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2009                         if ((txq - tc->offset) < tc->count)
2010                                 return i;
2011                 }
2012 
2013                 return -1;
2014         }
2015 
2016         return 0;
2017 }
2018 
2019 #ifdef CONFIG_XPS
2020 static DEFINE_MUTEX(xps_map_mutex);
2021 #define xmap_dereference(P)             \
2022         rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2023 
2024 static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2025                              int tci, u16 index)
2026 {
2027         struct xps_map *map = NULL;
2028         int pos;
2029 
2030         if (dev_maps)
2031                 map = xmap_dereference(dev_maps->cpu_map[tci]);
2032         if (!map)
2033                 return false;
2034 
2035         for (pos = map->len; pos--;) {
2036                 if (map->queues[pos] != index)
2037                         continue;
2038 
2039                 if (map->len > 1) {
2040                         map->queues[pos] = map->queues[--map->len];
2041                         break;
2042                 }
2043 
2044                 RCU_INIT_POINTER(dev_maps->cpu_map[tci], NULL);
2045                 kfree_rcu(map, rcu);
2046                 return false;
2047         }
2048 
2049         return true;
2050 }
2051 
2052 static bool remove_xps_queue_cpu(struct net_device *dev,
2053                                  struct xps_dev_maps *dev_maps,
2054                                  int cpu, u16 offset, u16 count)
2055 {
2056         int num_tc = dev->num_tc ? : 1;
2057         bool active = false;
2058         int tci;
2059 
2060         for (tci = cpu * num_tc; num_tc--; tci++) {
2061                 int i, j;
2062 
2063                 for (i = count, j = offset; i--; j++) {
2064                         if (!remove_xps_queue(dev_maps, cpu, j))
2065                                 break;
2066                 }
2067 
2068                 active |= i < 0;
2069         }
2070 
2071         return active;
2072 }
2073 
2074 static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2075                                    u16 count)
2076 {
2077         struct xps_dev_maps *dev_maps;
2078         int cpu, i;
2079         bool active = false;
2080 
2081         mutex_lock(&xps_map_mutex);
2082         dev_maps = xmap_dereference(dev->xps_maps);
2083 
2084         if (!dev_maps)
2085                 goto out_no_maps;
2086 
2087         for_each_possible_cpu(cpu)
2088                 active |= remove_xps_queue_cpu(dev, dev_maps, cpu,
2089                                                offset, count);
2090 
2091         if (!active) {
2092                 RCU_INIT_POINTER(dev->xps_maps, NULL);
2093                 kfree_rcu(dev_maps, rcu);
2094         }
2095 
2096         for (i = offset + (count - 1); count--; i--)
2097                 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
2098                                              NUMA_NO_NODE);
2099 
2100 out_no_maps:
2101         mutex_unlock(&xps_map_mutex);
2102 }
2103 
2104 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2105 {
2106         netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2107 }
2108 
2109 static struct xps_map *expand_xps_map(struct xps_map *map,
2110                                       int cpu, u16 index)
2111 {
2112         struct xps_map *new_map;
2113         int alloc_len = XPS_MIN_MAP_ALLOC;
2114         int i, pos;
2115 
2116         for (pos = 0; map && pos < map->len; pos++) {
2117                 if (map->queues[pos] != index)
2118                         continue;
2119                 return map;
2120         }
2121 
2122         /* Need to add queue to this CPU's existing map */
2123         if (map) {
2124                 if (pos < map->alloc_len)
2125                         return map;
2126 
2127                 alloc_len = map->alloc_len * 2;
2128         }
2129 
2130         /* Need to allocate new map to store queue on this CPU's map */
2131         new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2132                                cpu_to_node(cpu));
2133         if (!new_map)
2134                 return NULL;
2135 
2136         for (i = 0; i < pos; i++)
2137                 new_map->queues[i] = map->queues[i];
2138         new_map->alloc_len = alloc_len;
2139         new_map->len = pos;
2140 
2141         return new_map;
2142 }
2143 
2144 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2145                         u16 index)
2146 {
2147         struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
2148         int i, cpu, tci, numa_node_id = -2;
2149         int maps_sz, num_tc = 1, tc = 0;
2150         struct xps_map *map, *new_map;
2151         bool active = false;
2152 
2153         if (dev->num_tc) {
2154                 num_tc = dev->num_tc;
2155                 tc = netdev_txq_to_tc(dev, index);
2156                 if (tc < 0)
2157                         return -EINVAL;
2158         }
2159 
2160         maps_sz = XPS_DEV_MAPS_SIZE(num_tc);
2161         if (maps_sz < L1_CACHE_BYTES)
2162                 maps_sz = L1_CACHE_BYTES;
2163 
2164         mutex_lock(&xps_map_mutex);
2165 
2166         dev_maps = xmap_dereference(dev->xps_maps);
2167 
2168         /* allocate memory for queue storage */
2169         for_each_cpu_and(cpu, cpu_online_mask, mask) {
2170                 if (!new_dev_maps)
2171                         new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2172                 if (!new_dev_maps) {
2173                         mutex_unlock(&xps_map_mutex);
2174                         return -ENOMEM;
2175                 }
2176 
2177                 tci = cpu * num_tc + tc;
2178                 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[tci]) :
2179                                  NULL;
2180 
2181                 map = expand_xps_map(map, cpu, index);
2182                 if (!map)
2183                         goto error;
2184 
2185                 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2186         }
2187 
2188         if (!new_dev_maps)
2189                 goto out_no_new_maps;
2190 
2191         for_each_possible_cpu(cpu) {
2192                 /* copy maps belonging to foreign traffic classes */
2193                 for (i = tc, tci = cpu * num_tc; dev_maps && i--; tci++) {
2194                         /* fill in the new device map from the old device map */
2195                         map = xmap_dereference(dev_maps->cpu_map[tci]);
2196                         RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2197                 }
2198 
2199                 /* We need to explicitly update tci as prevous loop
2200                  * could break out early if dev_maps is NULL.
2201                  */
2202                 tci = cpu * num_tc + tc;
2203 
2204                 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
2205                         /* add queue to CPU maps */
2206                         int pos = 0;
2207 
2208                         map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2209                         while ((pos < map->len) && (map->queues[pos] != index))
2210                                 pos++;
2211 
2212                         if (pos == map->len)
2213                                 map->queues[map->len++] = index;
2214 #ifdef CONFIG_NUMA
2215                         if (numa_node_id == -2)
2216                                 numa_node_id = cpu_to_node(cpu);
2217                         else if (numa_node_id != cpu_to_node(cpu))
2218                                 numa_node_id = -1;
2219 #endif
2220                 } else if (dev_maps) {
2221                         /* fill in the new device map from the old device map */
2222                         map = xmap_dereference(dev_maps->cpu_map[tci]);
2223                         RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2224                 }
2225 
2226                 /* copy maps belonging to foreign traffic classes */
2227                 for (i = num_tc - tc, tci++; dev_maps && --i; tci++) {
2228                         /* fill in the new device map from the old device map */
2229                         map = xmap_dereference(dev_maps->cpu_map[tci]);
2230                         RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2231                 }
2232         }
2233 
2234         rcu_assign_pointer(dev->xps_maps, new_dev_maps);
2235 
2236         /* Cleanup old maps */
2237         if (!dev_maps)
2238                 goto out_no_old_maps;
2239 
2240         for_each_possible_cpu(cpu) {
2241                 for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2242                         new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2243                         map = xmap_dereference(dev_maps->cpu_map[tci]);
2244                         if (map && map != new_map)
2245                                 kfree_rcu(map, rcu);
2246                 }
2247         }
2248 
2249         kfree_rcu(dev_maps, rcu);
2250 
2251 out_no_old_maps:
2252         dev_maps = new_dev_maps;
2253         active = true;
2254 
2255 out_no_new_maps:
2256         /* update Tx queue numa node */
2257         netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2258                                      (numa_node_id >= 0) ? numa_node_id :
2259                                      NUMA_NO_NODE);
2260 
2261         if (!dev_maps)
2262                 goto out_no_maps;
2263 
2264         /* removes queue from unused CPUs */
2265         for_each_possible_cpu(cpu) {
2266                 for (i = tc, tci = cpu * num_tc; i--; tci++)
2267                         active |= remove_xps_queue(dev_maps, tci, index);
2268                 if (!cpumask_test_cpu(cpu, mask) || !cpu_online(cpu))
2269                         active |= remove_xps_queue(dev_maps, tci, index);
2270                 for (i = num_tc - tc, tci++; --i; tci++)
2271                         active |= remove_xps_queue(dev_maps, tci, index);
2272         }
2273 
2274         /* free map if not active */
2275         if (!active) {
2276                 RCU_INIT_POINTER(dev->xps_maps, NULL);
2277                 kfree_rcu(dev_maps, rcu);
2278         }
2279 
2280 out_no_maps:
2281         mutex_unlock(&xps_map_mutex);
2282 
2283         return 0;
2284 error:
2285         /* remove any maps that we added */
2286         for_each_possible_cpu(cpu) {
2287                 for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2288                         new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2289                         map = dev_maps ?
2290                               xmap_dereference(dev_maps->cpu_map[tci]) :
2291                               NULL;
2292                         if (new_map && new_map != map)
2293                                 kfree(new_map);
2294                 }
2295         }
2296 
2297         mutex_unlock(&xps_map_mutex);
2298 
2299         kfree(new_dev_maps);
2300         return -ENOMEM;
2301 }
2302 EXPORT_SYMBOL(netif_set_xps_queue);
2303 
2304 #endif
2305 void netdev_reset_tc(struct net_device *dev)
2306 {
2307 #ifdef CONFIG_XPS
2308         netif_reset_xps_queues_gt(dev, 0);
2309 #endif
2310         dev->num_tc = 0;
2311         memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2312         memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2313 }
2314 EXPORT_SYMBOL(netdev_reset_tc);
2315 
2316 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2317 {
2318         if (tc >= dev->num_tc)
2319                 return -EINVAL;
2320 
2321 #ifdef CONFIG_XPS
2322         netif_reset_xps_queues(dev, offset, count);
2323 #endif
2324         dev->tc_to_txq[tc].count = count;
2325         dev->tc_to_txq[tc].offset = offset;
2326         return 0;
2327 }
2328 EXPORT_SYMBOL(netdev_set_tc_queue);
2329 
2330 int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2331 {
2332         if (num_tc > TC_MAX_QUEUE)
2333                 return -EINVAL;
2334 
2335 #ifdef CONFIG_XPS
2336         netif_reset_xps_queues_gt(dev, 0);
2337 #endif
2338         dev->num_tc = num_tc;
2339         return 0;
2340 }
2341 EXPORT_SYMBOL(netdev_set_num_tc);
2342 
2343 /*
2344  * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2345  * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2346  */
2347 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2348 {
2349         int rc;
2350 
2351         if (txq < 1 || txq > dev->num_tx_queues)
2352                 return -EINVAL;
2353 
2354         if (dev->reg_state == NETREG_REGISTERED ||
2355             dev->reg_state == NETREG_UNREGISTERING) {
2356                 ASSERT_RTNL();
2357 
2358                 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2359                                                   txq);
2360                 if (rc)
2361                         return rc;
2362 
2363                 if (dev->num_tc)
2364                         netif_setup_tc(dev, txq);
2365 
2366                 if (txq < dev->real_num_tx_queues) {
2367                         qdisc_reset_all_tx_gt(dev, txq);
2368 #ifdef CONFIG_XPS
2369                         netif_reset_xps_queues_gt(dev, txq);
2370 #endif
2371                 }
2372         }
2373 
2374         dev->real_num_tx_queues = txq;
2375         return 0;
2376 }
2377 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2378 
2379 #ifdef CONFIG_SYSFS
2380 /**
2381  *      netif_set_real_num_rx_queues - set actual number of RX queues used
2382  *      @dev: Network device
2383  *      @rxq: Actual number of RX queues
2384  *
2385  *      This must be called either with the rtnl_lock held or before
2386  *      registration of the net device.  Returns 0 on success, or a
2387  *      negative error code.  If called before registration, it always
2388  *      succeeds.
2389  */
2390 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2391 {
2392         int rc;
2393 
2394         if (rxq < 1 || rxq > dev->num_rx_queues)
2395                 return -EINVAL;
2396 
2397         if (dev->reg_state == NETREG_REGISTERED) {
2398                 ASSERT_RTNL();
2399 
2400                 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2401                                                   rxq);
2402                 if (rc)
2403                         return rc;
2404         }
2405 
2406         dev->real_num_rx_queues = rxq;
2407         return 0;
2408 }
2409 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2410 #endif
2411 
2412 /**
2413  * netif_get_num_default_rss_queues - default number of RSS queues
2414  *
2415  * This routine should set an upper limit on the number of RSS queues
2416  * used by default by multiqueue devices.
2417  */
2418 int netif_get_num_default_rss_queues(void)
2419 {
2420         return is_kdump_kernel() ?
2421                 1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2422 }
2423 EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2424 
2425 static void __netif_reschedule(struct Qdisc *q)
2426 {
2427         struct softnet_data *sd;
2428         unsigned long flags;
2429 
2430         local_irq_save(flags);
2431         sd = this_cpu_ptr(&softnet_data);
2432         q->next_sched = NULL;
2433         *sd->output_queue_tailp = q;
2434         sd->output_queue_tailp = &q->next_sched;
2435         raise_softirq_irqoff(NET_TX_SOFTIRQ);
2436         local_irq_restore(flags);
2437 }
2438 
2439 void __netif_schedule(struct Qdisc *q)
2440 {
2441         if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2442                 __netif_reschedule(q);
2443 }
2444 EXPORT_SYMBOL(__netif_schedule);
2445 
2446 struct dev_kfree_skb_cb {
2447         enum skb_free_reason reason;
2448 };
2449 
2450 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
2451 {
2452         return (struct dev_kfree_skb_cb *)skb->cb;
2453 }
2454 
2455 void netif_schedule_queue(struct netdev_queue *txq)
2456 {
2457         rcu_read_lock();
2458         if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2459                 struct Qdisc *q = rcu_dereference(txq->qdisc);
2460 
2461                 __netif_schedule(q);
2462         }
2463         rcu_read_unlock();
2464 }
2465 EXPORT_SYMBOL(netif_schedule_queue);
2466 
2467 void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2468 {
2469         if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2470                 struct Qdisc *q;
2471 
2472                 rcu_read_lock();
2473                 q = rcu_dereference(dev_queue->qdisc);
2474                 __netif_schedule(q);
2475                 rcu_read_unlock();
2476         }
2477 }
2478 EXPORT_SYMBOL(netif_tx_wake_queue);
2479 
2480 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2481 {
2482         unsigned long flags;
2483 
2484         if (unlikely(!skb))
2485                 return;
2486 
2487         if (likely(refcount_read(&skb->users) == 1)) {
2488                 smp_rmb();
2489                 refcount_set(&skb->users, 0);
2490         } else if (likely(!refcount_dec_and_test(&skb->users))) {
2491                 return;
2492         }
2493         get_kfree_skb_cb(skb)->reason = reason;
2494         local_irq_save(flags);
2495         skb->next = __this_cpu_read(softnet_data.completion_queue);
2496         __this_cpu_write(softnet_data.completion_queue, skb);
2497         raise_softirq_irqoff(NET_TX_SOFTIRQ);
2498         local_irq_restore(flags);
2499 }
2500 EXPORT_SYMBOL(__dev_kfree_skb_irq);
2501 
2502 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
2503 {
2504         if (in_irq() || irqs_disabled())
2505                 __dev_kfree_skb_irq(skb, reason);
2506         else
2507                 dev_kfree_skb(skb);
2508 }
2509 EXPORT_SYMBOL(__dev_kfree_skb_any);
2510 
2511 
2512 /**
2513  * netif_device_detach - mark device as removed
2514  * @dev: network device
2515  *
2516  * Mark device as removed from system and therefore no longer available.
2517  */
2518 void netif_device_detach(struct net_device *dev)
2519 {
2520         if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2521             netif_running(dev)) {
2522                 netif_tx_stop_all_queues(dev);
2523         }
2524 }
2525 EXPORT_SYMBOL(netif_device_detach);
2526 
2527 /**
2528  * netif_device_attach - mark device as attached
2529  * @dev: network device
2530  *
2531  * Mark device as attached from system and restart if needed.
2532  */
2533 void netif_device_attach(struct net_device *dev)
2534 {
2535         if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2536             netif_running(dev)) {
2537                 netif_tx_wake_all_queues(dev);
2538                 __netdev_watchdog_up(dev);
2539         }
2540 }
2541 EXPORT_SYMBOL(netif_device_attach);
2542 
2543 /*
2544  * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2545  * to be used as a distribution range.
2546  */
2547 u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
2548                   unsigned int num_tx_queues)
2549 {
2550         u32 hash;
2551         u16 qoffset = 0;
2552         u16 qcount = num_tx_queues;
2553 
2554         if (skb_rx_queue_recorded(skb)) {
2555                 hash = skb_get_rx_queue(skb);
2556                 while (unlikely(hash >= num_tx_queues))
2557                         hash -= num_tx_queues;
2558                 return hash;
2559         }
2560 
2561         if (dev->num_tc) {
2562                 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2563 
2564                 qoffset = dev->tc_to_txq[tc].offset;
2565                 qcount = dev->tc_to_txq[tc].count;
2566         }
2567 
2568         return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2569 }
2570 EXPORT_SYMBOL(__skb_tx_hash);
2571 
2572 static void skb_warn_bad_offload(const struct sk_buff *skb)
2573 {
2574         static const netdev_features_t null_features;
2575         struct net_device *dev = skb->dev;
2576         const char *name = "";
2577 
2578         if (!net_ratelimit())
2579                 return;
2580 
2581         if (dev) {
2582                 if (dev->dev.parent)
2583                         name = dev_driver_string(dev->dev.parent);
2584                 else
2585                         name = netdev_name(dev);
2586         }
2587         WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2588              "gso_type=%d ip_summed=%d\n",
2589              name, dev ? &dev->features : &null_features,
2590              skb->sk ? &skb->sk->sk_route_caps : &null_features,
2591              skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2592              skb_shinfo(skb)->gso_type, skb->ip_summed);
2593 }
2594 
2595 /*
2596  * Invalidate hardware checksum when packet is to be mangled, and
2597  * complete checksum manually on outgoing path.
2598  */
2599 int skb_checksum_help(struct sk_buff *skb)
2600 {
2601         __wsum csum;
2602         int ret = 0, offset;
2603 
2604         if (skb->ip_summed == CHECKSUM_COMPLETE)
2605                 goto out_set_summed;
2606 
2607         if (unlikely(skb_shinfo(skb)->gso_size)) {
2608                 skb_warn_bad_offload(skb);
2609                 return -EINVAL;
2610         }
2611 
2612         /* Before computing a checksum, we should make sure no frag could
2613          * be modified by an external entity : checksum could be wrong.
2614          */
2615         if (skb_has_shared_frag(skb)) {
2616                 ret = __skb_linearize(skb);
2617                 if (ret)
2618                         goto out;
2619         }
2620 
2621         offset = skb_checksum_start_offset(skb);
2622         BUG_ON(offset >= skb_headlen(skb));
2623         csum = skb_checksum(skb, offset, skb->len - offset, 0);
2624 
2625         offset += skb->csum_offset;
2626         BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2627 
2628         if (skb_cloned(skb) &&
2629             !skb_clone_writable(skb, offset + sizeof(__sum16))) {
2630                 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2631                 if (ret)
2632                         goto out;
2633         }
2634 
2635         *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
2636 out_set_summed:
2637         skb->ip_summed = CHECKSUM_NONE;
2638 out:
2639         return ret;
2640 }
2641 EXPORT_SYMBOL(skb_checksum_help);
2642 
2643 int skb_crc32c_csum_help(struct sk_buff *skb)
2644 {
2645         __le32 crc32c_csum;
2646         int ret = 0, offset, start;
2647 
2648         if (skb->ip_summed != CHECKSUM_PARTIAL)
2649                 goto out;
2650 
2651         if (unlikely(skb_is_gso(skb)))
2652                 goto out;
2653 
2654         /* Before computing a checksum, we should make sure no frag could
2655          * be modified by an external entity : checksum could be wrong.
2656          */
2657         if (unlikely(skb_has_shared_frag(skb))) {
2658                 ret = __skb_linearize(skb);
2659                 if (ret)
2660                         goto out;
2661         }
2662         start = skb_checksum_start_offset(skb);
2663         offset = start + offsetof(struct sctphdr, checksum);
2664         if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
2665                 ret = -EINVAL;
2666                 goto out;
2667         }
2668         if (skb_cloned(skb) &&
2669             !skb_clone_writable(skb, offset + sizeof(__le32))) {
2670                 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2671                 if (ret)
2672                         goto out;
2673         }
2674         crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
2675                                                   skb->len - start, ~(__u32)0,
2676                                                   crc32c_csum_stub));
2677         *(__le32 *)(skb->data + offset) = crc32c_csum;
2678         skb->ip_summed = CHECKSUM_NONE;
2679         skb->csum_not_inet = 0;
2680 out:
2681         return ret;
2682 }
2683 
2684 __be16 skb_network_protocol(struct sk_buff *skb, int *depth)
2685 {
2686         __be16 type = skb->protocol;
2687 
2688         /* Tunnel gso handlers can set protocol to ethernet. */
2689         if (type == htons(ETH_P_TEB)) {
2690                 struct ethhdr *eth;
2691 
2692                 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2693                         return 0;
2694 
2695                 eth = (struct ethhdr *)skb_mac_header(skb);
2696                 type = eth->h_proto;
2697         }
2698 
2699         return __vlan_get_protocol(skb, type, depth);
2700 }
2701 
2702 /**
2703  *      skb_mac_gso_segment - mac layer segmentation handler.
2704  *      @skb: buffer to segment
2705  *      @features: features for the output path (see dev->features)
2706  */
2707 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2708                                     netdev_features_t features)
2709 {
2710         struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2711         struct packet_offload *ptype;
2712         int vlan_depth = skb->mac_len;
2713         __be16 type = skb_network_protocol(skb, &vlan_depth);
2714 
2715         if (unlikely(!type))
2716                 return ERR_PTR(-EINVAL);
2717 
2718         __skb_pull(skb, vlan_depth);
2719 
2720         rcu_read_lock();
2721         list_for_each_entry_rcu(ptype, &offload_base, list) {
2722                 if (ptype->type == type && ptype->callbacks.gso_segment) {
2723                         segs = ptype->callbacks.gso_segment(skb, features);
2724                         break;
2725                 }
2726         }
2727         rcu_read_unlock();
2728 
2729         __skb_push(skb, skb->data - skb_mac_header(skb));
2730 
2731         return segs;
2732 }
2733 EXPORT_SYMBOL(skb_mac_gso_segment);
2734 
2735 
2736 /* openvswitch calls this on rx path, so we need a different check.
2737  */
2738 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2739 {
2740         if (tx_path)
2741                 return skb->ip_summed != CHECKSUM_PARTIAL &&
2742                        skb->ip_summed != CHECKSUM_UNNECESSARY;
2743 
2744         return skb->ip_summed == CHECKSUM_NONE;
2745 }
2746 
2747 /**
2748  *      __skb_gso_segment - Perform segmentation on skb.
2749  *      @skb: buffer to segment
2750  *      @features: features for the output path (see dev->features)
2751  *      @tx_path: whether it is called in TX path
2752  *
2753  *      This function segments the given skb and returns a list of segments.
2754  *
2755  *      It may return NULL if the skb requires no segmentation.  This is
2756  *      only possible when GSO is used for verifying header integrity.
2757  *
2758  *      Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb.
2759  */
2760 struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2761                                   netdev_features_t features, bool tx_path)
2762 {
2763         struct sk_buff *segs;
2764 
2765         if (unlikely(skb_needs_check(skb, tx_path))) {
2766                 int err;
2767 
2768                 /* We're going to init ->check field in TCP or UDP header */
2769                 err = skb_cow_head(skb, 0);
2770                 if (err < 0)
2771                         return ERR_PTR(err);
2772         }
2773 
2774         /* Only report GSO partial support if it will enable us to
2775          * support segmentation on this frame without needing additional
2776          * work.
2777          */
2778         if (features & NETIF_F_GSO_PARTIAL) {
2779                 netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
2780                 struct net_device *dev = skb->dev;
2781 
2782                 partial_features |= dev->features & dev->gso_partial_features;
2783                 if (!skb_gso_ok(skb, features | partial_features))
2784                         features &= ~NETIF_F_GSO_PARTIAL;
2785         }
2786 
2787         BUILD_BUG_ON(SKB_SGO_CB_OFFSET +
2788                      sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
2789 
2790         SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
2791         SKB_GSO_CB(skb)->encap_level = 0;
2792 
2793         skb_reset_mac_header(skb);
2794         skb_reset_mac_len(skb);
2795 
2796         segs = skb_mac_gso_segment(skb, features);
2797 
2798         if (unlikely(skb_needs_check(skb, tx_path)))
2799                 skb_warn_bad_offload(skb);
2800 
2801         return segs;
2802 }
2803 EXPORT_SYMBOL(__skb_gso_segment);
2804 
2805 /* Take action when hardware reception checksum errors are detected. */
2806 #ifdef CONFIG_BUG
2807 void netdev_rx_csum_fault(struct net_device *dev)
2808 {
2809         if (net_ratelimit()) {
2810                 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
2811                 dump_stack();
2812         }
2813 }
2814 EXPORT_SYMBOL(netdev_rx_csum_fault);
2815 #endif
2816 
2817 /* Actually, we should eliminate this check as soon as we know, that:
2818  * 1. IOMMU is present and allows to map all the memory.
2819  * 2. No high memory really exists on this machine.
2820  */
2821 
2822 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
2823 {
2824 #ifdef CONFIG_HIGHMEM
2825         int i;
2826 
2827         if (!(dev->features & NETIF_F_HIGHDMA)) {
2828                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2829                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2830 
2831                         if (PageHighMem(skb_frag_page(frag)))
2832                                 return 1;
2833                 }
2834         }
2835 
2836         if (PCI_DMA_BUS_IS_PHYS) {
2837                 struct device *pdev = dev->dev.parent;
2838 
2839                 if (!pdev)
2840                         return 0;
2841                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2842                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2843                         dma_addr_t addr = page_to_phys(skb_frag_page(frag));
2844 
2845                         if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2846                                 return 1;
2847                 }
2848         }
2849 #endif
2850         return 0;
2851 }
2852 
2853 /* If MPLS offload request, verify we are testing hardware MPLS features
2854  * instead of standard features for the netdev.
2855  */
2856 #if IS_ENABLED(CONFIG_NET_MPLS_GSO)
2857 static netdev_features_t net_mpls_features(struct sk_buff *skb,
2858                                            netdev_features_t features,
2859                                            __be16 type)
2860 {
2861         if (eth_p_mpls(type))
2862                 features &= skb->dev->mpls_features;
2863 
2864         return features;
2865 }
2866 #else
2867 static netdev_features_t net_mpls_features(struct sk_buff *skb,
2868                                            netdev_features_t features,
2869                                            __be16 type)
2870 {
2871         return features;
2872 }
2873 #endif
2874 
2875 static netdev_features_t harmonize_features(struct sk_buff *skb,
2876         netdev_features_t features)
2877 {
2878         int tmp;
2879         __be16 type;
2880 
2881         type = skb_network_protocol(skb, &tmp);
2882         features = net_mpls_features(skb, features, type);
2883 
2884         if (skb->ip_summed != CHECKSUM_NONE &&
2885             !can_checksum_protocol(features, type)) {
2886                 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
2887         }
2888         if (illegal_highdma(skb->dev, skb))
2889                 features &= ~NETIF_F_SG;
2890 
2891         return features;
2892 }
2893 
2894 netdev_features_t passthru_features_check(struct sk_buff *skb,
2895                                           struct net_device *dev,
2896                                           netdev_features_t features)
2897 {
2898         return features;
2899 }
2900 EXPORT_SYMBOL(passthru_features_check);
2901 
2902 static netdev_features_t dflt_features_check(const struct sk_buff *skb,
2903                                              struct net_device *dev,
2904                                              netdev_features_t features)
2905 {
2906         return vlan_features_check(skb, features);
2907 }
2908 
2909 static netdev_features_t gso_features_check(const struct sk_buff *skb,
2910                                             struct net_device *dev,
2911                                             netdev_features_t features)
2912 {
2913         u16 gso_segs = skb_shinfo(skb)->gso_segs;
2914 
2915         if (gso_segs > dev->gso_max_segs)
2916                 return features & ~NETIF_F_GSO_MASK;
2917 
2918         /* Support for GSO partial features requires software
2919          * intervention before we can actually process the packets
2920          * so we need to strip support for any partial features now
2921          * and we can pull them back in after we have partially
2922          * segmented the frame.
2923          */
2924         if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
2925                 features &= ~dev->gso_partial_features;
2926 
2927         /* Make sure to clear the IPv4 ID mangling feature if the
2928          * IPv4 header has the potential to be fragmented.
2929          */
2930         if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
2931                 struct iphdr *iph = skb->encapsulation ?
2932                                     inner_ip_hdr(skb) : ip_hdr(skb);
2933 
2934                 if (!(iph->frag_off & htons(IP_DF)))
2935                         features &= ~NETIF_F_TSO_MANGLEID;
2936         }
2937 
2938         return features;
2939 }
2940 
2941 netdev_features_t netif_skb_features(struct sk_buff *skb)
2942 {
2943         struct net_device *dev = skb->dev;
2944         netdev_features_t features = dev->features;
2945 
2946         if (skb_is_gso(skb))
2947                 features = gso_features_check(skb, dev, features);
2948 
2949         /* If encapsulation offload request, verify we are testing
2950          * hardware encapsulation features instead of standard
2951          * features for the netdev
2952          */
2953         if (skb->encapsulation)
2954                 features &= dev->hw_enc_features;
2955 
2956         if (skb_vlan_tagged(skb))
2957                 features = netdev_intersect_features(features,
2958                                                      dev->vlan_features |
2959                                                      NETIF_F_HW_VLAN_CTAG_TX |
2960                                                      NETIF_F_HW_VLAN_STAG_TX);
2961 
2962         if (dev->netdev_ops->ndo_features_check)
2963                 features &= dev->netdev_ops->ndo_features_check(skb, dev,
2964                                                                 features);
2965         else
2966                 features &= dflt_features_check(skb, dev, features);
2967 
2968         return harmonize_features(skb, features);
2969 }
2970 EXPORT_SYMBOL(netif_skb_features);
2971 
2972 static int xmit_one(struct sk_buff *skb, struct net_device *dev,
2973                     struct netdev_queue *txq, bool more)
2974 {
2975         unsigned int len;
2976         int rc;
2977 
2978         if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
2979                 dev_queue_xmit_nit(skb, dev);
2980 
2981         len = skb->len;
2982         trace_net_dev_start_xmit(skb, dev);
2983         rc = netdev_start_xmit(skb, dev, txq, more);
2984         trace_net_dev_xmit(skb, rc, dev, len);
2985 
2986         return rc;
2987 }
2988 
2989 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
2990                                     struct netdev_queue *txq, int *ret)
2991 {
2992         struct sk_buff *skb = first;
2993         int rc = NETDEV_TX_OK;
2994 
2995         while (skb) {
2996                 struct sk_buff *next = skb->next;
2997 
2998                 skb->next = NULL;
2999                 rc = xmit_one(skb, dev, txq, next != NULL);
3000                 if (unlikely(!dev_xmit_complete(rc))) {
3001                         skb->next = next;
3002                         goto out;
3003                 }
3004 
3005                 skb = next;
3006                 if (netif_xmit_stopped(txq) && skb) {
3007                         rc = NETDEV_TX_BUSY;
3008                         break;
3009                 }
3010         }
3011 
3012 out:
3013         *ret = rc;
3014         return skb;
3015 }
3016 
3017 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
3018                                           netdev_features_t features)
3019 {
3020         if (skb_vlan_tag_present(skb) &&
3021             !vlan_hw_offload_capable(features, skb->vlan_proto))
3022                 skb = __vlan_hwaccel_push_inside(skb);
3023         return skb;
3024 }
3025 
3026 int skb_csum_hwoffload_help(struct sk_buff *skb,
3027                             const netdev_features_t features)
3028 {
3029         if (unlikely(skb->csum_not_inet))
3030                 return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3031                         skb_crc32c_csum_help(skb);
3032 
3033         return !!(features & NETIF_F_CSUM_MASK) ? 0 : skb_checksum_help(skb);
3034 }
3035 EXPORT_SYMBOL(skb_csum_hwoffload_help);
3036 
3037 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
3038 {
3039         netdev_features_t features;
3040 
3041         features = netif_skb_features(skb);
3042         skb = validate_xmit_vlan(skb, features);
3043         if (unlikely(!skb))
3044                 goto out_null;
3045 
3046         if (netif_needs_gso(skb, features)) {
3047                 struct sk_buff *segs;
3048 
3049                 segs = skb_gso_segment(skb, features);
3050                 if (IS_ERR(segs)) {
3051                         goto out_kfree_skb;
3052                 } else if (segs) {
3053                         consume_skb(skb);
3054                         skb = segs;
3055                 }
3056         } else {
3057                 if (skb_needs_linearize(skb, features) &&
3058                     __skb_linearize(skb))
3059                         goto out_kfree_skb;
3060 
3061                 if (validate_xmit_xfrm(skb, features))
3062                         goto out_kfree_skb;
3063 
3064                 /* If packet is not checksummed and device does not
3065                  * support checksumming for this protocol, complete
3066                  * checksumming here.
3067                  */
3068                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3069                         if (skb->encapsulation)
3070                                 skb_set_inner_transport_header(skb,
3071                                                                skb_checksum_start_offset(skb));
3072                         else
3073                                 skb_set_transport_header(skb,
3074                                                          skb_checksum_start_offset(skb));
3075                         if (skb_csum_hwoffload_help(skb, features))
3076                                 goto out_kfree_skb;
3077                 }
3078         }
3079 
3080         return skb;
3081 
3082 out_kfree_skb:
3083         kfree_skb(skb);
3084 out_null:
3085         atomic_long_inc(&dev->tx_dropped);
3086         return NULL;
3087 }
3088 
3089 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
3090 {
3091         struct sk_buff *next, *head = NULL, *tail;
3092 
3093         for (; skb != NULL; skb = next) {
3094                 next = skb->next;
3095                 skb->next = NULL;
3096 
3097                 /* in case skb wont be segmented, point to itself */
3098                 skb->prev = skb;
3099 
3100                 skb = validate_xmit_skb(skb, dev);
3101                 if (!skb)
3102                         continue;
3103 
3104                 if (!head)
3105                         head = skb;
3106                 else
3107                         tail->next = skb;
3108                 /* If skb was segmented, skb->prev points to
3109                  * the last segment. If not, it still contains skb.
3110                  */
3111                 tail = skb->prev;
3112         }
3113         return head;
3114 }
3115 EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3116 
3117 static void qdisc_pkt_len_init(struct sk_buff *skb)
3118 {
3119         const struct skb_shared_info *shinfo = skb_shinfo(skb);
3120 
3121         qdisc_skb_cb(skb)->pkt_len = skb->len;
3122 
3123         /* To get more precise estimation of bytes sent on wire,
3124          * we add to pkt_len the headers size of all segments
3125          */
3126         if (shinfo->gso_size)  {
3127                 unsigned int hdr_len;
3128                 u16 gso_segs = shinfo->gso_segs;
3129 
3130                 /* mac layer + network layer */
3131                 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3132 
3133                 /* + transport layer */
3134                 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
3135                         hdr_len += tcp_hdrlen(skb);
3136                 else
3137                         hdr_len += sizeof(struct udphdr);
3138 
3139                 if (shinfo->gso_type & SKB_GSO_DODGY)
3140                         gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3141                                                 shinfo->gso_size);
3142 
3143                 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3144         }
3145 }
3146 
3147 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3148                                  struct net_device *dev,
3149                                  struct netdev_queue *txq)
3150 {
3151         spinlock_t *root_lock = qdisc_lock(q);
3152         struct sk_buff *to_free = NULL;
3153         bool contended;
3154         int rc;
3155 
3156         qdisc_calculate_pkt_len(skb, q);
3157         /*
3158          * Heuristic to force contended enqueues to serialize on a
3159          * separate lock before trying to get qdisc main lock.
3160          * This permits qdisc->running owner to get the lock more
3161          * often and dequeue packets faster.
3162          */
3163         contended = qdisc_is_running(q);
3164         if (unlikely(contended))
3165                 spin_lock(&q->busylock);
3166 
3167         spin_lock(root_lock);
3168         if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3169                 __qdisc_drop(skb, &to_free);
3170                 rc = NET_XMIT_DROP;
3171         } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3172                    qdisc_run_begin(q)) {
3173                 /*
3174                  * This is a work-conserving queue; there are no old skbs
3175                  * waiting to be sent out; and the qdisc is not running -
3176                  * xmit the skb directly.
3177                  */
3178 
3179                 qdisc_bstats_update(q, skb);
3180 
3181                 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3182                         if (unlikely(contended)) {
3183                                 spin_unlock(&q->busylock);
3184                                 contended = false;
3185                         }
3186                         __qdisc_run(q);
3187                 } else
3188                         qdisc_run_end(q);
3189 
3190                 rc = NET_XMIT_SUCCESS;
3191         } else {
3192                 rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
3193                 if (qdisc_run_begin(q)) {
3194                         if (unlikely(contended)) {
3195                                 spin_unlock(&q->busylock);
3196                                 contended = false;
3197                         }
3198                         __qdisc_run(q);
3199                 }
3200         }
3201         spin_unlock(root_lock);
3202         if (unlikely(to_free))
3203                 kfree_skb_list(to_free);
3204         if (unlikely(contended))
3205                 spin_unlock(&q->busylock);
3206         return rc;
3207 }
3208 
3209 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3210 static void skb_update_prio(struct sk_buff *skb)
3211 {
3212         struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
3213 
3214         if (!skb->priority && skb->sk && map) {
3215                 unsigned int prioidx =
3216                         sock_cgroup_prioidx(&skb->sk->sk_cgrp_data);
3217 
3218                 if (prioidx < map->priomap_len)
3219                         skb->priority = map->priomap[prioidx];
3220         }
3221 }
3222 #else
3223 #define skb_update_prio(skb)
3224 #endif
3225 
3226 DEFINE_PER_CPU(int, xmit_recursion);
3227 EXPORT_SYMBOL(xmit_recursion);
3228 
3229 /**
3230  *      dev_loopback_xmit - loop back @skb
3231  *      @net: network namespace this loopback is happening in
3232  *      @sk:  sk needed to be a netfilter okfn
3233  *      @skb: buffer to transmit
3234  */
3235 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3236 {
3237         skb_reset_mac_header(skb);
3238         __skb_pull(skb, skb_network_offset(skb));
3239         skb->pkt_type = PACKET_LOOPBACK;
3240         skb->ip_summed = CHECKSUM_UNNECESSARY;
3241         WARN_ON(!skb_dst(skb));
3242         skb_dst_force(skb);
3243         netif_rx_ni(skb);
3244         return 0;
3245 }
3246 EXPORT_SYMBOL(dev_loopback_xmit);
3247 
3248 #ifdef CONFIG_NET_EGRESS
3249 static struct sk_buff *
3250 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3251 {
3252         struct tcf_proto *cl = rcu_dereference_bh(dev->egress_cl_list);
3253         struct tcf_result cl_res;
3254 
3255         if (!cl)
3256                 return skb;
3257 
3258         /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
3259         qdisc_bstats_cpu_update(cl->q, skb);
3260 
3261         switch (tcf_classify(skb, cl, &cl_res, false)) {
3262         case TC_ACT_OK:
3263         case TC_ACT_RECLASSIFY:
3264                 skb->tc_index = TC_H_MIN(cl_res.classid);
3265                 break;
3266         case TC_ACT_SHOT:
3267                 qdisc_qstats_cpu_drop(cl->q);
3268                 *ret = NET_XMIT_DROP;
3269                 kfree_skb(skb);
3270                 return NULL;
3271         case TC_ACT_STOLEN:
3272         case TC_ACT_QUEUED:
3273         case TC_ACT_TRAP:
3274                 *ret = NET_XMIT_SUCCESS;
3275                 consume_skb(skb);
3276                 return NULL;
3277         case TC_ACT_REDIRECT:
3278                 /* No need to push/pop skb's mac_header here on egress! */
3279                 skb_do_redirect(skb);
3280                 *ret = NET_XMIT_SUCCESS;
3281                 return NULL;
3282         default:
3283                 break;
3284         }
3285 
3286         return skb;
3287 }
3288 #endif /* CONFIG_NET_EGRESS */
3289 
3290 static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
3291 {
3292 #ifdef CONFIG_XPS
3293         struct xps_dev_maps *dev_maps;
3294         struct xps_map *map;
3295         int queue_index = -1;
3296 
3297         rcu_read_lock();
3298         dev_maps = rcu_dereference(dev->xps_maps);
3299         if (dev_maps) {
3300                 unsigned int tci = skb->sender_cpu - 1;
3301 
3302                 if (dev->num_tc) {
3303                         tci *= dev->num_tc;
3304                         tci += netdev_get_prio_tc_map(dev, skb->priority);
3305                 }
3306 
3307                 map = rcu_dereference(dev_maps->cpu_map[tci]);
3308                 if (map) {
3309                         if (map->len == 1)
3310                                 queue_index = map->queues[0];
3311                         else
3312                                 queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
3313                                                                            map->len)];
3314                         if (unlikely(queue_index >= dev->real_num_tx_queues))
3315                                 queue_index = -1;
3316                 }
3317         }
3318         rcu_read_unlock();
3319 
3320         return queue_index;
3321 #else
3322         return -1;
3323 #endif
3324 }
3325 
3326 static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
3327 {
3328         struct sock *sk = skb->sk;
3329         int queue_index = sk_tx_queue_get(sk);
3330 
3331         if (queue_index < 0 || skb->ooo_okay ||
3332             queue_index >= dev->real_num_tx_queues) {
3333                 int new_index = get_xps_queue(dev, skb);
3334 
3335                 if (new_index < 0)
3336                         new_index = skb_tx_hash(dev, skb);
3337 
3338                 if (queue_index != new_index && sk &&
3339                     sk_fullsock(sk) &&
3340                     rcu_access_pointer(sk->sk_dst_cache))
3341                         sk_tx_queue_set(sk, new_index);
3342 
3343                 queue_index = new_index;
3344         }
3345 
3346         return queue_index;
3347 }
3348 
3349 struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3350                                     struct sk_buff *skb,
3351                                     void *accel_priv)
3352 {
3353         int queue_index = 0;
3354 
3355 #ifdef CONFIG_XPS
3356         u32 sender_cpu = skb->sender_cpu - 1;
3357 
3358         if (sender_cpu >= (u32)NR_CPUS)
3359                 skb->sender_cpu = raw_smp_processor_id() + 1;
3360 #endif
3361 
3362         if (dev->real_num_tx_queues != 1) {
3363                 const struct net_device_ops *ops = dev->netdev_ops;
3364 
3365                 if (ops->ndo_select_queue)
3366                         queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
3367                                                             __netdev_pick_tx);
3368                 else
3369                         queue_index = __netdev_pick_tx(dev, skb);
3370 
3371                 if (!accel_priv)
3372                         queue_index = netdev_cap_txqueue(dev, queue_index);
3373         }
3374 
3375         skb_set_queue_mapping(skb, queue_index);
3376         return netdev_get_tx_queue(dev, queue_index);
3377 }
3378 
3379 /**
3380  *      __dev_queue_xmit - transmit a buffer
3381  *      @skb: buffer to transmit
3382  *      @accel_priv: private data used for L2 forwarding offload
3383  *
3384  *      Queue a buffer for transmission to a network device. The caller must
3385  *      have set the device and priority and built the buffer before calling
3386  *      this function. The function can be called from an interrupt.
3387  *
3388  *      A negative errno code is returned on a failure. A success does not
3389  *      guarantee the frame will be transmitted as it may be dropped due
3390  *      to congestion or traffic shaping.
3391  *
3392  * -----------------------------------------------------------------------------------
3393  *      I notice this method can also return errors from the queue disciplines,
3394  *      including NET_XMIT_DROP, which is a positive value.  So, errors can also
3395  *      be positive.
3396  *
3397  *      Regardless of the return value, the skb is consumed, so it is currently
3398  *      difficult to retry a send to this method.  (You can bump the ref count
3399  *      before sending to hold a reference for retry if you are careful.)
3400  *
3401  *      When calling this method, interrupts MUST be enabled.  This is because
3402  *      the BH enable code must have IRQs enabled so that it will not deadlock.
3403  *          --BLG
3404  */
3405 static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
3406 {
3407         struct net_device *dev = skb->dev;
3408         struct netdev_queue *txq;
3409         struct Qdisc *q;
3410         int rc = -ENOMEM;
3411 
3412         skb_reset_mac_header(skb);
3413 
3414         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3415                 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3416 
3417         /* Disable soft irqs for various locks below. Also
3418          * stops preemption for RCU.
3419          */
3420         rcu_read_lock_bh();
3421 
3422         skb_update_prio(skb);
3423 
3424         qdisc_pkt_len_init(skb);
3425 #ifdef CONFIG_NET_CLS_ACT
3426         skb->tc_at_ingress = 0;
3427 # ifdef CONFIG_NET_EGRESS
3428         if (static_key_false(&egress_needed)) {
3429                 skb = sch_handle_egress(skb, &rc, dev);
3430                 if (!skb)
3431                         goto out;
3432         }
3433 # endif
3434 #endif
3435         /* If device/qdisc don't need skb->dst, release it right now while
3436          * its hot in this cpu cache.
3437          */
3438         if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3439                 skb_dst_drop(skb);
3440         else
3441                 skb_dst_force(skb);
3442 
3443         txq = netdev_pick_tx(dev, skb, accel_priv);
3444         q = rcu_dereference_bh(txq->qdisc);
3445 
3446         trace_net_dev_queue(skb);
3447         if (q->enqueue) {
3448                 rc = __dev_xmit_skb(skb, q, dev, txq);
3449                 goto out;
3450         }
3451 
3452         /* The device has no queue. Common case for software devices:
3453          * loopback, all the sorts of tunnels...
3454 
3455          * Really, it is unlikely that netif_tx_lock protection is necessary
3456          * here.  (f.e. loopback and IP tunnels are clean ignoring statistics
3457          * counters.)
3458          * However, it is possible, that they rely on protection
3459          * made by us here.
3460 
3461          * Check this and shot the lock. It is not prone from deadlocks.
3462          *Either shot noqueue qdisc, it is even simpler 8)
3463          */
3464         if (dev->flags & IFF_UP) {
3465                 int cpu = smp_processor_id(); /* ok because BHs are off */
3466 
3467                 if (txq->xmit_lock_owner != cpu) {
3468                         if (unlikely(__this_cpu_read(xmit_recursion) >
3469                                      XMIT_RECURSION_LIMIT))
3470                                 goto recursion_alert;
3471 
3472                         skb = validate_xmit_skb(skb, dev);
3473                         if (!skb)
3474                                 goto out;
3475 
3476                         HARD_TX_LOCK(dev, txq, cpu);
3477 
3478                         if (!netif_xmit_stopped(txq)) {
3479                                 __this_cpu_inc(xmit_recursion);
3480                                 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
3481                                 __this_cpu_dec(xmit_recursion);
3482                                 if (dev_xmit_complete(rc)) {
3483                                         HARD_TX_UNLOCK(dev, txq);
3484                                         goto out;
3485                                 }
3486                         }
3487                         HARD_TX_UNLOCK(dev, txq);
3488                         net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3489                                              dev->name);
3490                 } else {
3491                         /* Recursion is detected! It is possible,
3492                          * unfortunately
3493                          */
3494 recursion_alert:
3495                         net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3496                                              dev->name);
3497                 }
3498         }
3499 
3500         rc = -ENETDOWN;
3501         rcu_read_unlock_bh();
3502 
3503         atomic_long_inc(&dev->tx_dropped);
3504         kfree_skb_list(skb);
3505         return rc;
3506 out:
3507         rcu_read_unlock_bh();
3508         return rc;
3509 }
3510 
3511 int dev_queue_xmit(struct sk_buff *skb)
3512 {
3513         return __dev_queue_xmit(skb, NULL);
3514 }
3515 EXPORT_SYMBOL(dev_queue_xmit);
3516 
3517 int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
3518 {
3519         return __dev_queue_xmit(skb, accel_priv);
3520 }
3521 EXPORT_SYMBOL(dev_queue_xmit_accel);
3522 
3523 
3524 /*************************************************************************
3525  *                      Receiver routines
3526  *************************************************************************/
3527 
3528 int netdev_max_backlog __read_mostly = 1000;
3529 EXPORT_SYMBOL(netdev_max_backlog);
3530 
3531 int netdev_tstamp_prequeue __read_mostly = 1;
3532 int netdev_budget __read_mostly = 300;
3533 unsigned int __read_mostly netdev_budget_usecs = 2000;
3534 int weight_p __read_mostly = 64;           /* old backlog weight */
3535 int dev_weight_rx_bias __read_mostly = 1;  /* bias for backlog weight */
3536 int dev_weight_tx_bias __read_mostly = 1;  /* bias for output_queue quota */
3537 int dev_rx_weight __read_mostly = 64;
3538 int dev_tx_weight __read_mostly = 64;
3539 
3540 /* Called with irq disabled */
3541 static inline void ____napi_schedule(struct softnet_data *sd,
3542                                      struct napi_struct *napi)
3543 {
3544         list_add_tail(&napi->poll_list, &sd->poll_list);
3545         __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3546 }
3547 
3548 #ifdef CONFIG_RPS
3549 
3550 /* One global table that all flow-based protocols share. */
3551 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
3552 EXPORT_SYMBOL(rps_sock_flow_table);
3553 u32 rps_cpu_mask __read_mostly;
3554 EXPORT_SYMBOL(rps_cpu_mask);
3555 
3556 struct static_key rps_needed __read_mostly;
3557 EXPORT_SYMBOL(rps_needed);
3558 struct static_key rfs_needed __read_mostly;
3559 EXPORT_SYMBOL(rfs_needed);
3560 
3561 static struct rps_dev_flow *
3562 set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3563             struct rps_dev_flow *rflow, u16 next_cpu)
3564 {
3565         if (next_cpu < nr_cpu_ids) {
3566 #ifdef CONFIG_RFS_ACCEL
3567                 struct netdev_rx_queue *rxqueue;
3568                 struct rps_dev_flow_table *flow_table;
3569                 struct rps_dev_flow *old_rflow;
3570                 u32 flow_id;
3571                 u16 rxq_index;
3572                 int rc;
3573 
3574                 /* Should we steer this flow to a different hardware queue? */
3575                 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3576                     !(dev->features & NETIF_F_NTUPLE))
3577                         goto out;
3578                 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3579                 if (rxq_index == skb_get_rx_queue(skb))
3580                         goto out;
3581 
3582                 rxqueue = dev->_rx + rxq_index;
3583                 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3584                 if (!flow_table)
3585                         goto out;
3586                 flow_id = skb_get_hash(skb) & flow_table->mask;
3587                 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3588                                                         rxq_index, flow_id);
3589                 if (rc < 0)
3590                         goto out;
3591                 old_rflow = rflow;
3592                 rflow = &flow_table->flows[flow_id];
3593                 rflow->filter = rc;
3594                 if (old_rflow->filter == rflow->filter)
3595                         old_rflow->filter = RPS_NO_FILTER;
3596         out:
3597 #endif
3598                 rflow->last_qtail =
3599                         per_cpu(softnet_data, next_cpu).input_queue_head;
3600         }
3601 
3602         rflow->cpu = next_cpu;
3603         return rflow;
3604 }
3605 
3606 /*
3607  * get_rps_cpu is called from netif_receive_skb and returns the target
3608  * CPU from the RPS map of the receiving queue for a given skb.
3609  * rcu_read_lock must be held on entry.
3610  */
3611 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3612                        struct rps_dev_flow **rflowp)
3613 {
3614         const struct rps_sock_flow_table *sock_flow_table;
3615         struct netdev_rx_queue *rxqueue = dev->_rx;
3616         struct rps_dev_flow_table *flow_table;
3617         struct rps_map *map;
3618         int cpu = -1;
3619         u32 tcpu;
3620         u32 hash;
3621 
3622         if (skb_rx_queue_recorded(skb)) {
3623                 u16 index = skb_get_rx_queue(skb);
3624 
3625                 if (unlikely(index >= dev->real_num_rx_queues)) {
3626                         WARN_ONCE(dev->real_num_rx_queues > 1,
3627                                   "%s received packet on queue %u, but number "
3628                                   "of RX queues is %u\n",
3629                                   dev->name, index, dev->real_num_rx_queues);
3630                         goto done;
3631                 }
3632                 rxqueue += index;
3633         }
3634 
3635         /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
3636 
3637         flow_table = rcu_dereference(rxqueue->rps_flow_table);
3638         map = rcu_dereference(rxqueue->rps_map);
3639         if (!flow_table && !map)
3640                 goto done;
3641 
3642         skb_reset_network_header(skb);
3643         hash = skb_get_hash(skb);
3644         if (!hash)
3645                 goto done;
3646 
3647         sock_flow_table = rcu_dereference(rps_sock_flow_table);
3648         if (flow_table && sock_flow_table) {
3649                 struct rps_dev_flow *rflow;
3650                 u32 next_cpu;
3651                 u32 ident;
3652 
3653                 /* First check into global flow table if there is a match */
3654                 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
3655                 if ((ident ^ hash) & ~rps_cpu_mask)
3656                         goto try_rps;
3657 
3658                 next_cpu = ident & rps_cpu_mask;
3659 
3660                 /* OK, now we know there is a match,
3661                  * we can look at the local (per receive queue) flow table
3662                  */
3663                 rflow = &flow_table->flows[hash & flow_table->mask];
3664                 tcpu = rflow->cpu;
3665 
3666                 /*
3667                  * If the desired CPU (where last recvmsg was done) is
3668                  * different from current CPU (one in the rx-queue flow
3669                  * table entry), switch if one of the following holds:
3670                  *   - Current CPU is unset (>= nr_cpu_ids).
3671                  *   - Current CPU is offline.
3672                  *   - The current CPU's queue tail has advanced beyond the
3673                  *     last packet that was enqueued using this table entry.
3674                  *     This guarantees that all previous packets for the flow
3675                  *     have been dequeued, thus preserving in order delivery.
3676                  */
3677                 if (unlikely(tcpu != next_cpu) &&
3678                     (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
3679                      ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
3680                       rflow->last_qtail)) >= 0)) {
3681                         tcpu = next_cpu;
3682                         rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
3683                 }
3684 
3685                 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
3686                         *rflowp = rflow;
3687                         cpu = tcpu;
3688                         goto done;
3689                 }
3690         }
3691 
3692 try_rps:
3693 
3694         if (map) {
3695                 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
3696                 if (cpu_online(tcpu)) {
3697                         cpu = tcpu;
3698                         goto done;
3699                 }
3700         }
3701 
3702 done:
3703         return cpu;
3704 }
3705 
3706 #ifdef CONFIG_RFS_ACCEL
3707 
3708 /**
3709  * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3710  * @dev: Device on which the filter was set
3711  * @rxq_index: RX queue index
3712  * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3713  * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3714  *
3715  * Drivers that implement ndo_rx_flow_steer() should periodically call
3716  * this function for each installed filter and remove the filters for
3717  * which it returns %true.
3718  */
3719 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3720                          u32 flow_id, u16 filter_id)
3721 {
3722         struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3723         struct rps_dev_flow_table *flow_table;
3724         struct rps_dev_flow *rflow;
3725         bool expire = true;
3726         unsigned int cpu;
3727 
3728         rcu_read_lock();
3729         flow_table = rcu_dereference(rxqueue->rps_flow_table);
3730         if (flow_table && flow_id <= flow_table->mask) {
3731                 rflow = &flow_table->flows[flow_id];
3732                 cpu = ACCESS_ONCE(rflow->cpu);
3733                 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
3734                     ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3735                            rflow->last_qtail) <
3736                      (int)(10 * flow_table->mask)))
3737                         expire = false;
3738         }
3739         rcu_read_unlock();
3740         return expire;
3741 }
3742 EXPORT_SYMBOL(rps_may_expire_flow);
3743 
3744 #endif /* CONFIG_RFS_ACCEL */
3745 
3746 /* Called from hardirq (IPI) context */
3747 static void rps_trigger_softirq(void *data)
3748 {
3749         struct softnet_data *sd = data;
3750 
3751         ____napi_schedule(sd, &sd->backlog);
3752         sd->received_rps++;
3753 }
3754 
3755 #endif /* CONFIG_RPS */
3756 
3757 /*
3758  * Check if this softnet_data structure is another cpu one
3759  * If yes, queue it to our IPI list and return 1
3760  * If no, return 0
3761  */
3762 static int rps_ipi_queued(struct softnet_data *sd)
3763 {
3764 #ifdef CONFIG_RPS
3765         struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
3766 
3767         if (sd != mysd) {
3768                 sd->rps_ipi_next = mysd->rps_ipi_list;
3769                 mysd->rps_ipi_list = sd;
3770 
3771                 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3772                 return 1;
3773         }
3774 #endif /* CONFIG_RPS */
3775         return 0;
3776 }
3777 
3778 #ifdef CONFIG_NET_FLOW_LIMIT
3779 int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3780 #endif
3781 
3782 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3783 {
3784 #ifdef CONFIG_NET_FLOW_LIMIT
3785         struct sd_flow_limit *fl;
3786         struct softnet_data *sd;
3787         unsigned int old_flow, new_flow;
3788 
3789         if (qlen < (netdev_max_backlog >> 1))
3790                 return false;
3791 
3792         sd = this_cpu_ptr(&softnet_data);
3793 
3794         rcu_read_lock();
3795         fl = rcu_dereference(sd->flow_limit);
3796         if (fl) {
3797                 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
3798                 old_flow = fl->history[fl->history_head];
3799                 fl->history[fl->history_head] = new_flow;
3800 
3801                 fl->history_head++;
3802                 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3803 
3804                 if (likely(fl->buckets[old_flow]))
3805                         fl->buckets[old_flow]--;
3806 
3807                 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3808                         fl->count++;
3809                         rcu_read_unlock();
3810                         return true;
3811                 }
3812         }
3813         rcu_read_unlock();
3814 #endif
3815         return false;
3816 }
3817 
3818 /*
3819  * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3820  * queue (may be a remote CPU queue).
3821  */
3822 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3823                               unsigned int *qtail)
3824 {
3825         struct softnet_data *sd;
3826         unsigned long flags;
3827         unsigned int qlen;
3828 
3829         sd = &per_cpu(softnet_data, cpu);
3830 
3831         local_irq_save(flags);
3832 
3833         rps_lock(sd);
3834         if (!netif_running(skb->dev))
3835                 goto drop;
3836         qlen = skb_queue_len(&sd->input_pkt_queue);
3837         if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
3838                 if (qlen) {
3839 enqueue:
3840                         __skb_queue_tail(&sd->input_pkt_queue, skb);
3841                         input_queue_tail_incr_save(sd, qtail);
3842                         rps_unlock(sd);
3843                         local_irq_restore(flags);
3844                         return NET_RX_SUCCESS;
3845                 }
3846 
3847                 /* Schedule NAPI for backlog device
3848                  * We can use non atomic operation since we own the queue lock
3849                  */
3850                 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
3851                         if (!rps_ipi_queued(sd))
3852                                 ____napi_schedule(sd, &sd->backlog);
3853                 }
3854                 goto enqueue;
3855         }
3856 
3857 drop:
3858         sd->dropped++;
3859         rps_unlock(sd);
3860 
3861         local_irq_restore(flags);
3862 
3863         atomic_long_inc(&skb->dev->rx_dropped);
3864         kfree_skb(skb);
3865         return NET_RX_DROP;
3866 }
3867 
3868 static int netif_rx_internal(struct sk_buff *skb)
3869 {
3870         int ret;
3871 
3872         net_timestamp_check(netdev_tstamp_prequeue, skb);
3873 
3874         trace_netif_rx(skb);
3875 #ifdef CONFIG_RPS
3876         if (static_key_false(&rps_needed)) {
3877                 struct rps_dev_flow voidflow, *rflow = &voidflow;
3878                 int cpu;
3879 
3880                 preempt_disable();
3881                 rcu_read_lock();
3882 
3883                 cpu = get_rps_cpu(skb->dev, skb, &rflow);
3884                 if (cpu < 0)
3885                         cpu = smp_processor_id();
3886 
3887                 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3888 
3889                 rcu_read_unlock();
3890                 preempt_enable();
3891         } else
3892 #endif
3893         {
3894                 unsigned int qtail;
3895 
3896                 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3897                 put_cpu();
3898         }
3899         return ret;
3900 }
3901 
3902 /**
3903  *      netif_rx        -       post buffer to the network code
3904  *      @skb: buffer to post
3905  *
3906  *      This function receives a packet from a device driver and queues it for
3907  *      the upper (protocol) levels to process.  It always succeeds. The buffer
3908  *      may be dropped during processing for congestion control or by the
3909  *      protocol layers.
3910  *
3911  *      return values:
3912  *      NET_RX_SUCCESS  (no congestion)
3913  *      NET_RX_DROP     (packet was dropped)
3914  *
3915  */
3916 
3917 int netif_rx(struct sk_buff *skb)
3918 {
3919         trace_netif_rx_entry(skb);
3920 
3921         return netif_rx_internal(skb);
3922 }
3923 EXPORT_SYMBOL(netif_rx);
3924 
3925 int netif_rx_ni(struct sk_buff *skb)
3926 {
3927         int err;
3928 
3929         trace_netif_rx_ni_entry(skb);
3930 
3931         preempt_disable();
3932         err = netif_rx_internal(skb);
3933         if (local_softirq_pending())
3934                 do_softirq();
3935         preempt_enable();
3936 
3937         return err;
3938 }
3939 EXPORT_SYMBOL(netif_rx_ni);
3940 
3941 static __latent_entropy void net_tx_action(struct softirq_action *h)
3942 {
3943         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
3944 
3945         if (sd->completion_queue) {
3946                 struct sk_buff *clist;
3947 
3948                 local_irq_disable();
3949                 clist = sd->completion_queue;
3950                 sd->completion_queue = NULL;
3951                 local_irq_enable();
3952 
3953                 while (clist) {
3954                         struct sk_buff *skb = clist;
3955 
3956                         clist = clist->next;
3957 
3958                         WARN_ON(refcount_read(&skb->users));
3959                         if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
3960                                 trace_consume_skb(skb);
3961                         else
3962                                 trace_kfree_skb(skb, net_tx_action);
3963 
3964                         if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
3965                                 __kfree_skb(skb);
3966                         else
3967                                 __kfree_skb_defer(skb);
3968                 }
3969 
3970                 __kfree_skb_flush();
3971         }
3972 
3973         if (sd->output_queue) {
3974                 struct Qdisc *head;
3975 
3976                 local_irq_disable();
3977                 head = sd->output_queue;
3978                 sd->output_queue = NULL;
3979                 sd->output_queue_tailp = &sd->output_queue;
3980                 local_irq_enable();
3981 
3982                 while (head) {
3983                         struct Qdisc *q = head;
3984                         spinlock_t *root_lock;
3985 
3986                         head = head->next_sched;
3987 
3988                         root_lock = qdisc_lock(q);
3989                         spin_lock(root_lock);
3990                         /* We need to make sure head->next_sched is read
3991                          * before clearing __QDISC_STATE_SCHED
3992                          */
3993                         smp_mb__before_atomic();
3994                         clear_bit(__QDISC_STATE_SCHED, &q->state);
3995                         qdisc_run(q);
3996                         spin_unlock(root_lock);
3997                 }
3998         }
3999 }
4000 
4001 #if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
4002 /* This hook is defined here for ATM LANE */
4003 int (*br_fdb_test_addr_hook)(struct net_device *dev,
4004                              unsigned char *addr) __read_mostly;
4005 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
4006 #endif
4007 
4008 static inline struct sk_buff *
4009 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
4010                    struct net_device *orig_dev)
4011 {
4012 #ifdef CONFIG_NET_CLS_ACT
4013         struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list);
4014         struct tcf_result cl_res;
4015 
4016         /* If there's at least one ingress present somewhere (so
4017          * we get here via enabled static key), remaining devices
4018          * that are not configured with an ingress qdisc will bail
4019          * out here.
4020          */
4021         if (!cl)
4022                 return skb;
4023         if (*pt_prev) {
4024                 *ret = deliver_skb(skb, *pt_prev, orig_dev);
4025                 *pt_prev = NULL;
4026         }
4027 
4028         qdisc_skb_cb(skb)->pkt_len = skb->len;
4029         skb->tc_at_ingress = 1;
4030         qdisc_bstats_cpu_update(cl->q, skb);
4031 
4032         switch (tcf_classify(skb, cl, &cl_res, false)) {
4033         case TC_ACT_OK:
4034         case TC_ACT_RECLASSIFY:
4035                 skb->tc_index = TC_H_MIN(cl_res.classid);
4036                 break;
4037         case TC_ACT_SHOT:
4038                 qdisc_qstats_cpu_drop(cl->q);
4039                 kfree_skb(skb);
4040                 return NULL;
4041         case TC_ACT_STOLEN:
4042         case TC_ACT_QUEUED:
4043         case TC_ACT_TRAP:
4044                 consume_skb(skb);
4045                 return NULL;
4046         case TC_ACT_REDIRECT:
4047                 /* skb_mac_header check was done by cls/act_bpf, so
4048                  * we can safely push the L2 header back before
4049                  * redirecting to another netdev
4050                  */
4051                 __skb_push(skb, skb->mac_len);
4052                 skb_do_redirect(skb);
4053                 return NULL;
4054         default:
4055                 break;
4056         }
4057 #endif /* CONFIG_NET_CLS_ACT */
4058         return skb;
4059 }
4060 
4061 /**
4062  *      netdev_is_rx_handler_busy - check if receive handler is registered
4063  *      @dev: device to check
4064  *
4065  *      Check if a receive handler is already registered for a given device.
4066  *      Return true if there one.
4067  *
4068  *      The caller must hold the rtnl_mutex.
4069  */
4070 bool netdev_is_rx_handler_busy(struct net_device *dev)
4071 {
4072         ASSERT_RTNL();
4073         return dev && rtnl_dereference(dev->rx_handler);
4074 }
4075 EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
4076 
4077 /**
4078  *      netdev_rx_handler_register - register receive handler
4079  *      @dev: device to register a handler for
4080  *      @rx_handler: receive handler to register
4081  *      @rx_handler_data: data pointer that is used by rx handler
4082  *
4083  *      Register a receive handler for a device. This handler will then be
4084  *      called from __netif_receive_skb. A negative errno code is returned
4085  *      on a failure.
4086  *
4087  *      The caller must hold the rtnl_mutex.
4088  *
4089  *      For a general description of rx_handler, see enum rx_handler_result.
4090  */
4091 int netdev_rx_handler_register(struct net_device *dev,
4092                                rx_handler_func_t *rx_handler,
4093                                void *rx_handler_data)
4094 {
4095         if (netdev_is_rx_handler_busy(dev))
4096                 return -EBUSY;
4097 
4098         /* Note: rx_handler_data must be set before rx_handler */
4099         rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
4100         rcu_assign_pointer(dev->rx_handler, rx_handler);
4101 
4102         return 0;
4103 }
4104 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
4105 
4106 /**
4107  *      netdev_rx_handler_unregister - unregister receive handler
4108  *      @dev: device to unregister a handler from
4109  *
4110  *      Unregister a receive handler from a device.
4111  *
4112  *      The caller must hold the rtnl_mutex.
4113  */
4114 void netdev_rx_handler_unregister(struct net_device *dev)
4115 {
4116 
4117         ASSERT_RTNL();
4118         RCU_INIT_POINTER(dev->rx_handler, NULL);
4119         /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
4120          * section has a guarantee to see a non NULL rx_handler_data
4121          * as well.
4122          */
4123         synchronize_net();
4124         RCU_INIT_POINTER(dev->rx_handler_data, NULL);
4125 }
4126 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
4127 
4128 /*
4129  * Limit the use of PFMEMALLOC reserves to those protocols that implement
4130  * the special handling of PFMEMALLOC skbs.
4131  */
4132 static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
4133 {
4134         switch (skb->protocol) {
4135         case htons(ETH_P_ARP):
4136         case htons(ETH_P_IP):
4137         case htons(ETH_P_IPV6):
4138         case htons(ETH_P_8021Q):
4139         case htons(ETH_P_8021AD):
4140                 return true;
4141         default:
4142                 return false;
4143         }
4144 }
4145 
4146 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
4147                              int *ret, struct net_device *orig_dev)
4148 {
4149 #ifdef CONFIG_NETFILTER_INGRESS
4150         if (nf_hook_ingress_active(skb)) {
4151                 int ingress_retval;
4152 
4153                 if (*pt_prev) {
4154                         *ret = deliver_skb(skb, *pt_prev, orig_dev);
4155                         *pt_prev = NULL;
4156                 }
4157 
4158                 rcu_read_lock();
4159                 ingress_retval = nf_hook_ingress(skb);
4160                 rcu_read_unlock();
4161                 return ingress_retval;
4162         }
4163 #endif /* CONFIG_NETFILTER_INGRESS */
4164         return 0;
4165 }
4166 
4167 static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
4168 {
4169         struct packet_type *ptype, *pt_prev;
4170         rx_handler_func_t *rx_handler;
4171         struct net_device *orig_dev;
4172         bool deliver_exact = false;
4173         int ret = NET_RX_DROP;
4174         __be16 type;
4175 
4176         net_timestamp_check(!netdev_tstamp_prequeue, skb);
4177 
4178         trace_netif_receive_skb(skb);
4179 
4180         orig_dev = skb->dev;
4181 
4182         skb_reset_network_header(skb);
4183         if (!skb_transport_header_was_set(skb))
4184                 skb_reset_transport_header(skb);
4185         skb_reset_mac_len(skb);
4186 
4187         pt_prev = NULL;
4188 
4189 another_round:
4190         skb->skb_iif = skb->dev->ifindex;
4191 
4192         __this_cpu_inc(softnet_data.processed);
4193 
4194         if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
4195             skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
4196                 skb = skb_vlan_untag(skb);
4197                 if (unlikely(!skb))
4198                         goto out;
4199         }
4200 
4201         if (skb_skip_tc_classify(skb))
4202                 goto skip_classify;
4203 
4204         if (pfmemalloc)
4205                 goto skip_taps;
4206 
4207         list_for_each_entry_rcu(ptype, &ptype_all, list) {
4208                 if (pt_prev)
4209                         ret = deliver_skb(skb, pt_prev, orig_dev);
4210                 pt_prev = ptype;
4211         }
4212 
4213         list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
4214                 if (pt_prev)
4215                         ret = deliver_skb(skb, pt_prev, orig_dev);
4216                 pt_prev = ptype;
4217         }
4218 
4219 skip_taps:
4220 #ifdef CONFIG_NET_INGRESS
4221         if (static_key_false(&ingress_needed)) {
4222                 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev);
4223                 if (!skb)
4224                         goto out;
4225 
4226                 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
4227                         goto out;
4228         }
4229 #endif
4230         skb_reset_tc(skb);
4231 skip_classify:
4232         if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
4233                 goto drop;
4234 
4235         if (skb_vlan_tag_present(skb)) {
4236                 if (pt_prev) {
4237                         ret = deliver_skb(skb, pt_prev, orig_dev);
4238                         pt_prev = NULL;
4239                 }
4240                 if (vlan_do_receive(&skb))
4241                         goto another_round;
4242                 else if (unlikely(!skb))
4243                         goto out;
4244         }
4245 
4246         rx_handler = rcu_dereference(skb->dev->rx_handler);
4247         if (rx_handler) {
4248                 if (pt_prev) {
4249                         ret = deliver_skb(skb, pt_prev, orig_dev);
4250                         pt_prev = NULL;
4251                 }
4252                 switch (rx_handler(&skb)) {
4253                 case RX_HANDLER_CONSUMED:
4254                         ret = NET_RX_SUCCESS;
4255                         goto out;
4256                 case RX_HANDLER_ANOTHER:
4257                         goto another_round;
4258                 case RX_HANDLER_EXACT:
4259                         deliver_exact = true;
4260                 case RX_HANDLER_PASS:
4261                         break;
4262                 default:
4263                         BUG();
4264                 }
4265         }
4266 
4267         if (unlikely(skb_vlan_tag_present(skb))) {
4268                 if (skb_vlan_tag_get_id(skb))
4269                         skb->pkt_type = PACKET_OTHERHOST;
4270                 /* Note: we might in the future use prio bits
4271                  * and set skb->priority like in vlan_do_receive()
4272                  * For the time being, just ignore Priority Code Point
4273                  */
4274                 skb->vlan_tci = 0;
4275         }
4276 
4277         type = skb->protocol;
4278 
4279         /* deliver only exact match when indicated */
4280         if (likely(!deliver_exact)) {
4281                 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4282                                        &ptype_base[ntohs(type) &
4283                                                    PTYPE_HASH_MASK]);
4284         }
4285 
4286         deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4287                                &orig_dev->ptype_specific);
4288 
4289         if (unlikely(skb->dev != orig_dev)) {
4290                 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4291                                        &skb->dev->ptype_specific);
4292         }
4293 
4294         if (pt_prev) {
4295                 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
4296                         goto drop;
4297                 else
4298                         ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
4299         } else {
4300 drop:
4301                 if (!deliver_exact)
4302                         atomic_long_inc(&skb->dev->rx_dropped);
4303                 else
4304                         atomic_long_inc(&skb->dev->rx_nohandler);
4305                 kfree_skb(skb);
4306                 /* Jamal, now you will not able to escape explaining
4307                  * me how you were going to use this. :-)
4308                  */
4309                 ret = NET_RX_DROP;
4310         }
4311 
4312 out:
4313         return ret;
4314 }
4315 
4316 static int __netif_receive_skb(struct sk_buff *skb)
4317 {
4318         int ret;
4319 
4320         if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
4321                 unsigned int noreclaim_flag;
4322 
4323                 /*
4324                  * PFMEMALLOC skbs are special, they should
4325                  * - be delivered to SOCK_MEMALLOC sockets only
4326                  * - stay away from userspace
4327                  * - have bounded memory usage
4328                  *
4329                  * Use PF_MEMALLOC as this saves us from propagating the allocation
4330                  * context down to all allocation sites.
4331                  */
4332                 noreclaim_flag = memalloc_noreclaim_save();
4333                 ret = __netif_receive_skb_core(skb, true);
4334                 memalloc_noreclaim_restore(noreclaim_flag);
4335         } else
4336                 ret = __netif_receive_skb_core(skb, false);
4337 
4338         return ret;
4339 }
4340 
4341 static struct static_key generic_xdp_needed __read_mostly;
4342 
4343 static int generic_xdp_install(struct net_device *dev, struct netdev_xdp *xdp)
4344 {
4345         struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
4346         struct bpf_prog *new = xdp->prog;
4347         int ret = 0;
4348 
4349         switch (xdp->command) {
4350         case XDP_SETUP_PROG:
4351                 rcu_assign_pointer(dev->xdp_prog, new);
4352                 if (old)
4353                         bpf_prog_put(old);
4354 
4355                 if (old && !new) {
4356                         static_key_slow_dec(&generic_xdp_needed);
4357                 } else if (new && !old) {
4358                         static_key_slow_inc(&generic_xdp_needed);
4359                         dev_disable_lro(dev);
4360                 }
4361                 break;
4362 
4363         case XDP_QUERY_PROG:
4364                 xdp->prog_attached = !!old;
4365                 xdp->prog_id = old ? old->aux->id : 0;
4366                 break;
4367 
4368         default:
4369                 ret = -EINVAL;
4370                 break;
4371         }
4372 
4373         return ret;
4374 }
4375 
4376 static u32 netif_receive_generic_xdp(struct sk_buff *skb,
4377                                      struct bpf_prog *xdp_prog)
4378 {
4379         struct xdp_buff xdp;
4380         u32 act = XDP_DROP;
4381         void *orig_data;
4382         int hlen, off;
4383         u32 mac_len;
4384 
4385         /* Reinjected packets coming from act_mirred or similar should
4386          * not get XDP generic processing.
4387          */
4388         if (skb_cloned(skb))
4389                 return XDP_PASS;
4390 
4391         if (skb_linearize(skb))
4392                 goto do_drop;
4393 
4394         /* The XDP program wants to see the packet starting at the MAC
4395          * header.
4396          */
4397         mac_len = skb->data - skb_mac_header(skb);
4398         hlen = skb_headlen(skb) + mac_len;
4399         xdp.data = skb->data - mac_len;
4400         xdp.data_end = xdp.data + hlen;
4401         xdp.data_hard_start = skb->data - skb_headroom(skb);
4402         orig_data = xdp.data;
4403 
4404         act = bpf_prog_run_xdp(xdp_prog, &xdp);
4405 
4406         off = xdp.data - orig_data;
4407         if (off > 0)
4408                 __skb_pull(skb, off);
4409         else if (off < 0)
4410                 __skb_push(skb, -off);
4411         skb->mac_header += off;
4412 
4413         switch (act) {
4414         case XDP_TX:
4415                 __skb_push(skb, mac_len);
4416                 /* fall through */
4417         case XDP_PASS:
4418                 break;
4419 
4420         default:
4421                 bpf_warn_invalid_xdp_action(act);
4422                 /* fall through */
4423         case XDP_ABORTED:
4424                 trace_xdp_exception(skb->dev, xdp_prog, act);
4425                 /* fall through */
4426         case XDP_DROP:
4427         do_drop:
4428                 kfree_skb(skb);
4429                 break;
4430         }
4431 
4432         return act;
4433 }
4434 
4435 /* When doing generic XDP we have to bypass the qdisc layer and the
4436  * network taps in order to match in-driver-XDP behavior.
4437  */
4438 static void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
4439 {
4440         struct net_device *dev = skb->dev;
4441         struct netdev_queue *txq;
4442         bool free_skb = true;
4443         int cpu, rc;
4444 
4445         txq = netdev_pick_tx(dev, skb, NULL);
4446         cpu = smp_processor_id();
4447         HARD_TX_LOCK(dev, txq, cpu);
4448         if (!netif_xmit_stopped(txq)) {
4449                 rc = netdev_start_xmit(skb, dev, txq, 0);
4450                 if (dev_xmit_complete(rc))
4451                         free_skb = false;
4452         }
4453         HARD_TX_UNLOCK(dev, txq);
4454         if (free_skb) {
4455                 trace_xdp_exception(dev, xdp_prog, XDP_TX);
4456                 kfree_skb(skb);
4457         }
4458 }
4459 
4460 static int netif_receive_skb_internal(struct sk_buff *skb)
4461 {
4462         int ret;
4463 
4464         net_timestamp_check(netdev_tstamp_prequeue, skb);
4465 
4466         if (skb_defer_rx_timestamp(skb))
4467                 return NET_RX_SUCCESS;
4468 
4469         rcu_read_lock();
4470 
4471         if (static_key_false(&generic_xdp_needed)) {
4472                 struct bpf_prog *xdp_prog = rcu_dereference(skb->dev->xdp_prog);
4473 
4474                 if (xdp_prog) {
4475                         u32 act = netif_receive_generic_xdp(skb, xdp_prog);
4476 
4477                         if (act != XDP_PASS) {
4478                                 rcu_read_unlock();
4479                                 if (act == XDP_TX)
4480                                         generic_xdp_tx(skb, xdp_prog);
4481                                 return NET_RX_DROP;
4482                         }
4483                 }
4484         }
4485 
4486 #ifdef CONFIG_RPS
4487         if (static_key_false(&rps_needed)) {
4488                 struct rps_dev_flow voidflow, *rflow = &voidflow;
4489                 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
4490 
4491                 if (cpu >= 0) {
4492                         ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4493                         rcu_read_unlock();
4494                         return ret;
4495                 }
4496         }
4497 #endif
4498         ret = __netif_receive_skb(skb);
4499         rcu_read_unlock();
4500         return ret;
4501 }
4502 
4503 /**
4504  *      netif_receive_skb - process receive buffer from network
4505  *      @skb: buffer to process
4506  *
4507  *      netif_receive_skb() is the main receive data processing function.
4508  *      It always succeeds. The buffer may be dropped during processing
4509  *      for congestion control or by the protocol layers.
4510  *
4511  *      This function may only be called from softirq context and interrupts
4512  *      should be enabled.
4513  *
4514  *      Return values (usually ignored):
4515  *      NET_RX_SUCCESS: no congestion
4516  *      NET_RX_DROP: packet was dropped
4517  */
4518 int netif_receive_skb(struct sk_buff *skb)
4519 {
4520         trace_netif_receive_skb_entry(skb);
4521 
4522         return netif_receive_skb_internal(skb);
4523 }
4524 EXPORT_SYMBOL(netif_receive_skb);
4525 
4526 DEFINE_PER_CPU(struct work_struct, flush_works);
4527 
4528 /* Network device is going away, flush any packets still pending */
4529 static void flush_backlog(struct work_struct *work)
4530 {
4531         struct sk_buff *skb, *tmp;
4532         struct softnet_data *sd;
4533 
4534         local_bh_disable();
4535         sd = this_cpu_ptr(&softnet_data);
4536 
4537         local_irq_disable();
4538         rps_lock(sd);
4539         skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
4540                 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
4541                         __skb_unlink(skb, &sd->input_pkt_queue);
4542                         kfree_skb(skb);
4543                         input_queue_head_incr(sd);
4544                 }
4545         }
4546         rps_unlock(sd);
4547         local_irq_enable();
4548 
4549         skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
4550                 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
4551                         __skb_unlink(skb, &sd->process_queue);
4552                         kfree_skb(skb);
4553                         input_queue_head_incr(sd);
4554                 }
4555         }
4556         local_bh_enable();
4557 }
4558 
4559 static void flush_all_backlogs(void)
4560 {
4561         unsigned int cpu;
4562 
4563         get_online_cpus();
4564 
4565         for_each_online_cpu(cpu)
4566                 queue_work_on(cpu, system_highpri_wq,
4567                               per_cpu_ptr(&flush_works, cpu));
4568 
4569         for_each_online_cpu(cpu)
4570                 flush_work(per_cpu_ptr(&flush_works, cpu));
4571 
4572         put_online_cpus();
4573 }
4574 
4575 static int napi_gro_complete(struct sk_buff *skb)
4576 {
4577         struct packet_offload *ptype;
4578         __be16 type = skb->protocol;
4579         struct list_head *head = &offload_base;
4580         int err = -ENOENT;
4581 
4582         BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
4583 
4584         if (NAPI_GRO_CB(skb)->count == 1) {
4585                 skb_shinfo(skb)->gso_size = 0;
4586                 goto out;
4587         }
4588 
4589         rcu_read_lock();
4590         list_for_each_entry_rcu(ptype, head, list) {
4591                 if (ptype->type != type || !ptype->callbacks.gro_complete)
4592                         continue;
4593 
4594                 err = ptype->callbacks.gro_complete(skb, 0);
4595                 break;
4596         }
4597         rcu_read_unlock();
4598 
4599         if (err) {
4600                 WARN_ON(&ptype->list == head);
4601                 kfree_skb(skb);
4602                 return NET_RX_SUCCESS;
4603         }
4604 
4605 out:
4606         return netif_receive_skb_internal(skb);
4607 }
4608 
4609 /* napi->gro_list contains packets ordered by age.
4610  * youngest packets at the head of it.
4611  * Complete skbs in reverse order to reduce latencies.
4612  */
4613 void napi_gro_flush(struct napi_struct *napi, bool flush_old)
4614 {
4615         struct sk_buff *skb, *prev = NULL;
4616 
4617         /* scan list and build reverse chain */
4618         for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
4619                 skb->prev = prev;
4620                 prev = skb;
4621         }
4622 
4623         for (skb = prev; skb; skb = prev) {
4624                 skb->next = NULL;
4625 
4626                 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
4627                         return;
4628 
4629                 prev = skb->prev;
4630                 napi_gro_complete(skb);
4631                 napi->gro_count--;
4632         }
4633 
4634         napi->gro_list = NULL;
4635 }
4636 EXPORT_SYMBOL(napi_gro_flush);
4637 
4638 static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
4639 {
4640         struct sk_buff *p;
4641         unsigned int maclen = skb->dev->hard_header_len;
4642         u32 hash = skb_get_hash_raw(skb);
4643 
4644         for (p = napi->gro_list; p; p = p->next) {
4645                 unsigned long diffs;
4646 
4647                 NAPI_GRO_CB(p)->flush = 0;
4648 
4649                 if (hash != skb_get_hash_raw(p)) {
4650                         NAPI_GRO_CB(p)->same_flow = 0;
4651                         continue;
4652                 }
4653 
4654                 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
4655                 diffs |= p->vlan_tci ^ skb->vlan_tci;
4656                 diffs |= skb_metadata_dst_cmp(p, skb);
4657                 if (maclen == ETH_HLEN)
4658                         diffs |= compare_ether_header(skb_mac_header(p),
4659                                                       skb_mac_header(skb));
4660                 else if (!diffs)
4661                         diffs = memcmp(skb_mac_header(p),
4662                                        skb_mac_header(skb),
4663                                        maclen);
4664                 NAPI_GRO_CB(p)->same_flow = !diffs;
4665         }
4666 }
4667 
4668 static void skb_gro_reset_offset(struct sk_buff *skb)
4669 {
4670         const struct skb_shared_info *pinfo = skb_shinfo(skb);
4671         const skb_frag_t *frag0 = &pinfo->frags[0];
4672 
4673         NAPI_GRO_CB(skb)->data_offset = 0;
4674         NAPI_GRO_CB(skb)->frag0 = NULL;
4675         NAPI_GRO_CB(skb)->frag0_len = 0;
4676 
4677         if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
4678             pinfo->nr_frags &&
4679             !PageHighMem(skb_frag_page(frag0))) {
4680                 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
4681                 NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
4682                                                     skb_frag_size(frag0),
4683                                                     skb->end - skb->tail);
4684         }
4685 }
4686 
4687 static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
4688 {
4689         struct skb_shared_info *pinfo = skb_shinfo(skb);
4690 
4691         BUG_ON(skb->end - skb->tail < grow);
4692 
4693         memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
4694 
4695         skb->data_len -= grow;
4696         skb->tail += grow;
4697 
4698         pinfo->frags[0].page_offset += grow;
4699         skb_frag_size_sub(&pinfo->frags[0], grow);
4700 
4701         if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
4702                 skb_frag_unref(skb, 0);
4703                 memmove(pinfo->frags, pinfo->frags + 1,
4704                         --pinfo->nr_frags * sizeof(pinfo->frags[0]));
4705         }
4706 }
4707 
4708 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4709 {
4710         struct sk_buff **pp = NULL;
4711         struct packet_offload *ptype;
4712         __be16 type = skb->protocol;
4713         struct list_head *head = &offload_base;
4714         int same_flow;
4715         enum gro_result ret;
4716         int grow;
4717 
4718         if (netif_elide_gro(skb->dev))
4719                 goto normal;
4720 
4721         gro_list_prepare(napi, skb);
4722 
4723         rcu_read_lock();
4724         list_for_each_entry_rcu(ptype, head, list) {
4725                 if (ptype->type != type || !ptype->callbacks.gro_receive)
4726                         continue;
4727 
4728                 skb_set_network_header(skb, skb_gro_offset(skb));
4729                 skb_reset_mac_len(skb);
4730                 NAPI_GRO_CB(skb)->same_flow = 0;
4731                 NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb);
4732                 NAPI_GRO_CB(skb)->free = 0;
4733                 NAPI_GRO_CB(skb)->encap_mark = 0;
4734                 NAPI_GRO_CB(skb)->recursion_counter = 0;
4735                 NAPI_GRO_CB(skb)->is_fou = 0;
4736                 NAPI_GRO_CB(skb)->is_atomic = 1;
4737                 NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
4738 
4739                 /* Setup for GRO checksum validation */
4740                 switch (skb->ip_summed) {
4741                 case CHECKSUM_COMPLETE:
4742                         NAPI_GRO_CB(skb)->csum = skb->csum;
4743                         NAPI_GRO_CB(skb)->csum_valid = 1;
4744                         NAPI_GRO_CB(skb)->csum_cnt = 0;
4745                         break;
4746                 case CHECKSUM_UNNECESSARY:
4747                         NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
4748                         NAPI_GRO_CB(skb)->csum_valid = 0;
4749                         break;
4750                 default:
4751                         NAPI_GRO_CB(skb)->csum_cnt = 0;
4752                         NAPI_GRO_CB(skb)->csum_valid = 0;
4753                 }
4754 
4755                 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
4756                 break;
4757         }
4758         rcu_read_unlock();
4759 
4760         if (&ptype->list == head)
4761                 goto normal;
4762 
4763         if (IS_ERR(pp) && PTR_ERR(pp) == -EINPROGRESS) {
4764                 ret = GRO_CONSUMED;
4765                 goto ok;
4766         }
4767 
4768         same_flow = NAPI_GRO_CB(skb)->same_flow;
4769         ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
4770 
4771         if (pp) {
4772                 struct sk_buff *nskb = *pp;
4773 
4774                 *pp = nskb->next;
4775                 nskb->next = NULL;
4776                 napi_gro_complete(nskb);
4777                 napi->gro_count--;
4778         }
4779 
4780         if (same_flow)
4781                 goto ok;
4782 
4783         if (NAPI_GRO_CB(skb)->flush)
4784                 goto normal;
4785 
4786         if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
4787                 struct sk_buff *nskb = napi->gro_list;
4788 
4789                 /* locate the end of the list to select the 'oldest' flow */
4790                 while (nskb->next) {
4791                         pp = &nskb->next;
4792                         nskb = *pp;
4793                 }
4794                 *pp = NULL;
4795                 nskb->next = NULL;
4796                 napi_gro_complete(nskb);
4797         } else {
4798                 napi->gro_count++;
4799         }
4800         NAPI_GRO_CB(skb)->count = 1;
4801         NAPI_GRO_CB(skb)->age = jiffies;
4802         NAPI_GRO_CB(skb)->last = skb;
4803         skb_shinfo(skb)->gso_size = skb_gro_len(skb);
4804         skb->next = napi->gro_list;
4805         napi->gro_list = skb;
4806         ret = GRO_HELD;
4807 
4808 pull:
4809         grow = skb_gro_offset(skb) - skb_headlen(skb);
4810         if (grow > 0)
4811                 gro_pull_from_frag0(skb, grow);
4812 ok:
4813         return ret;
4814 
4815 normal:
4816         ret = GRO_NORMAL;
4817         goto pull;
4818 }
4819 
4820 struct packet_offload *gro_find_receive_by_type(__be16 type)
4821 {
4822         struct list_head *offload_head = &offload_base;
4823         struct packet_offload *ptype;
4824 
4825         list_for_each_entry_rcu(ptype, offload_head, list) {
4826                 if (ptype->type != type || !ptype->callbacks.gro_receive)
4827                         continue;
4828                 return ptype;
4829         }
4830         return NULL;
4831 }
4832 EXPORT_SYMBOL(gro_find_receive_by_type);
4833 
4834 struct packet_offload *gro_find_complete_by_type(__be16 type)
4835 {
4836         struct list_head *offload_head = &offload_base;
4837         struct packet_offload *ptype;
4838 
4839         list_for_each_entry_rcu(ptype, offload_head, list) {
4840                 if (ptype->type != type || !ptype->callbacks.gro_complete)
4841                         continue;
4842                 return ptype;
4843         }
4844         return NULL;
4845 }
4846 EXPORT_SYMBOL(gro_find_complete_by_type);
4847 
4848 static void napi_skb_free_stolen_head(struct sk_buff *skb)
4849 {
4850         skb_dst_drop(skb);
4851         secpath_reset(skb);
4852         kmem_cache_free(skbuff_head_cache, skb);
4853 }
4854 
4855 static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
4856 {
4857         switch (ret) {
4858         case GRO_NORMAL:
4859                 if (netif_receive_skb_internal(skb))
4860                         ret = GRO_DROP;
4861                 break;
4862 
4863         case GRO_DROP:
4864                 kfree_skb(skb);
4865                 break;
4866 
4867         case GRO_MERGED_FREE:
4868                 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
4869                         napi_skb_free_stolen_head(skb);
4870                 else
4871                         __kfree_skb(skb);
4872                 break;
4873 
4874         case GRO_HELD:
4875         case GRO_MERGED:
4876         case GRO_CONSUMED:
4877                 break;
4878         }
4879 
4880         return ret;
4881 }
4882 
4883 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4884 {
4885         skb_mark_napi_id(skb, napi);
4886         trace_napi_gro_receive_entry(skb);
4887 
4888         skb_gro_reset_offset(skb);
4889 
4890         return napi_skb_finish(dev_gro_receive(napi, skb), skb);
4891 }
4892 EXPORT_SYMBOL(napi_gro_receive);
4893 
4894 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
4895 {
4896         if (unlikely(skb->pfmemalloc)) {
4897                 consume_skb(skb);
4898                 return;
4899         }
4900         __skb_pull(skb, skb_headlen(skb));
4901         /* restore the reserve we had after netdev_alloc_skb_ip_align() */
4902         skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
4903         skb->vlan_tci = 0;
4904         skb->dev = napi->dev;
4905         skb->skb_iif = 0;
4906         skb->encapsulation = 0;
4907         skb_shinfo(skb)->gso_type = 0;
4908         skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
4909         secpath_reset(skb);
4910 
4911         napi->skb = skb;
4912 }
4913 
4914 struct sk_buff *napi_get_frags(struct napi_struct *napi)
4915 {
4916         struct sk_buff *skb = napi->skb;
4917 
4918         if (!skb) {
4919                 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
4920                 if (skb) {
4921                         napi->skb = skb;
4922                         skb_mark_napi_id(skb, napi);
4923                 }
4924         }
4925         return skb;
4926 }
4927 EXPORT_SYMBOL(napi_get_frags);
4928 
4929 static gro_result_t napi_frags_finish(struct napi_struct *napi,
4930                                       struct sk_buff *skb,
4931                                       gro_result_t ret)
4932 {
4933         switch (ret) {
4934         case GRO_NORMAL:
4935         case GRO_HELD:
4936                 __skb_push(skb, ETH_HLEN);
4937                 skb->protocol = eth_type_trans(skb, skb->dev);
4938                 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
4939                         ret = GRO_DROP;
4940                 break;
4941 
4942         case GRO_DROP:
4943                 napi_reuse_skb(napi, skb);
4944                 break;
4945 
4946         case GRO_MERGED_FREE:
4947                 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
4948                         napi_skb_free_stolen_head(skb);
4949                 else
4950                         napi_reuse_skb(napi, skb);
4951                 break;
4952 
4953         case GRO_MERGED:
4954         case GRO_CONSUMED:
4955                 break;
4956         }
4957 
4958         return ret;
4959 }
4960 
4961 /* Upper GRO stack assumes network header starts at gro_offset=0
4962  * Drivers could call both napi_gro_frags() and napi_gro_receive()
4963  * We copy ethernet header into skb->data to have a common layout.
4964  */
4965 static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
4966 {
4967         struct sk_buff *skb = napi->skb;
4968         const struct ethhdr *eth;
4969         unsigned int hlen = sizeof(*eth);
4970 
4971         napi->skb = NULL;
4972 
4973         skb_reset_mac_header(skb);
4974         skb_gro_reset_offset(skb);
4975 
4976         eth = skb_gro_header_fast(skb, 0);
4977         if (unlikely(skb_gro_header_hard(skb, hlen))) {
4978                 eth = skb_gro_header_slow(skb, hlen, 0);
4979                 if (unlikely(!eth)) {
4980                         net_warn_ratelimited("%s: dropping impossible skb from %s\n",
4981                                              __func__, napi->dev->name);
4982                         napi_reuse_skb(napi, skb);
4983                         return NULL;
4984                 }
4985         } else {
4986                 gro_pull_from_frag0(skb, hlen);
4987                 NAPI_GRO_CB(skb)->frag0 += hlen;
4988                 NAPI_GRO_CB(skb)->frag0_len -= hlen;
4989         }
4990         __skb_pull(skb, hlen);
4991 
4992         /*
4993          * This works because the only protocols we care about don't require
4994          * special handling.
4995          * We'll fix it up properly in napi_frags_finish()
4996          */
4997         skb->protocol = eth->h_proto;
4998 
4999         return skb;
5000 }
5001 
5002 gro_result_t napi_gro_frags(struct napi_struct *napi)
5003 {
5004         struct sk_buff *skb = napi_frags_skb(napi);
5005 
5006         if (!skb)
5007                 return GRO_DROP;
5008 
5009         trace_napi_gro_frags_entry(skb);
5010 
5011         return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
5012 }
5013 EXPORT_SYMBOL(napi_gro_frags);
5014 
5015 /* Compute the checksum from gro_offset and return the folded value
5016  * after adding in any pseudo checksum.
5017  */
5018 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
5019 {
5020         __wsum wsum;
5021         __sum16 sum;
5022 
5023         wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
5024 
5025         /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
5026         sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
5027         if (likely(!sum)) {
5028                 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
5029                     !skb->csum_complete_sw)
5030                         netdev_rx_csum_fault(skb->dev);
5031         }
5032 
5033         NAPI_GRO_CB(skb)->csum = wsum;
5034         NAPI_GRO_CB(skb)->csum_valid = 1;
5035 
5036         return sum;
5037 }
5038 EXPORT_SYMBOL(__skb_gro_checksum_complete);
5039 
5040 static void net_rps_send_ipi(struct softnet_data *remsd)
5041 {
5042 #ifdef CONFIG_RPS
5043         while (remsd) {
5044                 struct softnet_data *next = remsd->rps_ipi_next;
5045 
5046                 if (cpu_online(remsd->cpu))
5047                         smp_call_function_single_async(remsd->cpu, &remsd->csd);
5048                 remsd = next;
5049         }
5050 #endif
5051 }
5052 
5053 /*
5054  * net_rps_action_and_irq_enable sends any pending IPI's for rps.
5055  * Note: called with local irq disabled, but exits with local irq enabled.
5056  */
5057 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
5058 {
5059 #ifdef CONFIG_RPS
5060         struct softnet_data *remsd = sd->rps_ipi_list;
5061 
5062         if (remsd) {
5063                 sd->rps_ipi_list = NULL;
5064 
5065                 local_irq_enable();
5066 
5067                 /* Send pending IPI's to kick RPS processing on remote cpus. */
5068                 net_rps_send_ipi(remsd);
5069         } else
5070 #endif
5071                 local_irq_enable();
5072 }
5073 
5074 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
5075 {
5076 #ifdef CONFIG_RPS
5077         return sd->rps_ipi_list != NULL;
5078 #else
5079         return false;
5080 #endif
5081 }
5082 
5083 static int process_backlog(struct napi_struct *napi, int quota)
5084 {
5085         struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
5086         bool again = true;
5087         int work = 0;
5088 
5089         /* Check if we have pending ipi, its better to send them now,
5090          * not waiting net_rx_action() end.
5091          */
5092         if (sd_has_rps_ipi_waiting(sd)) {
5093                 local_irq_disable();
5094                 net_rps_action_and_irq_enable(sd);
5095         }
5096 
5097         napi->weight = dev_rx_weight;
5098         while (again) {
5099                 struct sk_buff *skb;
5100 
5101                 while ((skb = __skb_dequeue(&sd->process_queue))) {
5102                         rcu_read_lock();
5103                         __netif_receive_skb(skb);
5104                         rcu_read_unlock();
5105                         input_queue_head_incr(sd);
5106                         if (++work >= quota)
5107                                 return work;
5108 
5109                 }
5110 
5111                 local_irq_disable();
5112                 rps_lock(sd);
5113                 if (skb_queue_empty(&sd->input_pkt_queue)) {
5114                         /*
5115                          * Inline a custom version of __napi_complete().
5116                          * only current cpu owns and manipulates this napi,
5117                          * and NAPI_STATE_SCHED is the only possible flag set
5118                          * on backlog.
5119                          * We can use a plain write instead of clear_bit(),
5120                          * and we dont need an smp_mb() memory barrier.
5121                          */
5122                         napi->state = 0;
5123                         again = false;
5124                 } else {
5125                         skb_queue_splice_tail_init(&sd->input_pkt_queue,
5126                                                    &sd->process_queue);
5127                 }
5128                 rps_unlock(sd);
5129                 local_irq_enable();
5130         }
5131 
5132         return work;
5133 }
5134 
5135 /**
5136  * __napi_schedule - schedule for receive
5137  * @n: entry to schedule
5138  *
5139  * The entry's receive function will be scheduled to run.
5140  * Consider using __napi_schedule_irqoff() if hard irqs are masked.
5141  */
5142 void __napi_schedule(struct napi_struct *n)
5143 {
5144         unsigned long flags;
5145 
5146         local_irq_save(flags);
5147         ____napi_schedule(this_cpu_ptr(&softnet_data), n);
5148         local_irq_restore(flags);
5149 }
5150 EXPORT_SYMBOL(__napi_schedule);
5151 
5152 /**
5153  *      napi_schedule_prep - check if napi can be scheduled
5154  *      @n: napi context
5155  *
5156  * Test if NAPI routine is already running, and if not mark
5157  * it as running.  This is used as a condition variable
5158  * insure only one NAPI poll instance runs.  We also make
5159  * sure there is no pending NAPI disable.
5160  */
5161 bool napi_schedule_prep(struct napi_struct *n)
5162 {
5163         unsigned long val, new;
5164 
5165         do {
5166                 val = READ_ONCE(n->state);
5167                 if (unlikely(val & NAPIF_STATE_DISABLE))
5168                         return false;
5169                 new = val | NAPIF_STATE_SCHED;
5170 
5171                 /* Sets STATE_MISSED bit if STATE_SCHED was already set
5172                  * This was suggested by Alexander Duyck, as compiler
5173                  * emits better code than :
5174                  * if (val & NAPIF_STATE_SCHED)
5175                  *     new |= NAPIF_STATE_MISSED;
5176                  */
5177                 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
5178                                                    NAPIF_STATE_MISSED;
5179         } while (cmpxchg(&n->state, val, new) != val);
5180 
5181         return !(val & NAPIF_STATE_SCHED);
5182 }
5183 EXPORT_SYMBOL(napi_schedule_prep);
5184 
5185 /**
5186  * __napi_schedule_irqoff - schedule for receive
5187  * @n: entry to schedule
5188  *
5189  * Variant of __napi_schedule() assuming hard irqs are masked
5190  */
5191 void __napi_schedule_irqoff(struct napi_struct *n)
5192 {
5193         ____napi_schedule(this_cpu_ptr(&softnet_data), n);
5194 }
5195 EXPORT_SYMBOL(__napi_schedule_irqoff);
5196 
5197 bool napi_complete_done(struct napi_struct *n, int work_done)
5198 {
5199         unsigned long flags, val, new;
5200 
5201         /*
5202          * 1) Don't let napi dequeue from the cpu poll list
5203          *    just in case its running on a different cpu.
5204          * 2) If we are busy polling, do nothing here, we have
5205          *    the guarantee we will be called later.
5206          */
5207         if (unlikely(n->state & (NAPIF_STATE_NPSVC |
5208                                  NAPIF_STATE_IN_BUSY_POLL)))
5209                 return false;
5210 
5211         if (n->gro_list) {
5212                 unsigned long timeout = 0;
5213 
5214                 if (work_done)
5215                         timeout = n->dev->gro_flush_timeout;
5216 
5217                 if (timeout)
5218                         hrtimer_start(&n->timer, ns_to_ktime(timeout),
5219                                       HRTIMER_MODE_REL_PINNED);
5220                 else
5221                         napi_gro_flush(n, false);
5222         }
5223         if (unlikely(!list_empty(&n->poll_list))) {
5224                 /* If n->poll_list is not empty, we need to mask irqs */
5225                 local_irq_save(flags);
5226                 list_del_init(&n->poll_list);
5227                 local_irq_restore(flags);
5228         }
5229 
5230         do {
5231                 val = READ_ONCE(n->state);
5232 
5233                 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
5234 
5235                 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED);
5236 
5237                 /* If STATE_MISSED was set, leave STATE_SCHED set,
5238                  * because we will call napi->poll() one more time.
5239                  * This C code was suggested by Alexander Duyck to help gcc.
5240                  */
5241                 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
5242                                                     NAPIF_STATE_SCHED;
5243         } while (cmpxchg(&n->state, val, new) != val);
5244 
5245         if (unlikely(val & NAPIF_STATE_MISSED)) {
5246                 __napi_schedule(n);
5247                 return false;
5248         }
5249 
5250         return true;
5251 }
5252 EXPORT_SYMBOL(napi_complete_done);
5253 
5254 /* must be called under rcu_read_lock(), as we dont take a reference */
5255 static struct napi_struct *napi_by_id(unsigned int napi_id)
5256 {
5257         unsigned int hash = napi_id % HASH_SIZE(napi_hash);
5258         struct napi_struct *napi;
5259 
5260         hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
5261                 if (napi->napi_id == napi_id)
5262                         return napi;
5263 
5264         return NULL;
5265 }
5266 
5267 #if defined(CONFIG_NET_RX_BUSY_POLL)
5268 
5269 #define BUSY_POLL_BUDGET 8
5270 
5271 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock)
5272 {
5273         int rc;
5274 
5275         /* Busy polling means there is a high chance device driver hard irq
5276          * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
5277          * set in napi_schedule_prep().
5278          * Since we are about to call napi->poll() once more, we can safely
5279          * clear NAPI_STATE_MISSED.
5280          *
5281          * Note: x86 could use a single "lock and ..." instruction
5282          * to perform these two clear_bit()
5283          */
5284         clear_bit(NAPI_STATE_MISSED, &napi->state);
5285         clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
5286 
5287         local_bh_disable();
5288 
5289         /* All we really want here is to re-enable device interrupts.
5290          * Ideally, a new ndo_busy_poll_stop() could avoid another round.
5291          */
5292         rc = napi->poll(napi, BUSY_POLL_BUDGET);
5293         trace_napi_poll(napi, rc, BUSY_POLL_BUDGET);
5294         netpoll_poll_unlock(have_poll_lock);
5295         if (rc == BUSY_POLL_BUDGET)
5296                 __napi_schedule(napi);
5297         local_bh_enable();
5298 }
5299 
5300 void napi_busy_loop(unsigned int napi_id,
5301                     bool (*loop_end)(void *, unsigned long),
5302                     void *loop_end_arg)
5303 {
5304         unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
5305         int (*napi_poll)(struct napi_struct *napi, int budget);
5306         void *have_poll_lock = NULL;
5307         struct napi_struct *napi;
5308 
5309 restart:
5310         napi_poll = NULL;
5311 
5312         rcu_read_lock();
5313 
5314         napi = napi_by_id(napi_id);
5315         if (!napi)
5316                 goto out;
5317 
5318         preempt_disable();
5319         for (;;) {
5320                 int work = 0;
5321 
5322                 local_bh_disable();
5323                 if (!napi_poll) {
5324                         unsigned long val = READ_ONCE(napi->state);
5325 
5326                         /* If multiple threads are competing for this napi,
5327                          * we avoid dirtying napi->state as much as we can.
5328                          */
5329                         if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
5330                                    NAPIF_STATE_IN_BUSY_POLL))
5331                                 goto count;
5332                         if (cmpxchg(&napi->state, val,
5333                                     val | NAPIF_STATE_IN_BUSY_POLL |
5334                                           NAPIF_STATE_SCHED) != val)
5335                                 goto count;
5336                         have_poll_lock = netpoll_poll_lock(napi);
5337                         napi_poll = napi->poll;
5338                 }
5339                 work = napi_poll(napi, BUSY_POLL_BUDGET);
5340                 trace_napi_poll(napi, work, BUSY_POLL_BUDGET);
5341 count:
5342                 if (work > 0)
5343                         __NET_ADD_STATS(dev_net(napi->dev),
5344                                         LINUX_MIB_BUSYPOLLRXPACKETS, work);
5345                 local_bh_enable();
5346 
5347                 if (!loop_end || loop_end(loop_end_arg, start_time))
5348                         break;
5349 
5350                 if (unlikely(need_resched())) {
5351                         if (napi_poll)
5352                                 busy_poll_stop(napi, have_poll_lock);
5353                         preempt_enable();
5354                         rcu_read_unlock();
5355                         cond_resched();
5356                         if (loop_end(loop_end_arg, start_time))
5357                                 return;
5358                         goto restart;
5359                 }
5360                 cpu_relax();
5361         }
5362         if (napi_poll)
5363                 busy_poll_stop(napi, have_poll_lock);
5364         preempt_enable();
5365 out:
5366         rcu_read_unlock();
5367 }
5368 EXPORT_SYMBOL(napi_busy_loop);
5369 
5370 #endif /* CONFIG_NET_RX_BUSY_POLL */
5371 
5372 static void napi_hash_add(struct napi_struct *napi)
5373 {
5374         if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) ||
5375             test_and_set_bit(NAPI_STATE_HASHED, &napi->state))
5376                 return;
5377 
5378         spin_lock(&napi_hash_lock);
5379 
5380         /* 0..NR_CPUS range is reserved for sender_cpu use */
5381         do {
5382                 if (unlikely(++napi_gen_id < MIN_NAPI_ID))
5383                         napi_gen_id = MIN_NAPI_ID;
5384         } while (napi_by_id(napi_gen_id));
5385         napi->napi_id = napi_gen_id;
5386 
5387         hlist_add_head_rcu(&napi->napi_hash_node,
5388                            &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
5389 
5390         spin_unlock(&napi_hash_lock);
5391 }
5392 
5393 /* Warning : caller is responsible to make sure rcu grace period
5394  * is respected before freeing memory containing @napi
5395  */
5396 bool napi_hash_del(struct napi_struct *napi)
5397 {
5398         bool rcu_sync_needed = false;
5399 
5400         spin_lock(&napi_hash_lock);
5401 
5402         if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) {
5403                 rcu_sync_needed = true;
5404                 hlist_del_rcu(&napi->napi_hash_node);
5405         }
5406         spin_unlock(&napi_hash_lock);
5407         return rcu_sync_needed;
5408 }
5409 EXPORT_SYMBOL_GPL(napi_hash_del);
5410 
5411 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
5412 {
5413         struct napi_struct *napi;
5414 
5415         napi = container_of(timer, struct napi_struct, timer);
5416 
5417         /* Note : we use a relaxed variant of napi_schedule_prep() not setting
5418          * NAPI_STATE_MISSED, since we do not react to a device IRQ.
5419          */
5420         if (napi->gro_list && !napi_disable_pending(napi) &&
5421             !test_and_set_bit(NAPI_STATE_SCHED, &napi->state))
5422                 __napi_schedule_irqoff(napi);
5423 
5424         return HRTIMER_NORESTART;
5425 }
5426 
5427 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
5428                     int (*poll)(struct napi_struct *, int), int weight)
5429 {
5430         INIT_LIST_HEAD(&napi->poll_list);
5431         hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
5432         napi->timer.function = napi_watchdog;
5433         napi->gro_count = 0;
5434         napi->gro_list = NULL;
5435         napi->skb = NULL;
5436         napi->poll = poll;
5437         if (weight > NAPI_POLL_WEIGHT)
5438                 pr_err_once("netif_napi_add() called with weight %d on device %s\n",
5439                             weight, dev->name);
5440         napi->weight = weight;
5441         list_add(&napi->dev_list, &dev->napi_list);
5442         napi->dev = dev;
5443 #ifdef CONFIG_NETPOLL
5444         napi->poll_owner = -1;
5445 #endif
5446         set_bit(NAPI_STATE_SCHED, &napi->state);
5447         napi_hash_add(napi);
5448 }
5449 EXPORT_SYMBOL(netif_napi_add);
5450 
5451 void napi_disable(struct napi_struct *n)
5452 {
5453         might_sleep();
5454         set_bit(NAPI_STATE_DISABLE, &n->state);
5455 
5456         while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
5457                 msleep(1);
5458         while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
5459                 msleep(1);
5460 
5461         hrtimer_cancel(&n->timer);
5462 
5463         clear_bit(NAPI_STATE_DISABLE, &n->state);
5464 }
5465 EXPORT_SYMBOL(napi_disable);
5466 
5467 /* Must be called in process context */
5468 void netif_napi_del(struct napi_struct *napi)
5469 {
5470         might_sleep();
5471         if (napi_hash_del(napi))
5472                 synchronize_net();
5473         list_del_init(&napi->dev_list);
5474         napi_free_frags(napi);
5475 
5476         kfree_skb_list(napi->gro_list);
5477         napi->gro_list = NULL;
5478         napi->gro_count = 0;
5479 }
5480 EXPORT_SYMBOL(netif_napi_del);
5481 
5482 static int napi_poll(struct napi_struct *n, struct list_head *repoll)
5483 {
5484         void *have;
5485         int work, weight;
5486 
5487         list_del_init(&n->poll_list);
5488 
5489         have = netpoll_poll_lock(n);
5490 
5491         weight = n->weight;
5492 
5493         /* This NAPI_STATE_SCHED test is for avoiding a race
5494          * with netpoll's poll_napi().  Only the entity which
5495          * obtains the lock and sees NAPI_STATE_SCHED set will
5496          * actually make the ->poll() call.  Therefore we avoid
5497          * accidentally calling ->poll() when NAPI is not scheduled.
5498          */
5499         work = 0;
5500         if (test_bit(NAPI_STATE_SCHED, &n->state)) {
5501                 work = n->poll(n, weight);
5502                 trace_napi_poll(n, work, weight);
5503         }
5504 
5505         WARN_ON_ONCE(work > weight);
5506 
5507         if (likely(work < weight))
5508                 goto out_unlock;
5509 
5510         /* Drivers must not modify the NAPI state if they
5511          * consume the entire weight.  In such cases this code
5512          * still "owns" the NAPI instance and therefore can
5513          * move the instance around on the list at-will.
5514          */
5515         if (unlikely(napi_disable_pending(n))) {
5516                 napi_complete(n);
5517                 goto out_unlock;
5518         }
5519 
5520         if (n->gro_list) {
5521                 /* flush too old packets
5522                  * If HZ < 1000, flush all packets.
5523                  */
5524                 napi_gro_flush(n, HZ >= 1000);
5525         }
5526 
5527         /* Some drivers may have called napi_schedule
5528          * prior to exhausting their budget.
5529          */
5530         if (unlikely(!list_empty(&n->poll_list))) {
5531                 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
5532                              n->dev ? n->dev->name : "backlog");
5533                 goto out_unlock;
5534         }
5535 
5536         list_add_tail(&n->poll_list, repoll);
5537 
5538 out_unlock:
5539         netpoll_poll_unlock(have);
5540 
5541         return work;
5542 }
5543 
5544 static __latent_entropy void net_rx_action(struct softirq_action *h)
5545 {
5546         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
5547         unsigned long time_limit = jiffies +
5548                 usecs_to_jiffies(netdev_budget_usecs);
5549         int budget = netdev_budget;
5550         LIST_HEAD(list);
5551         LIST_HEAD(repoll);
5552 
5553         local_irq_disable();
5554         list_splice_init(&sd->poll_list, &list);
5555         local_irq_enable();
5556 
5557         for (;;) {
5558                 struct napi_struct *n;
5559 
5560                 if (list_empty(&list)) {
5561                         if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
5562                                 goto out;
5563                         break;
5564                 }
5565 
5566                 n = list_first_entry(&list, struct napi_struct, poll_list);
5567                 budget -= napi_poll(n, &repoll);
5568 
5569                 /* If softirq window is exhausted then punt.
5570                  * Allow this to run for 2 jiffies since which will allow
5571                  * an average latency of 1.5/HZ.
5572                  */
5573                 if (unlikely(budget <= 0 ||
5574                              time_after_eq(jiffies, time_limit))) {
5575                         sd->time_squeeze++;
5576                         break;
5577                 }
5578         }
5579 
5580         local_irq_disable();
5581 
5582         list_splice_tail_init(&sd->poll_list, &list);
5583         list_splice_tail(&repoll, &list);
5584         list_splice(&list, &sd->poll_list);
5585         if (!list_empty(&sd->poll_list))
5586                 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
5587 
5588         net_rps_action_and_irq_enable(sd);
5589 out:
5590         __kfree_skb_flush();
5591 }
5592 
5593 struct netdev_adjacent {
5594         struct net_device *dev;
5595 
5596         /* upper master flag, there can only be one master device per list */
5597         bool master;
5598 
5599         /* counter for the number of times this device was added to us */
5600         u16 ref_nr;
5601 
5602         /* private field for the users */
5603         void *private;
5604 
5605         struct list_head list;
5606         struct rcu_head rcu;
5607 };
5608 
5609 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
5610                                                  struct list_head *adj_list)
5611 {
5612         struct netdev_adjacent *adj;
5613 
5614         list_for_each_entry(adj, adj_list, list) {
5615                 if (adj->dev == adj_dev)
5616                         return adj;
5617         }
5618         return NULL;
5619 }
5620 
5621 static int __netdev_has_upper_dev(struct net_device *upper_dev, void *data)
5622 {
5623         struct net_device *dev = data;
5624 
5625         return upper_dev == dev;
5626 }
5627 
5628 /**
5629  * netdev_has_upper_dev - Check if device is linked to an upper device
5630  * @dev: device
5631  * @upper_dev: upper device to check
5632  *
5633  * Find out if a device is linked to specified upper device and return true
5634  * in case it is. Note that this checks only immediate upper device,
5635  * not through a complete stack of devices. The caller must hold the RTNL lock.
5636  */
5637 bool netdev_has_upper_dev(struct net_device *dev,
5638                           struct net_device *upper_dev)
5639 {
5640         ASSERT_RTNL();
5641 
5642         return netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
5643                                              upper_dev);
5644 }
5645 EXPORT_SYMBOL(netdev_has_upper_dev);
5646 
5647 /**
5648  * netdev_has_upper_dev_all - Check if device is linked to an upper device
5649  * @dev: device
5650  * @upper_dev: upper device to check
5651  *
5652  * Find out if a device is linked to specified upper device and return true
5653  * in case it is. Note that this checks the entire upper device chain.
5654  * The caller must hold rcu lock.
5655  */
5656 
5657 bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
5658                                   struct net_device *upper_dev)
5659 {
5660         return !!netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
5661                                                upper_dev);
5662 }
5663 EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
5664 
5665 /**
5666  * netdev_has_any_upper_dev - Check if device is linked to some device
5667  * @dev: device
5668  *
5669  * Find out if a device is linked to an upper device and return true in case
5670  * it is. The caller must hold the RTNL lock.
5671  */
5672 bool netdev_has_any_upper_dev(struct net_device *dev)
5673 {
5674         ASSERT_RTNL();
5675 
5676         return !list_empty(&dev->adj_list.upper);
5677 }
5678 EXPORT_SYMBOL(netdev_has_any_upper_dev);
5679 
5680 /**
5681  * netdev_master_upper_dev_get - Get master upper device
5682  * @dev: device
5683  *
5684  * Find a master upper device and return pointer to it or NULL in case
5685  * it's not there. The caller must hold the RTNL lock.
5686  */
5687 struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
5688 {
5689         struct netdev_adjacent *upper;
5690 
5691         ASSERT_RTNL();
5692 
5693         if (list_empty(&dev->adj_list.upper))
5694                 return NULL;
5695 
5696         upper = list_first_entry(&dev->adj_list.upper,
5697                                  struct netdev_adjacent, list);
5698         if (likely(upper->master))
5699                 return upper->dev;
5700         return NULL;
5701 }
5702 EXPORT_SYMBOL(netdev_master_upper_dev_get);
5703 
5704 /**
5705  * netdev_has_any_lower_dev - Check if device is linked to some device
5706  * @dev: device
5707  *
5708  * Find out if a device is linked to a lower device and return true in case
5709  * it is. The caller must hold the RTNL lock.
5710  */
5711 static bool netdev_has_any_lower_dev(struct net_device *dev)
5712 {
5713         ASSERT_RTNL();
5714 
5715         return !list_empty(&dev->adj_list.lower);
5716 }
5717 
5718 void *netdev_adjacent_get_private(struct list_head *adj_list)
5719 {
5720         struct netdev_adjacent *adj;
5721 
5722         adj = list_entry(adj_list, struct netdev_adjacent, list);
5723 
5724         return adj->private;
5725 }
5726 EXPORT_SYMBOL(netdev_adjacent_get_private);
5727 
5728 /**
5729  * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
5730  * @dev: device
5731  * @iter: list_head ** of the current position
5732  *
5733  * Gets the next device from the dev's upper list, starting from iter
5734  * position. The caller must hold RCU read lock.
5735  */
5736 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
5737                                                  struct list_head **iter)
5738 {
5739         struct netdev_adjacent *upper;
5740 
5741         WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5742 
5743         upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5744 
5745         if (&upper->list == &dev->adj_list.upper)
5746                 return NULL;
5747 
5748         *iter = &upper->list;
5749 
5750         return upper->dev;
5751 }
5752 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
5753 
5754 static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
5755                                                     struct list_head **iter)
5756 {
5757         struct netdev_adjacent *upper;
5758 
5759         WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5760 
5761         upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5762 
5763         if (&upper->list == &dev->adj_list.upper)
5764                 return NULL;
5765 
5766         *iter = &upper->list;
5767 
5768         return upper->dev;
5769 }
5770 
5771 int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
5772                                   int (*fn)(struct net_device *dev,
5773                                             void *data),
5774                                   void *data)
5775 {
5776         struct net_device *udev;
5777         struct list_head *iter;
5778         int ret;
5779 
5780         for (iter = &dev->adj_list.upper,
5781              udev = netdev_next_upper_dev_rcu(dev, &iter);
5782              udev;
5783              udev = netdev_next_upper_dev_rcu(dev, &iter)) {
5784                 /* first is the upper device itself */
5785                 ret = fn(udev, data);
5786                 if (ret)
5787                         return ret;
5788 
5789                 /* then look at all of its upper devices */
5790                 ret = netdev_walk_all_upper_dev_rcu(udev, fn, data);
5791                 if (ret)
5792                         return ret;
5793         }
5794 
5795         return 0;
5796 }
5797 EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
5798 
5799 /**
5800  * netdev_lower_get_next_private - Get the next ->private from the
5801  *                                 lower neighbour list
5802  * @dev: device
5803  * @iter: list_head ** of the current position
5804  *
5805  * Gets the next netdev_adjacent->private from the dev's lower neighbour
5806  * list, starting from iter position. The caller must hold either hold the
5807  * RTNL lock or its own locking that guarantees that the neighbour lower
5808  * list will remain unchanged.
5809  */
5810 void *netdev_lower_get_next_private(struct net_device *dev,
5811                                     struct list_head **iter)
5812 {
5813         struct netdev_adjacent *lower;
5814 
5815         lower = list_entry(*iter, struct netdev_adjacent, list);
5816 
5817         if (&lower->list == &dev->adj_list.lower)
5818                 return NULL;
5819 
5820         *iter = lower->list.next;
5821 
5822         return lower->private;
5823 }
5824 EXPORT_SYMBOL(netdev_lower_get_next_private);
5825 
5826 /**
5827  * netdev_lower_get_next_private_rcu - Get the next ->private from the
5828  *                                     lower neighbour list, RCU
5829  *                                     variant
5830  * @dev: device
5831  * @iter: list_head ** of the current position
5832  *
5833  * Gets the next netdev_adjacent->private from the dev's lower neighbour
5834  * list, starting from iter position. The caller must hold RCU read lock.
5835  */
5836 void *netdev_lower_get_next_private_rcu(struct net_device *dev,
5837                                         struct list_head **iter)
5838 {
5839         struct netdev_adjacent *lower;
5840 
5841         WARN_ON_ONCE(!rcu_read_lock_held());
5842 
5843         lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5844 
5845         if (&lower->list == &dev->adj_list.lower)
5846                 return NULL;
5847 
5848         *iter = &lower->list;
5849 
5850         return lower->private;
5851 }
5852 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
5853 
5854 /**
5855  * netdev_lower_get_next - Get the next device from the lower neighbour
5856  *                         list
5857  * @dev: device
5858  * @iter: list_head ** of the current position
5859  *
5860  * Gets the next netdev_adjacent from the dev's lower neighbour
5861  * list, starting from iter position. The caller must hold RTNL lock or
5862  * its own locking that guarantees that the neighbour lower
5863  * list will remain unchanged.
5864  */
5865 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
5866 {
5867         struct netdev_adjacent *lower;
5868 
5869         lower = list_entry(*iter, struct netdev_adjacent, list);
5870 
5871         if (&lower->list == &dev->adj_list.lower)
5872                 return NULL;
5873 
5874         *iter = lower->list.next;
5875 
5876         return lower->dev;
5877 }
5878 EXPORT_SYMBOL(netdev_lower_get_next);
5879 
5880 static struct net_device *netdev_next_lower_dev(struct net_device *dev,
5881                                                 struct list_head **iter)
5882 {
5883         struct netdev_adjacent *lower;
5884 
5885         lower = list_entry((*iter)->next, struct netdev_adjacent, list);
5886 
5887         if (&lower->list == &dev->adj_list.lower)
5888                 return NULL;
5889 
5890         *iter = &lower->list;
5891 
5892         return lower->dev;
5893 }
5894 
5895 int netdev_walk_all_lower_dev(struct net_device *dev,
5896                               int (*fn)(struct net_device *dev,
5897                                         void *data),
5898                               void *data)
5899 {
5900         struct net_device *ldev;
5901         struct list_head *iter;
5902         int ret;
5903 
5904         for (iter = &dev->adj_list.lower,
5905              ldev = netdev_next_lower_dev(dev, &iter);
5906              ldev;
5907              ldev = netdev_next_lower_dev(dev, &iter)) {
5908                 /* first is the lower device itself */
5909                 ret = fn(ldev, data);
5910                 if (ret)
5911                         return ret;
5912 
5913                 /* then look at all of its lower devices */
5914                 ret = netdev_walk_all_lower_dev(ldev, fn, data);
5915                 if (ret)
5916                         return ret;
5917         }
5918 
5919         return 0;
5920 }
5921 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
5922 
5923 static struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
5924                                                     struct list_head **iter)
5925 {
5926         struct netdev_adjacent *lower;
5927 
5928         lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5929         if (&lower->list == &dev->adj_list.lower)
5930                 return NULL;
5931 
5932         *iter = &lower->list;
5933 
5934         return lower->dev;
5935 }
5936 
5937 int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
5938                                   int (*fn)(struct net_device *dev,
5939                                             void *data),
5940                                   void *data)
5941 {
5942         struct net_device *ldev;
5943         struct list_head *iter;
5944         int ret;
5945 
5946         for (iter = &dev->adj_list.lower,
5947              ldev = netdev_next_lower_dev_rcu(dev, &iter);
5948              ldev;
5949              ldev = netdev_next_lower_dev_rcu(dev, &iter)) {
5950                 /* first is the lower device itself */
5951                 ret = fn(ldev, data);
5952                 if (ret)
5953                         return ret;
5954 
5955                 /* then look at all of its lower devices */
5956                 ret = netdev_walk_all_lower_dev_rcu(ldev, fn, data);
5957                 if (ret)
5958                         return ret;
5959         }
5960 
5961         return 0;
5962 }
5963 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
5964 
5965 /**
5966  * netdev_lower_get_first_private_rcu - Get the first ->private from the
5967  *                                     lower neighbour list, RCU
5968  *                                     variant
5969  * @dev: device
5970  *
5971  * Gets the first netdev_adjacent->private from the dev's lower neighbour
5972  * list. The caller must hold RCU read lock.
5973  */
5974 void *netdev_lower_get_first_private_rcu(struct net_device *dev)
5975 {
5976         struct netdev_adjacent *lower;
5977 
5978         lower = list_first_or_null_rcu(&dev->adj_list.lower,
5979                         struct netdev_adjacent, list);
5980         if (lower)
5981                 return lower->private;
5982         return NULL;
5983 }
5984 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
5985 
5986 /**
5987  * netdev_master_upper_dev_get_rcu - Get master upper device
5988  * @dev: device
5989  *
5990  * Find a master upper device and return pointer to it or NULL in case
5991  * it's not there. The caller must hold the RCU read lock.
5992  */
5993 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
5994 {
5995         struct netdev_adjacent *upper;
5996 
5997         upper = list_first_or_null_rcu(&dev->adj_list.upper,
5998                                        struct netdev_adjacent, list);
5999         if (upper && likely(upper->master))
6000                 return upper->dev;
6001         return NULL;
6002 }
6003 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
6004 
6005 static int netdev_adjacent_sysfs_add(struct net_device *dev,
6006                               struct net_device *adj_dev,
6007                               struct list_head *dev_list)
6008 {
6009         char linkname[IFNAMSIZ+7];
6010 
6011         sprintf(linkname, dev_list == &dev->adj_list.upper ?
6012                 "upper_%s" : "lower_%s", adj_dev->name);
6013         return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
6014                                  linkname);
6015 }
6016 static void netdev_adjacent_sysfs_del(struct net_device *dev,
6017                                char *name,
6018                                struct list_head *dev_list)
6019 {
6020         char linkname[IFNAMSIZ+7];
6021 
6022         sprintf(linkname, dev_list == &dev->adj_list.upper ?
6023                 "upper_%s" : "lower_%s", name);
6024         sysfs_remove_link(&(dev->dev.kobj), linkname);
6025 }
6026 
6027 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
6028                                                  struct net_device *adj_dev,
6029                                                  struct list_head *dev_list)
6030 {
6031         return (dev_list == &dev->adj_list.upper ||
6032                 dev_list == &dev->adj_list.lower) &&
6033                 net_eq(dev_net(dev), dev_net(adj_dev));
6034 }
6035 
6036 static int __netdev_adjacent_dev_insert(struct net_device *dev,
6037                                         struct net_device *adj_dev,
6038                                         struct list_head *dev_list,
6039                                         void *private, bool master)
6040 {
6041         struct netdev_adjacent *adj;
6042         int ret;
6043 
6044         adj = __netdev_find_adj(adj_dev, dev_list);
6045 
6046         if (adj) {
6047                 adj->ref_nr += 1;
6048                 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
6049                          dev->name, adj_dev->name, adj->ref_nr);
6050 
6051                 return 0;
6052         }
6053 
6054         adj = kmalloc(sizeof(*adj), GFP_KERNEL);
6055         if (!adj)
6056                 return -ENOMEM;
6057 
6058         adj->dev = adj_dev;
6059         adj->master = master;
6060         adj->ref_nr = 1;
6061         adj->private = private;
6062         dev_hold(adj_dev);
6063 
6064         pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
6065                  dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
6066 
6067         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
6068                 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
6069                 if (ret)
6070                         goto free_adj;
6071         }
6072 
6073         /* Ensure that master link is always the first item in list. */
6074         if (master) {
6075                 ret = sysfs_create_link(&(dev->dev.kobj),
6076                                         &(adj_dev->dev.kobj), "master");
6077                 if (ret)
6078                         goto remove_symlinks;
6079 
6080                 list_add_rcu(&adj->list, dev_list);
6081         } else {
6082                 list_add_tail_rcu(&adj->list, dev_list);
6083         }
6084 
6085         return 0;
6086 
6087 remove_symlinks:
6088         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
6089                 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
6090 free_adj:
6091         kfree(adj);
6092         dev_put(adj_dev);
6093 
6094         return ret;
6095 }
6096 
6097 static void __netdev_adjacent_dev_remove(struct net_device *dev,
6098                                          struct net_device *adj_dev,
6099                                          u16 ref_nr,
6100                                          struct list_head *dev_list)
6101